wdpost: Handle skipped sectors correctly

api/test/window_post.go

@@ -192,7 +192,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 
 		// Drop the partition
 		err = parts[0].Sectors.ForEach(func(sid uint64) error {
-			return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(abi.SectorID{
+			return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkCorrupted(abi.SectorID{
 				Miner:  abi.ActorID(mid),
 				Number: abi.SectorNumber(sid),
 			}, true)

extern/sector-storage/ffiwrapper/verifier_cgo.go

@@ -32,14 +32,18 @@ func (sb *Sealer) GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID,
 	return ffi.GenerateWinningPoSt(minerID, privsectors, randomness)
 }
 
-func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.SectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, []abi.SectorID, []abi.SectorID, error) {
+func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.SectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, []abi.SectorID, error) {
 	randomness[31] &= 0x3f
 	privsectors, skipped, done, err := sb.pubSectorToPriv(ctx, minerID, sectorInfo, nil, abi.RegisteredSealProof.RegisteredWindowPoStProof)
 	if err != nil {
-		return nil, nil, nil, xerrors.Errorf("gathering sector info: %w", err)
+		return nil, nil, xerrors.Errorf("gathering sector info: %w", err)
 	}
 	defer done()
 
+	if len(skipped) > 0 {
+		return nil, skipped, xerrors.Errorf("pubSectorToPriv skipped some sectors")
+	}
+
 	proof, faulty, err := ffi.GenerateWindowPoSt(minerID, privsectors, randomness)
 
 	var faultyIDs []abi.SectorID
@@ -50,7 +54,7 @@ func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, s
 		})
 	}
 
-	return proof, skipped, faultyIDs, err
+	return proof, faultyIDs, err
 }
 
 func (sb *Sealer) pubSectorToPriv(ctx context.Context, mid abi.ActorID, sectorInfo []proof.SectorInfo, faults []abi.SectorNumber, rpt func(abi.RegisteredSealProof) (abi.RegisteredPoStProof, error)) (ffi.SortedPrivateSectorInfo, []abi.SectorID, func(), error) {

extern/sector-storage/mock/mock.go

@@ -68,6 +68,7 @@ const (
 type sectorState struct {
 	pieces []cid.Cid
 	failed bool
+	corrupted bool
 
 	state int
 
@@ -251,6 +252,18 @@ func (mgr *SectorMgr) MarkFailed(sid abi.SectorID, failed bool) error {
 	return nil
 }
 
+func (mgr *SectorMgr) MarkCorrupted(sid abi.SectorID, corrupted bool) error {
+	mgr.lk.Lock()
+	defer mgr.lk.Unlock()
+	ss, ok := mgr.sectors[sid]
+	if !ok {
+		return fmt.Errorf("no such sector in storage")
+	}
+
+	ss.corrupted = corrupted
+	return nil
+}
+
 func opFinishWait(ctx context.Context) {
 	val, ok := ctx.Value("opfinish").(chan struct{})
 	if !ok {
@@ -275,6 +288,8 @@ func (mgr *SectorMgr) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorI
 	si := make([]proof.SectorInfo, 0, len(sectorInfo))
 	var skipped []abi.SectorID
 
+	var err error
+
 	for _, info := range sectorInfo {
 		sid := abi.SectorID{
 			Miner:  minerID,
@@ -283,13 +298,18 @@ func (mgr *SectorMgr) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorI
 
 		_, found := mgr.sectors[sid]
 
-		if found && !mgr.sectors[sid].failed {
+		if found && !mgr.sectors[sid].failed && !mgr.sectors[sid].corrupted {
 			si = append(si, info)
 		} else {
 			skipped = append(skipped, sid)
+			err = xerrors.Errorf("skipped some sectors")
 		}
 	}
 
+	if err != nil {
+		return nil, skipped, err
+	}
+
 	return generateFakePoSt(si, abi.RegisteredSealProof.RegisteredWindowPoStProof, randomness), skipped, nil
 }
 

go.mod

@@ -140,5 +140,3 @@ replace github.com/filecoin-project/test-vectors => ./extern/test-vectors
 replace github.com/supranational/blst => ./extern/fil-blst/blst
 
 replace github.com/filecoin-project/fil-blst => ./extern/fil-blst
-
-replace github.com/filecoin-project/specs-storage => ../specs-storage

go.sum

@@ -250,6 +250,8 @@ github.com/filecoin-project/specs-actors v0.9.7 h1:7PAZ8kdqwBdmgf/23FCkQZLCXcVu0
 github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU=
 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk=
 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g=
+github.com/filecoin-project/specs-storage v0.1.1-0.20200909213410-c066548422be h1:UX457RrC0LwL7Bb5kd0WFyzJBxbHOCSw/64oYqeT+Zc=
+github.com/filecoin-project/specs-storage v0.1.1-0.20200909213410-c066548422be/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g=
 github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus=
 github.com/filecoin-project/test-vectors/schema v0.0.1/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=

storage/wdpost_run.go

@@ -335,96 +335,114 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo
 		Proofs:     nil,
 	}
 
-	var sinfos []proof.SectorInfo
-	sidToPart := map[abi.SectorNumber]uint64{}
 	skipCount := uint64(0)
+	postSkipped := bitfield.New()
+	var postOut []proof.PoStProof
 
-	for partIdx, partition := range partitions {
+	for retries := 0; retries < 5; retries++ {
-		// TODO: Can do this in parallel
+		var sinfos []proof.SectorInfo
-		toProve, err := partition.ActiveSectors()
+		sidToPart := map[abi.SectorNumber]int{}
+
+		for partIdx, partition := range partitions {
+			// TODO: Can do this in parallel
+			toProve, err := partition.ActiveSectors()
+			if err != nil {
+				return nil, xerrors.Errorf("getting active sectors: %w", err)
+			}
+
+			toProve, err = bitfield.MergeBitFields(toProve, partition.Recoveries)
+			if err != nil {
+				return nil, xerrors.Errorf("adding recoveries to set of sectors to prove: %w", err)
+			}
+
+			toProve, err = bitfield.SubtractBitField(toProve, postSkipped)
+			if err != nil {
+				return nil, xerrors.Errorf("toProve - postSkipped: %w", err)
+			}
+
+			good, err := s.checkSectors(ctx, toProve)
+			if err != nil {
+				return nil, xerrors.Errorf("checking sectors to skip: %w", err)
+			}
+
+			skipped, err := bitfield.SubtractBitField(toProve, good)
+			if err != nil {
+				return nil, xerrors.Errorf("toProve - good: %w", err)
+			}
+
+			sc, err := skipped.Count()
+			if err != nil {
+				return nil, xerrors.Errorf("getting skipped sector count: %w", err)
+			}
+
+			skipCount += sc
+
+			ssi, err := s.sectorsForProof(ctx, good, partition.Sectors, ts)
+			if err != nil {
+				return nil, xerrors.Errorf("getting sorted sector info: %w", err)
+			}
+
+			if len(ssi) == 0 {
+				continue
+			}
+
+			sinfos = append(sinfos, ssi...)
+			for _, si := range ssi {
+				sidToPart[si.SectorNumber] = partIdx
+			}
+
+			params.Partitions = append(params.Partitions, miner.PoStPartition{
+				Index:   uint64(partIdx),
+				Skipped: skipped,
+			})
+		}
+
+		if len(sinfos) == 0 {
+			// nothing to prove..
+			return nil, errNoPartitions
+		}
+
+		log.Infow("running windowPost",
+			"chain-random", rand,
+			"deadline", di,
+			"height", ts.Height(),
+			"skipped", skipCount)
+
+		tsStart := build.Clock.Now()
+
+		mid, err := address.IDFromAddress(s.actor)
 		if err != nil {
-			return nil, xerrors.Errorf("getting active sectors: %w", err)
+			return nil, err
 		}
 
-		toProve, err = bitfield.MergeBitFields(toProve, partition.Recoveries)
+		var ps []abi.SectorID
-		if err != nil {
+		postOut, ps, err = s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, abi.PoStRandomness(rand))
-			return nil, xerrors.Errorf("adding recoveries to set of sectors to prove: %w", err)
+		elapsed := time.Since(tsStart)
+
+		log.Infow("computing window PoSt", "elapsed", elapsed)
+
+		if err == nil {
+			break
 		}
 
-		good, err := s.checkSectors(ctx, toProve)
+		if len(ps) == 0 {
-		if err != nil {
+			return nil, xerrors.Errorf("running post failed: %w", err)
-			return nil, xerrors.Errorf("checking sectors to skip: %w", err)
 		}
 
-		skipped, err := bitfield.SubtractBitField(toProve, good)
+		log.Warnw("generate window PoSt skipped sectors", "sectors", ps, "error", err, "try", retries)
-		if err != nil {
+
-			return nil, xerrors.Errorf("toProve - good: %w", err)
+		skipCount += uint64(len(ps))
+		for _, sector := range ps {
+			postSkipped.Set(uint64(sector.Number))
 		}
-
-		sc, err := skipped.Count()
-		if err != nil {
-			return nil, xerrors.Errorf("getting skipped sector count: %w", err)
-		}
-
-		skipCount += sc
-
-		ssi, err := s.sectorsForProof(ctx, good, partition.Sectors, ts)
-		if err != nil {
-			return nil, xerrors.Errorf("getting sorted sector info: %w", err)
-		}
-
-		if len(ssi) == 0 {
-			continue
-		}
-
-		sinfos = append(sinfos, ssi...)
-		for _, si := range ssi {
-			sidToPart[si.SectorNumber] = uint64(partIdx)
-		}
-
-		params.Partitions = append(params.Partitions, miner.PoStPartition{
-			Index:   uint64(partIdx),
-			Skipped: skipped,
-		})
 	}
 
-	if len(sinfos) == 0 {
-		// nothing to prove..
-		return nil, errNoPartitions
-	}
-
-	log.Infow("running windowPost",
-		"chain-random", rand,
-		"deadline", di,
-		"height", ts.Height(),
-		"skipped", skipCount)
-
-	tsStart := build.Clock.Now()
-
-	mid, err := address.IDFromAddress(s.actor)
-	if err != nil {
-		return nil, err
-	}
-
-	postOut, postSkipped, faulty, err := s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, abi.PoStRandomness(rand))
-	if err != nil {
-		return nil, xerrors.Errorf("running post failed: %w", err)
-	}
-
-	_ = faulty // TODO(magik)
-
 	if len(postOut) == 0 {
-		return nil, xerrors.Errorf("received proofs back from generate window post")
+		return nil, xerrors.Errorf("received no proofs back from generate window post")
 	}
 
 	params.Proofs = postOut
 
-	for _, sector := range postSkipped {
-		params.Partitions[sidToPart[sector.Number]].Skipped.Set(uint64(sector.Number))
-	}
-
-	elapsed := time.Since(tsStart)
-
 	commEpoch := di.Open
 	commRand, err := s.api.ChainGetRandomnessFromTickets(ctx, ts.Key(), crypto.DomainSeparationTag_PoStChainCommit, commEpoch, nil)
 	if err != nil {
@@ -433,7 +451,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo
 	params.ChainCommitEpoch = commEpoch
 	params.ChainCommitRand = commRand
 
-	log.Infow("submitting window PoSt", "elapsed", elapsed)
+	log.Infow("submitting window PoSt")
 
 	return params, nil
 }