From 7a6ceebb34a95508acdef964519c75929d37cfc4 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 9 Sep 2020 14:00:15 -0700 Subject: [PATCH 1/6] windowed post generation now returns faulty sectors --- extern/filecoin-ffi | 2 +- .../sector-storage/ffiwrapper/verifier_cgo.go | 17 +++++++++++++---- go.mod | 8 ++++++-- storage/wdpost_run.go | 4 +++- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 405691046..79b324966 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 40569104603407c999d6c9e4c3f1228cbd4d0e5c +Subproject commit 79b3249666a809871fad12fbf50f68e22218b01c diff --git a/extern/sector-storage/ffiwrapper/verifier_cgo.go b/extern/sector-storage/ffiwrapper/verifier_cgo.go index 0af12da0b..59d0856d1 100644 --- a/extern/sector-storage/ffiwrapper/verifier_cgo.go +++ b/extern/sector-storage/ffiwrapper/verifier_cgo.go @@ -32,16 +32,25 @@ func (sb *Sealer) GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, return ffi.GenerateWinningPoSt(minerID, privsectors, randomness) } -func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.SectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, []abi.SectorID, error) { +func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.SectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, []abi.SectorID, []abi.SectorID, error) { randomness[31] &= 0x3f privsectors, skipped, done, err := sb.pubSectorToPriv(ctx, minerID, sectorInfo, nil, abi.RegisteredSealProof.RegisteredWindowPoStProof) if err != nil { - return nil, nil, xerrors.Errorf("gathering sector info: %w", err) + return nil, nil, nil, xerrors.Errorf("gathering sector info: %w", err) } defer done() - proof, err := ffi.GenerateWindowPoSt(minerID, privsectors, randomness) - return proof, skipped, err + proof, faulty, err := ffi.GenerateWindowPoSt(minerID, privsectors, randomness) + + var faultyIDs []abi.SectorID + for _, f := range faulty { + faultyIDs = append(faultyIDs, abi.SectorID{ + Miner: minerID, + Number: f, + }) + } + + return proof, skipped, faultyIDs, err } func (sb *Sealer) pubSectorToPriv(ctx context.Context, mid abi.ActorID, sectorInfo []proof.SectorInfo, faults []abi.SectorNumber, rpt func(abi.RegisteredSealProof) (abi.RegisteredPoStProof, error)) (ffi.SortedPrivateSectorInfo, []abi.SectorID, func(), error) { diff --git a/go.mod b/go.mod index bc4067d67..56c1420d0 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module github.com/filecoin-project/lotus go 1.14 -replace github.com/supranational/blst => github.com/supranational/blst v0.1.2-alpha.1 - require ( contrib.go.opencensus.io/exporter/jaeger v0.1.0 contrib.go.opencensus.io/exporter/prometheus v0.1.0 @@ -138,3 +136,9 @@ replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi replace github.com/dgraph-io/badger/v2 => github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200716180832-3ab515320794 replace github.com/filecoin-project/test-vectors => ./extern/test-vectors + +replace github.com/supranational/blst => ./extern/fil-blst/blst + +replace github.com/filecoin-project/fil-blst => ./extern/fil-blst + +replace github.com/filecoin-project/specs-storage => ../specs-storage diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 090648dbb..d3e91e450 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -406,11 +406,13 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo return nil, err } - postOut, postSkipped, err := s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, abi.PoStRandomness(rand)) + postOut, postSkipped, faulty, err := s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, abi.PoStRandomness(rand)) if err != nil { return nil, xerrors.Errorf("running post failed: %w", err) } + _ = faulty // TODO(magik) + if len(postOut) == 0 { return nil, xerrors.Errorf("received proofs back from generate window post") } From 5e7737f55d5f7b7b0b658d45c481e5e2fb536041 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 10 Sep 2020 02:59:37 +0200 Subject: [PATCH 2/6] wdpost: Handle skipped sectors correctly --- api/test/window_post.go | 2 +- .../sector-storage/ffiwrapper/verifier_cgo.go | 10 +- extern/sector-storage/mock/mock.go | 22 ++- go.mod | 2 - go.sum | 2 + storage/wdpost_run.go | 166 ++++++++++-------- 6 files changed, 123 insertions(+), 81 deletions(-) diff --git a/api/test/window_post.go b/api/test/window_post.go index 59d7ac1d6..3f15c754b 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -192,7 +192,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector // Drop the partition err = parts[0].Sectors.ForEach(func(sid uint64) error { - return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(abi.SectorID{ + return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkCorrupted(abi.SectorID{ Miner: abi.ActorID(mid), Number: abi.SectorNumber(sid), }, true) diff --git a/extern/sector-storage/ffiwrapper/verifier_cgo.go b/extern/sector-storage/ffiwrapper/verifier_cgo.go index 59d0856d1..d6c0ae35f 100644 --- a/extern/sector-storage/ffiwrapper/verifier_cgo.go +++ b/extern/sector-storage/ffiwrapper/verifier_cgo.go @@ -32,14 +32,18 @@ func (sb *Sealer) GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, return ffi.GenerateWinningPoSt(minerID, privsectors, randomness) } -func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.SectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, []abi.SectorID, []abi.SectorID, error) { +func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.SectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, []abi.SectorID, error) { randomness[31] &= 0x3f privsectors, skipped, done, err := sb.pubSectorToPriv(ctx, minerID, sectorInfo, nil, abi.RegisteredSealProof.RegisteredWindowPoStProof) if err != nil { - return nil, nil, nil, xerrors.Errorf("gathering sector info: %w", err) + return nil, nil, xerrors.Errorf("gathering sector info: %w", err) } defer done() + if len(skipped) > 0 { + return nil, skipped, xerrors.Errorf("pubSectorToPriv skipped some sectors") + } + proof, faulty, err := ffi.GenerateWindowPoSt(minerID, privsectors, randomness) var faultyIDs []abi.SectorID @@ -50,7 +54,7 @@ func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, s }) } - return proof, skipped, faultyIDs, err + return proof, faultyIDs, err } func (sb *Sealer) pubSectorToPriv(ctx context.Context, mid abi.ActorID, sectorInfo []proof.SectorInfo, faults []abi.SectorNumber, rpt func(abi.RegisteredSealProof) (abi.RegisteredPoStProof, error)) (ffi.SortedPrivateSectorInfo, []abi.SectorID, func(), error) { diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go index b70350ef8..7609f43b0 100644 --- a/extern/sector-storage/mock/mock.go +++ b/extern/sector-storage/mock/mock.go @@ -68,6 +68,7 @@ const ( type sectorState struct { pieces []cid.Cid failed bool + corrupted bool state int @@ -251,6 +252,18 @@ func (mgr *SectorMgr) MarkFailed(sid abi.SectorID, failed bool) error { return nil } +func (mgr *SectorMgr) MarkCorrupted(sid abi.SectorID, corrupted bool) error { + mgr.lk.Lock() + defer mgr.lk.Unlock() + ss, ok := mgr.sectors[sid] + if !ok { + return fmt.Errorf("no such sector in storage") + } + + ss.corrupted = corrupted + return nil +} + func opFinishWait(ctx context.Context) { val, ok := ctx.Value("opfinish").(chan struct{}) if !ok { @@ -275,6 +288,8 @@ func (mgr *SectorMgr) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorI si := make([]proof.SectorInfo, 0, len(sectorInfo)) var skipped []abi.SectorID + var err error + for _, info := range sectorInfo { sid := abi.SectorID{ Miner: minerID, @@ -283,13 +298,18 @@ func (mgr *SectorMgr) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorI _, found := mgr.sectors[sid] - if found && !mgr.sectors[sid].failed { + if found && !mgr.sectors[sid].failed && !mgr.sectors[sid].corrupted { si = append(si, info) } else { skipped = append(skipped, sid) + err = xerrors.Errorf("skipped some sectors") } } + if err != nil { + return nil, skipped, err + } + return generateFakePoSt(si, abi.RegisteredSealProof.RegisteredWindowPoStProof, randomness), skipped, nil } diff --git a/go.mod b/go.mod index 56c1420d0..4011577df 100644 --- a/go.mod +++ b/go.mod @@ -140,5 +140,3 @@ replace github.com/filecoin-project/test-vectors => ./extern/test-vectors replace github.com/supranational/blst => ./extern/fil-blst/blst replace github.com/filecoin-project/fil-blst => ./extern/fil-blst - -replace github.com/filecoin-project/specs-storage => ../specs-storage diff --git a/go.sum b/go.sum index 0fa13fe70..3f37bff04 100644 --- a/go.sum +++ b/go.sum @@ -250,6 +250,8 @@ github.com/filecoin-project/specs-actors v0.9.7 h1:7PAZ8kdqwBdmgf/23FCkQZLCXcVu0 github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= +github.com/filecoin-project/specs-storage v0.1.1-0.20200909213410-c066548422be h1:UX457RrC0LwL7Bb5kd0WFyzJBxbHOCSw/64oYqeT+Zc= +github.com/filecoin-project/specs-storage v0.1.1-0.20200909213410-c066548422be/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus= github.com/filecoin-project/test-vectors/schema v0.0.1/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index d3e91e450..4fd4d51e6 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -335,96 +335,114 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo Proofs: nil, } - var sinfos []proof.SectorInfo - sidToPart := map[abi.SectorNumber]uint64{} skipCount := uint64(0) + postSkipped := bitfield.New() + var postOut []proof.PoStProof - for partIdx, partition := range partitions { - // TODO: Can do this in parallel - toProve, err := partition.ActiveSectors() + for retries := 0; retries < 5; retries++ { + var sinfos []proof.SectorInfo + sidToPart := map[abi.SectorNumber]int{} + + for partIdx, partition := range partitions { + // TODO: Can do this in parallel + toProve, err := partition.ActiveSectors() + if err != nil { + return nil, xerrors.Errorf("getting active sectors: %w", err) + } + + toProve, err = bitfield.MergeBitFields(toProve, partition.Recoveries) + if err != nil { + return nil, xerrors.Errorf("adding recoveries to set of sectors to prove: %w", err) + } + + toProve, err = bitfield.SubtractBitField(toProve, postSkipped) + if err != nil { + return nil, xerrors.Errorf("toProve - postSkipped: %w", err) + } + + good, err := s.checkSectors(ctx, toProve) + if err != nil { + return nil, xerrors.Errorf("checking sectors to skip: %w", err) + } + + skipped, err := bitfield.SubtractBitField(toProve, good) + if err != nil { + return nil, xerrors.Errorf("toProve - good: %w", err) + } + + sc, err := skipped.Count() + if err != nil { + return nil, xerrors.Errorf("getting skipped sector count: %w", err) + } + + skipCount += sc + + ssi, err := s.sectorsForProof(ctx, good, partition.Sectors, ts) + if err != nil { + return nil, xerrors.Errorf("getting sorted sector info: %w", err) + } + + if len(ssi) == 0 { + continue + } + + sinfos = append(sinfos, ssi...) + for _, si := range ssi { + sidToPart[si.SectorNumber] = partIdx + } + + params.Partitions = append(params.Partitions, miner.PoStPartition{ + Index: uint64(partIdx), + Skipped: skipped, + }) + } + + if len(sinfos) == 0 { + // nothing to prove.. + return nil, errNoPartitions + } + + log.Infow("running windowPost", + "chain-random", rand, + "deadline", di, + "height", ts.Height(), + "skipped", skipCount) + + tsStart := build.Clock.Now() + + mid, err := address.IDFromAddress(s.actor) if err != nil { - return nil, xerrors.Errorf("getting active sectors: %w", err) + return nil, err } - toProve, err = bitfield.MergeBitFields(toProve, partition.Recoveries) - if err != nil { - return nil, xerrors.Errorf("adding recoveries to set of sectors to prove: %w", err) + var ps []abi.SectorID + postOut, ps, err = s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, abi.PoStRandomness(rand)) + elapsed := time.Since(tsStart) + + log.Infow("computing window PoSt", "elapsed", elapsed) + + if err == nil { + break } - good, err := s.checkSectors(ctx, toProve) - if err != nil { - return nil, xerrors.Errorf("checking sectors to skip: %w", err) + if len(ps) == 0 { + return nil, xerrors.Errorf("running post failed: %w", err) } - skipped, err := bitfield.SubtractBitField(toProve, good) - if err != nil { - return nil, xerrors.Errorf("toProve - good: %w", err) + log.Warnw("generate window PoSt skipped sectors", "sectors", ps, "error", err, "try", retries) + + skipCount += uint64(len(ps)) + for _, sector := range ps { + postSkipped.Set(uint64(sector.Number)) } - - sc, err := skipped.Count() - if err != nil { - return nil, xerrors.Errorf("getting skipped sector count: %w", err) - } - - skipCount += sc - - ssi, err := s.sectorsForProof(ctx, good, partition.Sectors, ts) - if err != nil { - return nil, xerrors.Errorf("getting sorted sector info: %w", err) - } - - if len(ssi) == 0 { - continue - } - - sinfos = append(sinfos, ssi...) - for _, si := range ssi { - sidToPart[si.SectorNumber] = uint64(partIdx) - } - - params.Partitions = append(params.Partitions, miner.PoStPartition{ - Index: uint64(partIdx), - Skipped: skipped, - }) } - if len(sinfos) == 0 { - // nothing to prove.. - return nil, errNoPartitions - } - - log.Infow("running windowPost", - "chain-random", rand, - "deadline", di, - "height", ts.Height(), - "skipped", skipCount) - - tsStart := build.Clock.Now() - - mid, err := address.IDFromAddress(s.actor) - if err != nil { - return nil, err - } - - postOut, postSkipped, faulty, err := s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, abi.PoStRandomness(rand)) - if err != nil { - return nil, xerrors.Errorf("running post failed: %w", err) - } - - _ = faulty // TODO(magik) - if len(postOut) == 0 { - return nil, xerrors.Errorf("received proofs back from generate window post") + return nil, xerrors.Errorf("received no proofs back from generate window post") } params.Proofs = postOut - for _, sector := range postSkipped { - params.Partitions[sidToPart[sector.Number]].Skipped.Set(uint64(sector.Number)) - } - - elapsed := time.Since(tsStart) - commEpoch := di.Open commRand, err := s.api.ChainGetRandomnessFromTickets(ctx, ts.Key(), crypto.DomainSeparationTag_PoStChainCommit, commEpoch, nil) if err != nil { @@ -433,7 +451,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo params.ChainCommitEpoch = commEpoch params.ChainCommitRand = commRand - log.Infow("submitting window PoSt", "elapsed", elapsed) + log.Infow("submitting window PoSt") return params, nil } From c7b0241a48b5f503cfc7a32e3ce2602e412a0fe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 10 Sep 2020 21:19:26 +0200 Subject: [PATCH 3/6] ffiwrapper: Test skipping corrupted sectors in PoSt --- .../sector-storage/ffiwrapper/sealer_test.go | 87 ++++++++++--------- 1 file changed, 46 insertions(+), 41 deletions(-) diff --git a/extern/sector-storage/ffiwrapper/sealer_test.go b/extern/sector-storage/ffiwrapper/sealer_test.go index ae1ede7eb..acbd9ca21 100644 --- a/extern/sector-storage/ffiwrapper/sealer_test.go +++ b/extern/sector-storage/ffiwrapper/sealer_test.go @@ -168,50 +168,34 @@ func (s *seal) unseal(t *testing.T, sb *Sealer, sp *basicfs.Provider, si abi.Sec } } -func post(t *testing.T, sealer *Sealer, seals ...seal) time.Time { - /*randomness := abi.PoStRandomness{0, 9, 2, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 7} +func post(t *testing.T, sealer *Sealer, skipped []abi.SectorID, seals ...seal) { + randomness := abi.PoStRandomness{0, 9, 2, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 7} - sis := make([]abi.SectorInfo, len(seals)) + sis := make([]saproof.SectorInfo, len(seals)) for i, s := range seals { - sis[i] = abi.SectorInfo{ - RegisteredProof: sealProofType, + sis[i] = saproof.SectorInfo{ + SealProof: sealProofType, SectorNumber: s.id.Number, SealedCID: s.cids.Sealed, } } - candidates, err := sealer.GenerateEPostCandidates(context.TODO(), seals[0].id.Miner, sis, randomness, []abi.SectorNumber{}) - if err != nil { - t.Fatalf("%+v", err) - }*/ - - fmt.Println("skipping post") - - genCandidates := time.Now() - - /*if len(candidates) != 1 { - t.Fatal("expected 1 candidate") + proofs, skp, err := sealer.GenerateWindowPoSt(context.TODO(), seals[0].id.Miner, sis, randomness) + if len(skipped) > 0 { + require.Error(t, err) + require.EqualValues(t, skipped, skp) + return } - candidatesPrime := make([]abi.PoStCandidate, len(candidates)) - for idx := range candidatesPrime { - candidatesPrime[idx] = candidates[idx].Candidate - } - - proofs, err := sealer.ComputeElectionPoSt(context.TODO(), seals[0].id.Miner, sis, randomness, candidatesPrime) if err != nil { t.Fatalf("%+v", err) } - ePoStChallengeCount := ElectionPostChallengeCount(uint64(len(sis)), 0) - - ok, err := ProofVerifier.VerifyElectionPost(context.TODO(), abi.PoStVerifyInfo{ + ok, err := ProofVerifier.VerifyWindowPoSt(context.TODO(), saproof.WindowPoStVerifyInfo{ Randomness: randomness, - Candidates: candidatesPrime, Proofs: proofs, - EligibleSectors: sis, + ChallengedSectors: sis, Prover: seals[0].id.Miner, - ChallengeCount: ePoStChallengeCount, }) if err != nil { t.Fatalf("%+v", err) @@ -219,8 +203,21 @@ func post(t *testing.T, sealer *Sealer, seals ...seal) time.Time { if !ok { t.Fatal("bad post") } - */ - return genCandidates +} + +func corrupt(t *testing.T, sealer *Sealer, id abi.SectorID) { + paths, done, err := sealer.sectors.AcquireSector(context.Background(), id, stores.FTSealed, 0, stores.PathStorage) + require.NoError(t, err) + defer done() + + log.Infof("corrupt %s", paths.Sealed) + f, err := os.OpenFile(paths.Sealed, os.O_RDWR, 0664) + require.NoError(t, err) + + _, err = f.WriteAt(bytes.Repeat([]byte{'d'}, 2048), 0) + require.NoError(t, err) + + require.NoError(t, f.Close()) } func getGrothParamFileAndVerifyingKeys(s abi.SectorSize) { @@ -299,11 +296,11 @@ func TestSealAndVerify(t *testing.T) { commit := time.Now() - genCandidiates := post(t, sb, s) + post(t, sb, nil, s) epost := time.Now() - post(t, sb, s) + post(t, sb, nil, s) if err := sb.FinalizeSector(context.TODO(), si, nil); err != nil { t.Fatalf("%+v", err) @@ -313,8 +310,7 @@ func TestSealAndVerify(t *testing.T) { fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String()) fmt.Printf("Commit: %s\n", commit.Sub(precommit).String()) - fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(commit).String()) - fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String()) + fmt.Printf("EPoSt: %s\n", epost.Sub(commit).String()) } func TestSealPoStNoCommit(t *testing.T) { @@ -370,16 +366,15 @@ func TestSealPoStNoCommit(t *testing.T) { t.Fatal(err) } - genCandidiates := post(t, sb, s) + post(t, sb, nil, s) epost := time.Now() fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String()) - fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(precommit).String()) - fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String()) + fmt.Printf("EPoSt: %s\n", epost.Sub(precommit).String()) } -func TestSealAndVerify2(t *testing.T) { +func TestSealAndVerify3(t *testing.T) { defer requireFDsClosed(t, openFDs(t)) if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware @@ -419,22 +414,32 @@ func TestSealAndVerify2(t *testing.T) { si1 := abi.SectorID{Miner: miner, Number: 1} si2 := abi.SectorID{Miner: miner, Number: 2} + si3 := abi.SectorID{Miner: miner, Number: 3} s1 := seal{id: si1} s2 := seal{id: si2} + s3 := seal{id: si3} - wg.Add(2) + wg.Add(3) go s1.precommit(t, sb, si1, wg.Done) //nolint: staticcheck time.Sleep(100 * time.Millisecond) go s2.precommit(t, sb, si2, wg.Done) //nolint: staticcheck + time.Sleep(100 * time.Millisecond) + go s3.precommit(t, sb, si3, wg.Done) //nolint: staticcheck wg.Wait() - wg.Add(2) + wg.Add(3) go s1.commit(t, sb, wg.Done) //nolint: staticcheck go s2.commit(t, sb, wg.Done) //nolint: staticcheck + go s3.commit(t, sb, wg.Done) //nolint: staticcheck wg.Wait() - post(t, sb, s1, s2) + post(t, sb, nil, s1, s2, s3) + + corrupt(t, sb, si1) + corrupt(t, sb, si2) + + post(t, sb, []abi.SectorID{si1, si2}, s1, s2, s3) } func BenchmarkWriteWithAlignment(b *testing.B) { From 2374cb1bc599cfab8c0a62707fc4b6ec15f4568c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 10 Sep 2020 21:44:01 +0200 Subject: [PATCH 4/6] Update ffi --- extern/filecoin-ffi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 79b324966..f640612a1 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 79b3249666a809871fad12fbf50f68e22218b01c +Subproject commit f640612a1a1f7a2dd8b3a49e1531db0aa0f63447 From 22556fb24c99c44390c914597e75d18357758647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 10 Sep 2020 22:04:08 +0200 Subject: [PATCH 5/6] Add fil-blst submodule --- .gitmodules | 3 +++ extern/fil-blst | 1 + 2 files changed, 4 insertions(+) create mode 160000 extern/fil-blst diff --git a/.gitmodules b/.gitmodules index ad09aba35..4b450aaf3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,3 +8,6 @@ [submodule "extern/test-vectors"] path = extern/test-vectors url = https://github.com/filecoin-project/test-vectors.git +[submodule "extern/fil-blst"] + path = extern/fil-blst + url = https://github.com/filecoin-project/fil-blst.git diff --git a/extern/fil-blst b/extern/fil-blst new file mode 160000 index 000000000..5f93488fc --- /dev/null +++ b/extern/fil-blst @@ -0,0 +1 @@ +Subproject commit 5f93488fc0dbfb450f2355269f18fc67010d59bb From bbac86f7454222adb1b4a89880b7625270ca4eb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 10 Sep 2020 22:07:20 +0200 Subject: [PATCH 6/6] gofmt, mod tidy --- extern/sector-storage/ffiwrapper/sealer_test.go | 12 ++++++------ extern/sector-storage/mock/mock.go | 4 ++-- go.sum | 4 ---- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/extern/sector-storage/ffiwrapper/sealer_test.go b/extern/sector-storage/ffiwrapper/sealer_test.go index acbd9ca21..d59de2cab 100644 --- a/extern/sector-storage/ffiwrapper/sealer_test.go +++ b/extern/sector-storage/ffiwrapper/sealer_test.go @@ -174,9 +174,9 @@ func post(t *testing.T, sealer *Sealer, skipped []abi.SectorID, seals ...seal) { sis := make([]saproof.SectorInfo, len(seals)) for i, s := range seals { sis[i] = saproof.SectorInfo{ - SealProof: sealProofType, - SectorNumber: s.id.Number, - SealedCID: s.cids.Sealed, + SealProof: sealProofType, + SectorNumber: s.id.Number, + SealedCID: s.cids.Sealed, } } @@ -192,10 +192,10 @@ func post(t *testing.T, sealer *Sealer, skipped []abi.SectorID, seals ...seal) { } ok, err := ProofVerifier.VerifyWindowPoSt(context.TODO(), saproof.WindowPoStVerifyInfo{ - Randomness: randomness, - Proofs: proofs, + Randomness: randomness, + Proofs: proofs, ChallengedSectors: sis, - Prover: seals[0].id.Miner, + Prover: seals[0].id.Miner, }) if err != nil { t.Fatalf("%+v", err) diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go index 7609f43b0..64207e66d 100644 --- a/extern/sector-storage/mock/mock.go +++ b/extern/sector-storage/mock/mock.go @@ -66,8 +66,8 @@ const ( ) type sectorState struct { - pieces []cid.Cid - failed bool + pieces []cid.Cid + failed bool corrupted bool state int diff --git a/go.sum b/go.sum index 3f37bff04..953e3bba6 100644 --- a/go.sum +++ b/go.sum @@ -250,8 +250,6 @@ github.com/filecoin-project/specs-actors v0.9.7 h1:7PAZ8kdqwBdmgf/23FCkQZLCXcVu0 github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/specs-storage v0.1.1-0.20200909213410-c066548422be h1:UX457RrC0LwL7Bb5kd0WFyzJBxbHOCSw/64oYqeT+Zc= -github.com/filecoin-project/specs-storage v0.1.1-0.20200909213410-c066548422be/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus= github.com/filecoin-project/test-vectors/schema v0.0.1/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -1318,8 +1316,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/supranational/blst v0.1.2-alpha.1 h1:v0UqVlvbRNZIaSeMPr+T01kvTUq1h0EZuZ6gnDR1Mlg= -github.com/supranational/blst v0.1.2-alpha.1/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=