From 915c5e0fdd78693efbe3658a03bf108b6fb903ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Nov 2020 13:43:05 +0100 Subject: [PATCH] Get most tests to pass --- api/test/window_post.go | 19 +++++--- cmd/lotus-bench/main.go | 74 ++++++++++++++++------------- cmd/lotus-seed/main.go | 7 ++- extern/sector-storage/mock/mock.go | 8 ++-- extern/sector-storage/sched_test.go | 5 +- node/test/builder.go | 5 +- storage/mockstorage/preseal.go | 15 +++--- 7 files changed, 72 insertions(+), 61 deletions(-) diff --git a/api/test/window_post.go b/api/test/window_post.go index 55fc4ad70..8dac187b5 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/mock" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -209,15 +210,17 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, // Drop the partition err = secs.ForEach(func(sid uint64) error { - return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkCorrupted(abi.SectorID{ - Miner: abi.ActorID(mid), - Number: abi.SectorNumber(sid), + return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkCorrupted(storage.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(mid), + Number: abi.SectorNumber(sid), + }, }, true) }) require.NoError(t, err) } - var s abi.SectorID + var s storage.SectorRef // Drop 1 sectors from deadline 3 partition 0 { @@ -238,9 +241,11 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, require.NoError(t, err) fmt.Println("the sectors", all) - s = abi.SectorID{ - Miner: abi.ActorID(mid), - Number: abi.SectorNumber(sn), + s = storage.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(mid), + Number: abi.SectorNumber(sn), + }, } err = miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(s, true) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 1af5c1c62..f502e429c 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -31,6 +31,7 @@ import ( lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/genesis" @@ -211,15 +212,6 @@ var sealBenchCmd = &cli.Command{ } sectorSize := abi.SectorSize(sectorSizeInt) - spt, err := ffiwrapper.SealProofTypeFromSectorSize(sectorSize) - if err != nil { - return err - } - - cfg := &ffiwrapper.Config{ - SealProofType: spt, - } - // Only fetch parameters if actually needed if !c.Bool("skip-commit2") { if err := paramfetch.GetParams(lcli.ReqContext(c), build.ParametersJSON(), uint64(sectorSize)); err != nil { @@ -231,7 +223,7 @@ var sealBenchCmd = &cli.Command{ Root: sbdir, } - sb, err := ffiwrapper.New(sbfs, cfg) + sb, err := ffiwrapper.New(sbfs) if err != nil { return err } @@ -295,7 +287,7 @@ var sealBenchCmd = &cli.Command{ if !c.Bool("skip-commit2") { log.Info("generating winning post candidates") - wipt, err := spt.RegisteredWinningPoStProof() + wipt, err := spt(sectorSize).RegisteredWinningPoStProof() if err != nil { return err } @@ -428,7 +420,7 @@ var sealBenchCmd = &cli.Command{ fmt.Println(string(data)) } else { - fmt.Printf("----\nresults (v27) (%d)\n", sectorSize) + fmt.Printf("----\nresults (v28) (%d)\n", sectorSize) if robench == "" { fmt.Printf("seal: addPiece: %s (%s)\n", bo.SealingResults[0].AddPiece, bps(bo.SectorSize, bo.SealingResults[0].AddPiece)) // TODO: average across multiple sealings fmt.Printf("seal: preCommit phase 1: %s (%s)\n", bo.SealingResults[0].PreCommit1, bps(bo.SectorSize, bo.SealingResults[0].PreCommit1)) @@ -477,9 +469,12 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par } for i := abi.SectorNumber(1); i <= abi.SectorNumber(numSectors); i++ { - sid := abi.SectorID{ - Miner: mid, - Number: i, + sid := storage.SectorRef{ + ID: abi.SectorID{ + Miner: mid, + Number: i, + }, + ProofType: spt(sectorSize), } start := time.Now() @@ -507,9 +502,12 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par end := start + sectorsPerWorker for i := abi.SectorNumber(start); i < abi.SectorNumber(end); i++ { ix := int(i - 1) - sid := abi.SectorID{ - Miner: mid, - Number: i, + sid := storage.SectorRef{ + ID: abi.SectorID{ + Miner: mid, + Number: i, + }, + ProofType: spt(sectorSize), } start := time.Now() @@ -538,7 +536,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par <-preCommit2Sema sealedSectors[ix] = saproof2.SectorInfo{ - SealProof: sb.SealProofType(), + SealProof: sid.ProofType, SectorNumber: i, SealedCID: cids.Sealed, } @@ -592,7 +590,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par svi := saproof2.SealVerifyInfo{ SectorID: abi.SectorID{Miner: mid, Number: i}, SealedCID: cids.Sealed, - SealProof: sb.SealProofType(), + SealProof: sid.ProofType, Proof: proof, DealIDs: nil, Randomness: ticket, @@ -614,7 +612,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par if !skipunseal { log.Infof("[%d] Unsealing sector", i) { - p, done, err := sbfs.AcquireSector(context.TODO(), abi.SectorID{Miner: mid, Number: 1}, storiface.FTUnsealed, storiface.FTNone, storiface.PathSealing) + p, done, err := sbfs.AcquireSector(context.TODO(), sid, storiface.FTUnsealed, storiface.FTNone, storiface.PathSealing) if err != nil { return xerrors.Errorf("acquire unsealed sector for removing: %w", err) } @@ -625,7 +623,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par } } - err := sb.UnsealPiece(context.TODO(), abi.SectorID{Miner: mid, Number: 1}, 0, abi.PaddedPieceSize(sectorSize).Unpadded(), ticket, cids.Unsealed) + err := sb.UnsealPiece(context.TODO(), sid, 0, abi.PaddedPieceSize(sectorSize).Unpadded(), ticket, cids.Unsealed) if err != nil { return err } @@ -708,23 +706,22 @@ var proveCmd = &cli.Command{ return err } - spt, err := ffiwrapper.SealProofTypeFromSectorSize(abi.SectorSize(c2in.SectorSize)) - if err != nil { - return err - } - - cfg := &ffiwrapper.Config{ - SealProofType: spt, - } - - sb, err := ffiwrapper.New(nil, cfg) + sb, err := ffiwrapper.New(nil) if err != nil { return err } start := time.Now() - proof, err := sb.SealCommit2(context.TODO(), abi.SectorID{Miner: abi.ActorID(mid), Number: abi.SectorNumber(c2in.SectorNum)}, c2in.Phase1Out) + ref := storage.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(mid), + Number: abi.SectorNumber(c2in.SectorNum), + }, + ProofType: spt(abi.SectorSize(c2in.SectorSize)), + } + + proof, err := sb.SealCommit2(context.TODO(), ref, c2in.Phase1Out) if err != nil { return err } @@ -733,7 +730,7 @@ var proveCmd = &cli.Command{ fmt.Printf("proof: %x\n", proof) - fmt.Printf("----\nresults (v27) (%d)\n", c2in.SectorSize) + fmt.Printf("----\nresults (v28) (%d)\n", c2in.SectorSize) dur := sealCommit2.Sub(start) fmt.Printf("seal: commit phase 2: %s (%s)\n", dur, bps(abi.SectorSize(c2in.SectorSize), dur)) @@ -747,3 +744,12 @@ func bps(data abi.SectorSize, d time.Duration) string { bps := bdata.Div(bdata, big.NewInt(d.Nanoseconds())) return types.SizeStr(types.BigInt{Int: bps}) + "/s" } + +func spt(ssize abi.SectorSize) abi.RegisteredSealProof { + spt, err := miner.SealProofTypeFromSectorSize(ssize, build.NewestNetworkVersion) + if err != nil { + panic(err) + } + + return spt +} diff --git a/cmd/lotus-seed/main.go b/cmd/lotus-seed/main.go index d365f6493..7822900e4 100644 --- a/cmd/lotus-seed/main.go +++ b/cmd/lotus-seed/main.go @@ -8,8 +8,6 @@ import ( "os" "github.com/docker/go-units" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" "github.com/urfave/cli/v2" @@ -19,6 +17,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" "github.com/filecoin-project/lotus/genesis" @@ -128,12 +127,12 @@ var preSealCmd = &cli.Command{ } sectorSize := abi.SectorSize(sectorSizeInt) - rp, err := ffiwrapper.SealProofTypeFromSectorSize(sectorSize) + spt, err := miner.SealProofTypeFromSectorSize(sectorSize, build.NewestNetworkVersion) if err != nil { return err } - gm, key, err := seed.PreSeal(maddr, rp, abi.SectorNumber(c.Uint64("sector-offset")), c.Int("num-sectors"), sbroot, []byte(c.String("ticket-preimage")), k, c.Bool("fake-sectors")) + gm, key, err := seed.PreSeal(maddr, spt, abi.SectorNumber(c.Uint64("sector-offset")), c.Int("num-sectors"), sbroot, []byte(c.String("ticket-preimage")), k, c.Bool("fake-sectors")) if err != nil { return err } diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go index be180f093..5e85d6ef7 100644 --- a/extern/sector-storage/mock/mock.go +++ b/extern/sector-storage/mock/mock.go @@ -404,14 +404,14 @@ func (mgr *SectorMgr) Remove(ctx context.Context, sector storage.SectorRef) erro return nil } -func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []abi.SectorID) ([]abi.SectorID, error) { +func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []storage.SectorRef) ([]abi.SectorID, error) { var bad []abi.SectorID for _, sid := range ids { - _, found := mgr.sectors[sid] + _, found := mgr.sectors[sid.ID] - if !found || mgr.sectors[sid].failed { - bad = append(bad, sid) + if !found || mgr.sectors[sid.ID].failed { + bad = append(bad, sid.ID) } } diff --git a/extern/sector-storage/sched_test.go b/extern/sector-storage/sched_test.go index 94e704f2b..a87d403b7 100644 --- a/extern/sector-storage/sched_test.go +++ b/extern/sector-storage/sched_test.go @@ -584,7 +584,10 @@ func TestWindowCompact(t *testing.T) { window := &schedWindow{} for _, task := range windowTasks { - window.todo = append(window.todo, &workerRequest{taskType: task}) + window.todo = append(window.todo, &workerRequest{ + taskType: task, + sector: storage.SectorRef{ProofType: spt}, + }) window.allocated.add(wh.info.Resources, ResourceTable[task][spt]) } diff --git a/node/test/builder.go b/node/test/builder.go index 491d6b7fe..f6599cf23 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -26,7 +26,6 @@ import ( "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/gen" genesis2 "github.com/filecoin-project/lotus/chain/gen/genesis" "github.com/filecoin-project/lotus/chain/messagepool" @@ -356,7 +355,7 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes preseals = test.GenesisPreseals } - genm, k, err := mockstorage.PreSeal(2048, maddr, preseals) + genm, k, err := mockstorage.PreSeal(abi.RegisteredSealProof_StackedDrg2KiBV1, maddr, preseals) if err != nil { t.Fatal(err) } @@ -458,7 +457,7 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes storers[i] = CreateTestStorageNode(ctx, t, genms[i].Worker, maddrs[i], pidKeys[i], f, mn, node.Options( node.Override(new(sectorstorage.SectorManager), func() (sectorstorage.SectorManager, error) { - return mock.NewMockSectorMgr(policy.GetDefaultSectorSize(), sectors), nil + return mock.NewMockSectorMgr(sectors), nil }), node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), node.Unset(new(*sectorstorage.Manager)), diff --git a/storage/mockstorage/preseal.go b/storage/mockstorage/preseal.go index 0417405c8..d119c273f 100644 --- a/storage/mockstorage/preseal.go +++ b/storage/mockstorage/preseal.go @@ -13,17 +13,21 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/zerocomm" "github.com/filecoin-project/lotus/genesis" ) -func PreSeal(ssize abi.SectorSize, maddr address.Address, sectors int) (*genesis.Miner, *types.KeyInfo, error) { +func PreSeal(spt abi.RegisteredSealProof, maddr address.Address, sectors int) (*genesis.Miner, *types.KeyInfo, error) { k, err := wallet.GenerateKey(types.KTBLS) if err != nil { return nil, nil, err } + ssize, err := spt.SectorSize() + if err != nil { + return nil, nil, err + } + genm := &genesis.Miner{ ID: maddr, Owner: k.Address, @@ -34,15 +38,10 @@ func PreSeal(ssize abi.SectorSize, maddr address.Address, sectors int) (*genesis Sectors: make([]*genesis.PreSeal, sectors), } - st, err := ffiwrapper.SealProofTypeFromSectorSize(ssize) - if err != nil { - return nil, nil, err - } - for i := range genm.Sectors { preseal := &genesis.PreSeal{} - preseal.ProofType = st + preseal.ProofType = spt preseal.CommD = zerocomm.ZeroPieceCommitment(abi.PaddedPieceSize(ssize).Unpadded()) d, _ := commcid.CIDToPieceCommitmentV1(preseal.CommD) r := mock.CommDR(d)