From d5b8defcbbdea66d94a76b0c241009138e97544c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 28 Feb 2020 18:17:40 +0100 Subject: [PATCH 01/24] Begin work on integrating new sectorbuilder interfaces --- go.mod | 2 +- go.sum | 2 + storage/sbmock/sbmock.go | 99 ++++++++++++---------------------------- 3 files changed, 32 insertions(+), 71 deletions(-) diff --git a/go.mod b/go.mod index 517ab0336..eb744a101 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/filecoin-project/go-fil-markets v0.0.0-20200227194154-8ef88c730c81 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 - github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200227214225-a550d0267977 + github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200228064721-be233370d8e9 github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/specs-actors v0.0.0-20200226233922-9ed222007d11 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index cdbd7da63..921506d7a 100644 --- a/go.sum +++ b/go.sum @@ -137,6 +137,8 @@ github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200227214127-332d57b390a github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200227214127-332d57b390aa/go.mod h1:u8n91UJUua9zEiyAPw8wRAgUEoyTH8BBM9LWJ5WuxOM= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200227214225-a550d0267977 h1:Q4VnMiorS/AcZMDHtoi+7lVzkTj+qDa0bmgMkwmvxIM= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200227214225-a550d0267977/go.mod h1:tzTc9BxxSbjlIzhFwm5h9oBkXKkRuLxeiWspntwnKyw= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200228064721-be233370d8e9 h1:P4IMEn3GDtAJzIVL/DqwHH8jotvafn6FLuxFhUNKhEI= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200228064721-be233370d8e9/go.mod h1:gEQJVRVqQX8Vx02IosTdC2UA4l2MgHfG3POJEI2GIpc= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 h1:k9qVR9ItcziSB2rxtlkN/MDWNlbsI6yzec+zjUatLW0= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= diff --git a/storage/sbmock/sbmock.go b/storage/sbmock/sbmock.go index e038affd8..e3906c3ce 100644 --- a/storage/sbmock/sbmock.go +++ b/storage/sbmock/sbmock.go @@ -10,13 +10,13 @@ import ( "math/rand" "sync" - ffi "github.com/filecoin-project/filecoin-ffi" commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/go-sectorbuilder/fs" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/ipfs/go-cid" "golang.org/x/xerrors" + + ffi "github.com/filecoin-project/filecoin-ffi" ) type SBMock struct { @@ -102,40 +102,16 @@ func (sb *SBMock) AcquireSectorNumber() (abi.SectorNumber, error) { return id, nil } -func (sb *SBMock) Scrub([]abi.SectorNumber) []*sectorbuilder.Fault { - sb.lk.Lock() - mcopy := make(map[abi.SectorNumber]*sectorState) - for k, v := range sb.sectors { - mcopy[k] = v - } - sb.lk.Unlock() - - var out []*sectorbuilder.Fault - for sid, ss := range mcopy { - ss.lk.Lock() - if ss.failed { - out = append(out, §orbuilder.Fault{ - SectorNum: sid, - Err: fmt.Errorf("mock sector failed"), - }) - - } - ss.lk.Unlock() - } - - return out -} - func (sb *SBMock) GenerateFallbackPoSt([]abi.SectorInfo, abi.PoStRandomness, []abi.SectorNumber) ([]ffi.PoStCandidateWithTicket, []abi.PoStProof, error) { panic("NYI") } -func (sb *SBMock) SealPreCommit(ctx context.Context, sid abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (cid.Cid, cid.Cid, error) { +func (sb *SBMock) SealPreCommit1(ctx context.Context, sid abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out []byte, err error) { sb.lk.Lock() ss, ok := sb.sectors[sid] sb.lk.Unlock() if !ok { - return cid.Undef, cid.Undef, xerrors.Errorf("no sector with id %d in sectorbuilder", sid) + return nil, xerrors.Errorf("no sector with id %d in sectorbuilder", sid) } ss.lk.Lock() @@ -151,11 +127,11 @@ func (sb *SBMock) SealPreCommit(ctx context.Context, sid abi.SectorNumber, ticke } if sum != ussize { - return cid.Undef, cid.Undef, xerrors.Errorf("aggregated piece sizes don't match up: %d != %d", sum, ussize) + return nil, xerrors.Errorf("aggregated piece sizes don't match up: %d != %d", sum, ussize) } if ss.state != statePacking { - return cid.Undef, cid.Undef, xerrors.Errorf("cannot call pre-seal on sector not in 'packing' state") + return nil, xerrors.Errorf("cannot call pre-seal on sector not in 'packing' state") } opFinishWait(ctx) @@ -172,25 +148,35 @@ func (sb *SBMock) SealPreCommit(ctx context.Context, sid abi.SectorNumber, ticke commd, err := MockVerifier.GenerateDataCommitment(abi.PaddedPieceSize(sb.sectorSize), pis) if err != nil { - return cid.Undef, cid.Undef, err + return nil, err } cc, _, err := commcid.CIDToCommitment(commd) if err != nil { panic(err) } + cc[0] ^= 'd' + + return cc, nil +} + +func (sb *SBMock) SealPreCommit2(ctx context.Context, sid abi.SectorNumber, phase1Out []byte) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { + db := []byte(string(phase1Out[0])) + db[0] ^= 'd' + + d := commcid.DataCommitmentV1ToCID(db) commr := make([]byte, 32) - for i := range cc { - commr[32-(i+1)] = cc[i] + for i := range db { + commr[32-(i+1)] = db[i] } commR := commcid.DataCommitmentV1ToCID(commr) - return commd, commR, nil + return commR, d, nil } -func (sb *SBMock) SealCommit(ctx context.Context, sid abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCid cid.Cid, unsealed cid.Cid) ([]byte, error) { +func (sb *SBMock) SealCommit1(ctx context.Context, sid abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCid cid.Cid, unsealed cid.Cid) (output []byte, err error) { sb.lk.Lock() ss, ok := sb.sectors[sid] sb.lk.Unlock() @@ -212,30 +198,19 @@ func (sb *SBMock) SealCommit(ctx context.Context, sid abi.SectorNumber, ticket a var out [32]byte for i := range out { - out[i] = unsealed.Bytes()[i] + sealedCid.Bytes()[31-i] - ticket[i]*seed[i] + out[i] = unsealed.Bytes()[i] + sealedCid.Bytes()[31-i] - ticket[i]*seed[i] ^ byte(sid & 0xff) } return out[:], nil } -func (sb *SBMock) GetPath(string, string) (string, error) { - panic("nyi") -} +func (sb *SBMock) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (proof []byte, err error) { + var out [32]byte + for i := range out { + out[i] = phase1Out[i] ^ byte(sectorNum & 0xff) + } -func (sb *SBMock) CanCommit(sectorID abi.SectorNumber) (bool, error) { - return true, nil -} - -func (sb *SBMock) WorkerStats() sectorbuilder.WorkerStats { - panic("nyi") -} - -func (sb *SBMock) AddWorker(context.Context, sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) { - panic("nyi") -} - -func (sb *SBMock) TaskDone(context.Context, uint64, sectorbuilder.SealRes) error { - panic("nyi") + return out[:], nil } // Test Instrumentation Methods @@ -331,22 +306,6 @@ func (sb *SBMock) FinalizeSector(context.Context, abi.SectorNumber) error { return nil } -func (sb *SBMock) DropStaged(context.Context, abi.SectorNumber) error { - return nil -} - -func (sb *SBMock) SectorPath(typ fs.DataType, sectorID abi.SectorNumber) (fs.SectorPath, error) { - panic("implement me") -} - -func (sb *SBMock) AllocSectorPath(typ fs.DataType, sectorID abi.SectorNumber, cache bool) (fs.SectorPath, error) { - panic("implement me") -} - -func (sb *SBMock) ReleaseSector(fs.DataType, fs.SectorPath) { - panic("implement me") -} - func (m mockVerif) VerifyElectionPost(ctx context.Context, pvi abi.PoStVerifyInfo) (bool, error) { panic("implement me") } @@ -383,4 +342,4 @@ func (m mockVerif) GenerateDataCommitment(ssize abi.PaddedPieceSize, pieces []ab var MockVerifier = mockVerif{} var _ sectorbuilder.Verifier = MockVerifier -var _ sectorbuilder.Interface = &SBMock{} +var _ sectorbuilder.Basic = &SBMock{} From 5a4f917c6061c15a50acdd6dc15b65cdfbe7fac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 28 Feb 2020 19:06:59 +0100 Subject: [PATCH 02/24] 4-stage v23 lotus-bench --- api/api_storage.go | 31 +++++++++++++--- api/apistruct/struct.go | 4 +- chain/types/voucher.go | 2 +- cmd/lotus-bench/main.go | 80 +++++++++++++++++++++++++++------------- storage/sbmock/sbmock.go | 4 +- 5 files changed, 85 insertions(+), 36 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index 57e1484b7..96a71ba03 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -7,8 +7,6 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/ipfs/go-cid" - - "github.com/filecoin-project/go-sectorbuilder" ) // alias because cbor-gen doesn't like non-alias types @@ -105,12 +103,35 @@ type StorageMiner interface { SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error - WorkerStats(context.Context) (sectorbuilder.WorkerStats, error) + WorkerStats(context.Context) (WorkerStats, error) // WorkerQueue registers a remote worker - WorkerQueue(context.Context, sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) + WorkerQueue(context.Context, WorkerCfg) (<-chan WorkerTask, error) - WorkerDone(ctx context.Context, task uint64, res sectorbuilder.SealRes) error + WorkerDone(ctx context.Context, task uint64, res SealRes) error +} + +type WorkerStats struct { + Total int + Free int + + AddPieceWait int + PreCommitWait int + CommitWait int + UnsealWait int +} + +type WorkerTask struct { +} +type WorkerCfg struct { + NoPreCommit bool + NoCommit bool +} +type SealRes struct { + Err string + GoErr error `json:"-"` + + Proof []byte } type SectorLog struct { diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index c53619990..09aa89a51 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -101,7 +101,7 @@ type FullNodeStruct struct { ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error) `perm:"read"` ClientListDeals func(ctx context.Context) ([]api.DealInfo, error) `perm:"write"` ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, path string) error `perm:"admin"` - ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) `perm:"read"` + ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) `perm:"read"` StateMinerSectors func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"` StateMinerProvingSet func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"` @@ -144,7 +144,7 @@ type FullNodeStruct struct { PaychVoucherCheckValid func(context.Context, address.Address, *paych.SignedVoucher) error `perm:"read"` PaychVoucherCheckSpendable func(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (bool, error) `perm:"read"` PaychVoucherAdd func(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error) `perm:"write"` - PaychVoucherCreate func(context.Context, address.Address, big.Int, uint64) (*paych.SignedVoucher, error) `perm:"sign"` + PaychVoucherCreate func(context.Context, address.Address, big.Int, uint64) (*paych.SignedVoucher, error) `perm:"sign"` PaychVoucherList func(context.Context, address.Address) ([]*paych.SignedVoucher, error) `perm:"write"` PaychVoucherSubmit func(context.Context, address.Address, *paych.SignedVoucher) (cid.Cid, error) `perm:"sign"` } diff --git a/chain/types/voucher.go b/chain/types/voucher.go index bbf7e1ed0..687109c33 100644 --- a/chain/types/voucher.go +++ b/chain/types/voucher.go @@ -1,6 +1,6 @@ package types -import( +import ( "encoding/base64" "github.com/filecoin-project/specs-actors/actors/builtin/paych" diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 073e40973..27eb5141b 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -14,8 +14,8 @@ import ( "github.com/docker/go-units" paramfetch "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/go-sectorbuilder/fs" "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" @@ -23,6 +23,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-sectorbuilder" + lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" @@ -44,11 +45,13 @@ type BenchResults struct { } type SealingResult struct { - AddPiece time.Duration - PreCommit time.Duration - Commit time.Duration - Verify time.Duration - Unseal time.Duration + AddPiece time.Duration + PreCommit1 time.Duration + PreCommit2 time.Duration + Commit1 time.Duration + Commit2 time.Duration + Verify time.Duration + Unseal time.Duration } func main() { @@ -70,7 +73,7 @@ func main() { }, &cli.StringFlag{ Name: "sector-size", - Value: "1GiB", + Value: "512MiB", Usage: "size of the sectors in bytes, i.e. 32GiB", }, &cli.BoolFlag{ @@ -80,7 +83,7 @@ func main() { &cli.StringFlag{ Name: "miner-addr", Usage: "pass miner address (only necessary if using existing sectorbuilder)", - Value: "t0101", + Value: "t01000", }, &cli.StringFlag{ Name: "benchmark-existing-sectorbuilder", @@ -146,13 +149,10 @@ func main() { return err } - mds := datastore.NewMapDatastore() cfg := §orbuilder.Config{ Miner: maddr, SealProofType: spt, PoStProofType: ppt, - WorkerThreads: 2, - Paths: sectorbuilder.SimplePath(sbdir), } if robench == "" { @@ -164,7 +164,14 @@ func main() { if err := paramfetch.GetParams(build.ParametersJson(), uint64(sectorSize)); err != nil { return xerrors.Errorf("getting params: %w", err) } - sb, err := sectorbuilder.New(cfg, mds) + + sbfs := &fs.Basic{ + Miner: maddr, + NextID: 1, + Root: sbdir, + } + + sb, err := sectorbuilder.New(sbfs, cfg) if err != nil { return err } @@ -195,14 +202,22 @@ func main() { trand := sha256.Sum256([]byte(c.String("ticket-preimage"))) ticket := abi.SealRandomness(trand[:]) - log.Info("Running replication...") + log.Info("Running replication(1)...") pieces := []abi.PieceInfo{pi} - commR, commD, err := sb.SealPreCommit(context.TODO(), i, ticket, pieces) + pc1o, err := sb.SealPreCommit1(context.TODO(), i, ticket, pieces) if err != nil { return xerrors.Errorf("commit: %w", err) } - precommit := time.Now() + precommit1 := time.Now() + + log.Info("Running replication(2)...") + commR, commD, err := sb.SealPreCommit2(context.TODO(), i, pc1o) + if err != nil { + return xerrors.Errorf("commit: %w", err) + } + + precommit2 := time.Now() sealedSectors = append(sealedSectors, abi.SectorInfo{ RegisteredProof: ppt, @@ -215,13 +230,22 @@ func main() { Value: []byte{1, 2, 3, 4, 5}, } - log.Info("Generating PoRep for sector") - proof, err := sb.SealCommit(context.TODO(), i, ticket, seed.Value, pieces, commR, commD) + log.Info("Generating PoRep for sector (1)") + c1o, err := sb.SealCommit1(context.TODO(), i, ticket, seed.Value, pieces, commR, commD) if err != nil { return err } - sealcommit := time.Now() + sealcommit1 := time.Now() + + log.Info("Generating PoRep for sector (2)") + + proof, err := sb.SealCommit2(context.TODO(), i, c1o) + if err != nil { + return err + } + + sealcommit2 := time.Now() svi := abi.SealVerifyInfo{ SectorID: abi.SectorID{Miner: abi.ActorID(mid), Number: i}, @@ -263,11 +287,13 @@ func main() { unseal := time.Now() sealTimings = append(sealTimings, SealingResult{ - AddPiece: addpiece.Sub(start), - PreCommit: precommit.Sub(addpiece), - Commit: sealcommit.Sub(precommit), - Verify: verifySeal.Sub(sealcommit), - Unseal: unseal.Sub(verifySeal), + AddPiece: addpiece.Sub(start), + PreCommit1: precommit1.Sub(addpiece), + PreCommit2: precommit2.Sub(precommit1), + Commit1: sealcommit1.Sub(precommit2), + Commit2: sealcommit2.Sub(sealcommit1), + Verify: verifySeal.Sub(sealcommit2), + Unseal: unseal.Sub(verifySeal), }) } @@ -390,11 +416,13 @@ func main() { fmt.Println(string(data)) } else { - fmt.Printf("results (%d)\n", sectorSize) + fmt.Printf("----\nresults (v23) (%d)\n", sectorSize) if robench == "" { fmt.Printf("seal: addPiece: %s (%s)\n", bo.SealingResults[0].AddPiece, bps(bo.SectorSize, bo.SealingResults[0].AddPiece)) // TODO: average across multiple sealings - fmt.Printf("seal: preCommit: %s (%s)\n", bo.SealingResults[0].PreCommit, bps(bo.SectorSize, bo.SealingResults[0].PreCommit)) - fmt.Printf("seal: commit: %s (%s)\n", bo.SealingResults[0].Commit, bps(bo.SectorSize, bo.SealingResults[0].Commit)) + fmt.Printf("seal: preCommit phase 1: %s (%s)\n", bo.SealingResults[0].PreCommit1, bps(bo.SectorSize, bo.SealingResults[0].PreCommit1)) + fmt.Printf("seal: preCommit phase 2: %s (%s)\n", bo.SealingResults[0].PreCommit2, bps(bo.SectorSize, bo.SealingResults[0].PreCommit2)) + fmt.Printf("seal: commit phase 1: %s (%s)\n", bo.SealingResults[0].Commit1, bps(bo.SectorSize, bo.SealingResults[0].Commit1)) + fmt.Printf("seal: commit phase 2: %s (%s)\n", bo.SealingResults[0].Commit2, bps(bo.SectorSize, bo.SealingResults[0].Commit2)) fmt.Printf("seal: verify: %s\n", bo.SealingResults[0].Verify) if !c.Bool("skip-unseal") { fmt.Printf("unseal: %s (%s)\n", bo.SealingResults[0].Unseal, bps(bo.SectorSize, bo.SealingResults[0].Unseal)) diff --git a/storage/sbmock/sbmock.go b/storage/sbmock/sbmock.go index e3906c3ce..cc2ad894d 100644 --- a/storage/sbmock/sbmock.go +++ b/storage/sbmock/sbmock.go @@ -198,7 +198,7 @@ func (sb *SBMock) SealCommit1(ctx context.Context, sid abi.SectorNumber, ticket var out [32]byte for i := range out { - out[i] = unsealed.Bytes()[i] + sealedCid.Bytes()[31-i] - ticket[i]*seed[i] ^ byte(sid & 0xff) + out[i] = unsealed.Bytes()[i] + sealedCid.Bytes()[31-i] - ticket[i]*seed[i] ^ byte(sid&0xff) } return out[:], nil @@ -207,7 +207,7 @@ func (sb *SBMock) SealCommit1(ctx context.Context, sid abi.SectorNumber, ticket func (sb *SBMock) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (proof []byte, err error) { var out [32]byte for i := range out { - out[i] = phase1Out[i] ^ byte(sectorNum & 0xff) + out[i] = phase1Out[i] ^ byte(sectorNum&0xff) } return out[:], nil From 9794c04b943337b6f13ccb1271220941c96ebbb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 29 Feb 2020 03:31:14 +0100 Subject: [PATCH 03/24] Update sectorbuilder with Fr fix --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6adb11e0c..8a767e7cd 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/filecoin-project/go-fil-markets v0.0.0-20200228192824-ee51014cc8c6 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 - github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200228064721-be233370d8e9 + github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200229022239-442fe78a3168 github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/specs-actors v0.0.0-20200229011003-1d726e3afd04 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index b582d3542..f48aed951 100644 --- a/go.sum +++ b/go.sum @@ -126,6 +126,8 @@ github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200227214225-a550d026797 github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200227214225-a550d0267977/go.mod h1:tzTc9BxxSbjlIzhFwm5h9oBkXKkRuLxeiWspntwnKyw= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200228064721-be233370d8e9 h1:P4IMEn3GDtAJzIVL/DqwHH8jotvafn6FLuxFhUNKhEI= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200228064721-be233370d8e9/go.mod h1:gEQJVRVqQX8Vx02IosTdC2UA4l2MgHfG3POJEI2GIpc= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200229022239-442fe78a3168 h1:6fBZezecouqHNRi4Q4YlH7QqgJ3yZZ7adMBpUBAx4F8= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200229022239-442fe78a3168/go.mod h1:gEQJVRVqQX8Vx02IosTdC2UA4l2MgHfG3POJEI2GIpc= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 h1:k9qVR9ItcziSB2rxtlkN/MDWNlbsI6yzec+zjUatLW0= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= From 342e85d37882c7cfa5a0c4ae261107375a4f931b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 29 Feb 2020 03:31:25 +0100 Subject: [PATCH 04/24] bench: flag to save commit2 input --- cmd/lotus-bench/main.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 0070ae2df..3bf86e93f 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -54,6 +54,12 @@ type SealingResult struct { Unseal time.Duration } +type Commit2In struct { + SectorNum int64 + Phase1Out []byte + SectorSize uint64 // for stats +} + func main() { logging.SetLogLevel("*", "INFO") @@ -97,6 +103,10 @@ func main() { Name: "skip-unseal", Usage: "skip the unseal portion of the benchmark", }, + &cli.StringFlag{ + Name: "save-commit2-input", + Usage: "Save commit2 input to a file", + }, }, Action: func(c *cli.Context) error { if c.Bool("no-gpu") { @@ -240,6 +250,23 @@ func main() { log.Info("Generating PoRep for sector (2)") + if c.String("save-commit2-input") != "" { + c2in := Commit2In{ + SectorNum: int64(i), + Phase1Out: c1o, + SectorSize: 0, + } + + b, err := json.Marshal(&c2in) + if err != nil { + return err + } + + if err := ioutil.WriteFile(c.String("save-commit2-input"), b, 0664); err != nil { + log.Warnf("%+v", err) + } + } + proof, err := sb.SealCommit2(context.TODO(), i, c1o) if err != nil { return err From ee3e631b1aed97e68ece88fa6ba167b59c5931c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 1 Mar 2020 03:52:23 +0100 Subject: [PATCH 05/24] bench: Prove command --- cmd/lotus-bench/main.go | 87 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 6 deletions(-) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 3bf86e93f..a131a5c9b 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -55,9 +55,9 @@ type SealingResult struct { } type Commit2In struct { - SectorNum int64 - Phase1Out []byte - SectorSize uint64 // for stats + SectorNum int64 + Phase1Out []byte + SectorSize uint64 } func main() { @@ -71,6 +71,9 @@ func main() { Name: "lotus-bench", Usage: "Benchmark performance of lotus on your hardware", Version: build.UserVersion, + Commands: []*cli.Command{ + proveCmd, + }, Flags: []cli.Flag{ &cli.StringFlag{ Name: "storage-dir", @@ -252,7 +255,7 @@ func main() { if c.String("save-commit2-input") != "" { c2in := Commit2In{ - SectorNum: int64(i), + SectorNum: int64(i), Phase1Out: c1o, SectorSize: 0, } @@ -393,7 +396,7 @@ func main() { Candidates: candidates[:1], Proofs: proof1, EligibleSectors: sealedSectors, - Prover: abi.ActorID(mid), + Prover: mid, ChallengeCount: ccount, } ok, err := sectorbuilder.ProofVerifier.VerifyElectionPost(context.TODO(), pvi1) @@ -411,7 +414,7 @@ func main() { Candidates: candidates[:1], Proofs: proof2, EligibleSectors: sealedSectors, - Prover: abi.ActorID(mid), + Prover: mid, ChallengeCount: ccount, } @@ -471,6 +474,78 @@ func main() { } } +var proveCmd = &cli.Command{ + Name: "prove", + Usage: "Benchmark a proof computation", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "no-gpu", + Usage: "disable gpu usage for the benchmark run", + }, + }, + Action: func(c *cli.Context) error { + if c.Bool("no-gpu") { + os.Setenv("BELLMAN_NO_GPU", "1") + } + + if !c.Args().Present() { + return xerrors.Errorf("Usage: lotus-bench prove [input.json]") + } + + inb, err := ioutil.ReadFile(c.Args().First()) + if err != nil { + return xerrors.Errorf("reading input file: %w", err) + } + + var c2in Commit2In + if err := json.Unmarshal(inb, &c2in); err != nil { + return xerrors.Errorf("unmarshalling input file: %w", err) + } + + if err := paramfetch.GetParams(build.ParametersJson(), c2in.SectorSize); err != nil { + return xerrors.Errorf("getting params: %w", err) + } + + maddr, err := address.NewFromString(c.String("miner-addr")) + if err != nil { + return err + } + + ppt, spt, err := lapi.ProofTypeFromSectorSize(abi.SectorSize(c2in.SectorSize)) + if err != nil { + return err + } + + cfg := §orbuilder.Config{ + Miner: maddr, + SealProofType: spt, + PoStProofType: ppt, + } + + sb, err := sectorbuilder.New(nil, cfg) + if err != nil { + return err + } + + start := time.Now() + + proof, err := sb.SealCommit2(context.TODO(), abi.SectorNumber(c2in.SectorNum), c2in.Phase1Out) + if err != nil { + return err + } + + sealCommit2 := time.Now() + + fmt.Printf("proof: %x\n", proof) + + fmt.Printf("----\nresults (v23) (%d)\n", c2in.SectorSize) + dur := sealCommit2.Sub(start) + + fmt.Printf("seal: commit phase 2: %s (%s)\n", dur, bps(abi.SectorSize(c2in.SectorSize), dur)) + return nil + }, +} + func bps(data abi.SectorSize, d time.Duration) string { bdata := new(big.Int).SetUint64(uint64(data)) bdata = bdata.Mul(bdata, big.NewInt(time.Second.Nanoseconds())) From f4310e8f69cbe5021d634090404445c83c8b386b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 1 Mar 2020 05:26:10 +0100 Subject: [PATCH 06/24] bench: set sector size in saved commit2 inputs --- cmd/lotus-bench/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index a131a5c9b..684ae24cc 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -257,7 +257,7 @@ func main() { c2in := Commit2In{ SectorNum: int64(i), Phase1Out: c1o, - SectorSize: 0, + SectorSize: uint64(sectorSize), } b, err := json.Marshal(&c2in) @@ -278,7 +278,7 @@ func main() { sealcommit2 := time.Now() svi := abi.SealVerifyInfo{ - SectorID: abi.SectorID{Miner: abi.ActorID(mid), Number: i}, + SectorID: abi.SectorID{Miner: mid, Number: i}, OnChain: abi.OnChainSealVerifyInfo{ SealedCID: commR, InteractiveEpoch: seed.Epoch, From 98dbb2f70fd948070d1c7eec2ad5f35f4867174e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 1 Mar 2020 22:05:13 +0100 Subject: [PATCH 07/24] bench: set correct registered proofs on things --- cmd/lotus-bench/main.go | 5 +++-- go.sum | 7 ------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 684ae24cc..ce825a9e1 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -233,7 +233,7 @@ func main() { precommit2 := time.Now() sealedSectors = append(sealedSectors, abi.SectorInfo{ - RegisteredProof: ppt, + RegisteredProof: spt, SectorNumber: i, SealedCID: commR, }) @@ -282,7 +282,7 @@ func main() { OnChain: abi.OnChainSealVerifyInfo{ SealedCID: commR, InteractiveEpoch: seed.Epoch, - RegisteredProof: ppt, + RegisteredProof: spt, Proof: proof, DealIDs: nil, SectorNumber: i, @@ -368,6 +368,7 @@ func main() { var candidates []abi.PoStCandidate for _, c := range fcandidates { + c.Candidate.RegisteredProof = ppt candidates = append(candidates, c.Candidate) } diff --git a/go.sum b/go.sum index f48aed951..54768a163 100644 --- a/go.sum +++ b/go.sum @@ -120,12 +120,6 @@ github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 h github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200226210935-4739f8749f56 h1:CqldkHf9HtAsewneyOZdi19Btc6kEvktVVeZsP9N7NQ= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200226210935-4739f8749f56/go.mod h1:tzTc9BxxSbjlIzhFwm5h9oBkXKkRuLxeiWspntwnKyw= -github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200227214127-332d57b390aa h1:EVToQ5L+49psMvYP5ICsOcFRua79oDMyYQAeEqxkzwU= -github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200227214127-332d57b390aa/go.mod h1:u8n91UJUua9zEiyAPw8wRAgUEoyTH8BBM9LWJ5WuxOM= -github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200227214225-a550d0267977 h1:Q4VnMiorS/AcZMDHtoi+7lVzkTj+qDa0bmgMkwmvxIM= -github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200227214225-a550d0267977/go.mod h1:tzTc9BxxSbjlIzhFwm5h9oBkXKkRuLxeiWspntwnKyw= -github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200228064721-be233370d8e9 h1:P4IMEn3GDtAJzIVL/DqwHH8jotvafn6FLuxFhUNKhEI= -github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200228064721-be233370d8e9/go.mod h1:gEQJVRVqQX8Vx02IosTdC2UA4l2MgHfG3POJEI2GIpc= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200229022239-442fe78a3168 h1:6fBZezecouqHNRi4Q4YlH7QqgJ3yZZ7adMBpUBAx4F8= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200229022239-442fe78a3168/go.mod h1:gEQJVRVqQX8Vx02IosTdC2UA4l2MgHfG3POJEI2GIpc= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 h1:k9qVR9ItcziSB2rxtlkN/MDWNlbsI6yzec+zjUatLW0= @@ -134,7 +128,6 @@ github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIi github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200226200336-94c9b92b2775/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= -github.com/filecoin-project/specs-actors v0.0.0-20200228215954-2c5be7cfad99/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= github.com/filecoin-project/specs-actors v0.0.0-20200229011003-1d726e3afd04 h1:O343OeQLkLWLj5ZqQ5nhevAGBTeB5LioiA53ddScqdY= github.com/filecoin-project/specs-actors v0.0.0-20200229011003-1d726e3afd04/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= From 3abb59a550a76882ac7c63833eb58bd7e0f19046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 3 Mar 2020 01:45:32 +0100 Subject: [PATCH 08/24] sealmgr: Interfaces; simple initial impl --- api/api_storage.go | 50 ++++-------------- api/apistruct/struct.go | 17 +++--- storage/sealmgr/simple.go | 92 +++++++++++++++++++++++++++++++++ storage/sealmgr/task.go | 9 ++++ storage/sealmgr/types.go | 80 ++++++++++++++++++++++++++++ storage/sealmgr/worker_local.go | 74 ++++++++++++++++++++++++++ 6 files changed, 272 insertions(+), 50 deletions(-) create mode 100644 storage/sealmgr/simple.go create mode 100644 storage/sealmgr/task.go create mode 100644 storage/sealmgr/types.go create mode 100644 storage/sealmgr/worker_local.go diff --git a/api/api_storage.go b/api/api_storage.go index 96a71ba03..0988d3bce 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -1,12 +1,14 @@ package api import ( - "bytes" "context" + "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lotus/storage/sealmgr" ) // alias because cbor-gen doesn't like non-alias types @@ -103,30 +105,14 @@ type StorageMiner interface { SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error - WorkerStats(context.Context) (WorkerStats, error) + /*WorkerStats(context.Context) (sealsched.WorkerStats, error)*/ - // WorkerQueue registers a remote worker + /*// WorkerQueue registers a remote worker WorkerQueue(context.Context, WorkerCfg) (<-chan WorkerTask, error) - WorkerDone(ctx context.Context, task uint64, res SealRes) error + WorkerDone(ctx context.Context, task uint64, res SealRes) error*/ } -type WorkerStats struct { - Total int - Free int - - AddPieceWait int - PreCommitWait int - CommitWait int - UnsealWait int -} - -type WorkerTask struct { -} -type WorkerCfg struct { - NoPreCommit bool - NoCommit bool -} type SealRes struct { Err string GoErr error `json:"-"` @@ -150,8 +136,8 @@ type SectorInfo struct { CommR *cid.Cid Proof []byte Deals []abi.DealID - Ticket SealTicket - Seed SealSeed + Ticket sealmgr.SealTicket + Seed sealmgr.SealSeed Retries uint64 LastErr string @@ -168,21 +154,3 @@ type SealedRef struct { type SealedRefs struct { Refs []SealedRef } - -type SealTicket struct { - Value abi.SealRandomness - Epoch abi.ChainEpoch -} - -type SealSeed struct { - Value abi.InteractiveSealRandomness - Epoch abi.ChainEpoch -} - -func (st *SealTicket) Equals(ost *SealTicket) bool { - return bytes.Equal(st.Value, ost.Value) && st.Epoch == ost.Epoch -} - -func (st *SealSeed) Equals(ost *SealSeed) bool { - return bytes.Equal(st.Value, ost.Value) && st.Epoch == ost.Epoch -} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 7d5e0ab44..291635e14 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -9,7 +9,6 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/storagemarket" - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/abi/big" "github.com/filecoin-project/specs-actors/actors/builtin/miner" @@ -168,10 +167,10 @@ type StorageMinerStruct struct { SectorsRefs func(context.Context) (map[string][]api.SealedRef, error) `perm:"read"` SectorsUpdate func(context.Context, abi.SectorNumber, api.SectorState) error `perm:"write"` - WorkerStats func(context.Context) (sectorbuilder.WorkerStats, error) `perm:"read"` - - WorkerQueue func(ctx context.Context, cfg sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) `perm:"admin"` // TODO: worker perm - WorkerDone func(ctx context.Context, task uint64, res sectorbuilder.SealRes) error `perm:"admin"` + /* WorkerStats func(context.Context) (sealsched.WorkerStats, error) `perm:"read"` + */ + /* WorkerQueue func(ctx context.Context, cfg sealsched.WorkerCfg) (<-chan sealsched.WorkerTask, error) `perm:"admin"` // TODO: worker perm + WorkerDone func(ctx context.Context, task uint64, res sealsched.SealRes) error `perm:"admin"`*/ } } @@ -601,17 +600,17 @@ func (c *StorageMinerStruct) SectorsUpdate(ctx context.Context, id abi.SectorNum return c.Internal.SectorsUpdate(ctx, id, state) } -func (c *StorageMinerStruct) WorkerStats(ctx context.Context) (sectorbuilder.WorkerStats, error) { +/*func (c *StorageMinerStruct) WorkerStats(ctx context.Context) (sealsched.WorkerStats, error) { return c.Internal.WorkerStats(ctx) -} +}*/ -func (c *StorageMinerStruct) WorkerQueue(ctx context.Context, cfg sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) { +/*func (c *StorageMinerStruct) WorkerQueue(ctx context.Context, cfg sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) { return c.Internal.WorkerQueue(ctx, cfg) } func (c *StorageMinerStruct) WorkerDone(ctx context.Context, task uint64, res sectorbuilder.SealRes) error { return c.Internal.WorkerDone(ctx, task, res) -} +}*/ var _ api.Common = &CommonStruct{} var _ api.FullNode = &FullNodeStruct{} diff --git a/storage/sealmgr/simple.go b/storage/sealmgr/simple.go new file mode 100644 index 000000000..7d8f13298 --- /dev/null +++ b/storage/sealmgr/simple.go @@ -0,0 +1,92 @@ +package sealmgr + +import ( + "context" + "io" + "sync" + + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/storedcounter" + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/specs-actors/actors/abi" + + ffi "github.com/filecoin-project/filecoin-ffi" +) + +// Simple implements a very basic storage manager which has one local worker, +// running one thing locally +type Simple struct { + sc *storedcounter.StoredCounter + maddr address.Address + + rateLimiter sync.Mutex + worker Worker +} + +func NewSimpleManager(sc *storedcounter.StoredCounter, maddr address.Address, sb sectorbuilder.Basic) (*Simple, error) { + mid, err := address.IDFromAddress(maddr) + if err != nil { + return nil, xerrors.Errorf("get miner id: %w", err) + } + + w := &LocalWorker{ + sealer: sb, + mid: abi.ActorID(mid), + } + + return &Simple{ + sc: sc, + maddr: maddr, + worker: w, + }, nil +} + +func (s *Simple) NewSector() (SectorInfo, error) { + n, err := s.sc.Next() + if err != nil { + return SectorInfo{}, xerrors.Errorf("acquire sector number: %w", err) + } + + mid, err := address.IDFromAddress(s.maddr) + if err != nil { + return SectorInfo{}, xerrors.Errorf("get miner id: %w", err) + } + + return SectorInfo{ + ID: abi.SectorID{ + Miner: abi.ActorID(mid), + Number: abi.SectorNumber(n), + }, + }, nil +} + +func (s *Simple) AddPiece(ctx context.Context, si SectorInfo, sz abi.UnpaddedPieceSize, r io.Reader) (cid.Cid, SectorInfo, error) { + s.rateLimiter.Lock() + defer s.rateLimiter.Unlock() + + return s.worker.AddPiece(ctx, si, sz, r) +} + +func (s *Simple) RunSeal(ctx context.Context, task TaskType, si SectorInfo) (SectorInfo, error) { + s.rateLimiter.Lock() + defer s.rateLimiter.Unlock() + + return s.worker.Run(ctx, task, si) +} + +func (s *Simple) GenerateEPostCandidates(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]ffi.PoStCandidateWithTicket, error) { + return s.worker.(*LocalWorker).sealer.GenerateEPostCandidates(sectorInfo, challengeSeed, faults) +} + +func (s *Simple) GenerateFallbackPoSt(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]ffi.PoStCandidateWithTicket, []abi.PoStProof, error) { + return s.worker.(*LocalWorker).sealer.GenerateFallbackPoSt(sectorInfo, challengeSeed, faults) +} + +func (s *Simple) ComputeElectionPoSt(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, winners []abi.PoStCandidate) ([]abi.PoStProof, error) { + return s.worker.(*LocalWorker).sealer.ComputeElectionPoSt(sectorInfo, challengeSeed, winners) +} + +var _ Manager = &Simple{} diff --git a/storage/sealmgr/task.go b/storage/sealmgr/task.go new file mode 100644 index 000000000..cbe7b5344 --- /dev/null +++ b/storage/sealmgr/task.go @@ -0,0 +1,9 @@ +package sealmgr + +type TaskType string + +const ( + TTPreCommit1 TaskType = "seal/v0/precommit/1" + TTPreCommit2 TaskType = "seal/v0/precommit/2" // Commit1 is called here too + TTCommit2 TaskType = "seal/v0/commit/2" +) diff --git a/storage/sealmgr/types.go b/storage/sealmgr/types.go new file mode 100644 index 000000000..a81346a97 --- /dev/null +++ b/storage/sealmgr/types.go @@ -0,0 +1,80 @@ +package sealmgr + +import ( + "bytes" + "context" + "io" + + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/specs-actors/actors/abi" +) + + +type SealTicket struct { + Value abi.SealRandomness + Epoch abi.ChainEpoch +} + +type SealSeed struct { + Value abi.InteractiveSealRandomness + Epoch abi.ChainEpoch +} + +func (st *SealTicket) Equals(ost *SealTicket) bool { + return bytes.Equal(st.Value, ost.Value) && st.Epoch == ost.Epoch +} + +func (st *SealSeed) Equals(ost *SealSeed) bool { + return bytes.Equal(st.Value, ost.Value) && st.Epoch == ost.Epoch +} + +// SectorInfo holds all sector-related metadata +type SectorInfo struct { + ID abi.SectorID + + Pieces []abi.PieceInfo + + Ticket SealTicket + Seed SealSeed + + PreCommit1Out []byte + + Sealed *cid.Cid + Unsealed *cid.Cid + + CommitInput []byte + Proof []byte +} + +func (si SectorInfo) PieceSizes() []abi.UnpaddedPieceSize { + out := make([]abi.UnpaddedPieceSize, len(si.Pieces)) + for i := range out { + out[i] = si.Pieces[i].Size.Unpadded() + } + + return nil +} + +type Worker interface { + AddPiece(context.Context, SectorInfo, abi.UnpaddedPieceSize, io.Reader) (cid.Cid, SectorInfo, error) + Run(context.Context, TaskType, SectorInfo) (SectorInfo, error) +} + +type Manager interface { + // NewSector allocates staging area for data + NewSector() (SectorInfo, error) + + // AddPiece appends the piece to the specified sector. Returns PieceCID, and + // mutated sector info + // + // Note: The passed reader can support other transfer mechanisms, making + // it possible to move the data between data transfer module and workers + AddPiece(context.Context, SectorInfo, abi.UnpaddedPieceSize, io.Reader) (cid.Cid, SectorInfo, error) + + RunSeal(ctx context.Context, task TaskType, si SectorInfo) (SectorInfo, error) + + // Storage manager forwards proving calls + sectorbuilder.Prover +} diff --git a/storage/sealmgr/worker_local.go b/storage/sealmgr/worker_local.go new file mode 100644 index 000000000..05435967b --- /dev/null +++ b/storage/sealmgr/worker_local.go @@ -0,0 +1,74 @@ +package sealmgr + +import ( + "context" + "io" + + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/specs-actors/actors/abi" +) + +type LocalWorker struct { + sealer sectorbuilder.Basic + mid abi.ActorID +} + +func (w *LocalWorker) Run(ctx context.Context, task TaskType, si SectorInfo) (SectorInfo, error) { + if si.ID.Miner != w.mid { + return si, xerrors.Errorf("received a task with wrong actor id; worker for %d, task for %d", w.mid, si.ID.Miner) + } + + switch task { + case TTPreCommit1: + pco, err := w.sealer.SealPreCommit1(ctx, si.ID.Number, si.Ticket.Value, si.Pieces) + if err != nil { + return si, xerrors.Errorf("calling sealer: %w", err) + } + si.PreCommit1Out = pco + case TTPreCommit2: + sealed, unsealed, err := w.sealer.SealPreCommit2(ctx, si.ID.Number, si.PreCommit1Out) + if err != nil { + return si, xerrors.Errorf("calling sealer (precommit2): %w", err) + } + + si.Sealed = &sealed + si.Unsealed = &unsealed + + // We also call Commit1 here as it only grabs some inputs for the snark, + // which is very fast (<1s), and it doesn't really make sense to have a separate + // task type for it + + c2in, err := w.sealer.SealCommit1(ctx, si.ID.Number, si.Ticket.Value, si.Seed.Value, si.Pieces, *si.Sealed, *si.Unsealed) + if err != nil { + return si, xerrors.Errorf("calling sealer (commit1): %w", err) + } + + si.CommitInput = c2in + case TTCommit2: + proof, err := w.sealer.SealCommit2(ctx, si.ID.Number, si.CommitInput) + if err != nil { + return SectorInfo{}, xerrors.Errorf("calling sealer: %w", err) + } + + si.Proof = proof + default: + return si, xerrors.Errorf("unknown task type '%s'", task) + } + + return si, nil +} + +func (w *LocalWorker) AddPiece(ctx context.Context, si SectorInfo, sz abi.UnpaddedPieceSize, r io.Reader) (cid.Cid, SectorInfo, error) { + pi, err := w.sealer.AddPiece(ctx, sz, si.ID.Number, r, si.PieceSizes()) + if err != nil { + return cid.Cid{}, SectorInfo{}, xerrors.Errorf("addPiece on local worker: %w", err) + } + + si.Pieces = append(si.Pieces, pi) + return pi.PieceCID, si, nil +} + +var _ Worker = &LocalWorker{} From a0dbb6bdd6df740b5e230524f02f5c61d4fd9dc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 3 Mar 2020 23:19:22 +0100 Subject: [PATCH 09/24] Storage Manager refactor --- api/api_storage.go | 25 ++- cmd/lotus-seed/main.go | 150 ------------------ cmd/lotus-seed/seed/seed.go | 77 +++++---- cmd/lotus-storage-miner/info.go | 5 +- cmd/lotus-storage-miner/init.go | 91 +++-------- documentation/en/local-dev-net.md | 8 +- markets/retrievaladapter/provider.go | 13 +- node/builder.go | 24 +-- node/config/def.go | 16 +- node/config/storage.go | 68 ++++++++ node/impl/storminer.go | 31 ++-- node/modules/storageminer.go | 90 ++++------- node/repo/fsrepo.go | 15 +- node/repo/interface.go | 4 + node/repo/memrepo.go | 13 ++ storage/fpost_run.go | 4 +- storage/fpost_sched.go | 4 +- storage/miner.go | 34 ++-- storage/sealing/garbage.go | 8 +- storage/sealing/sealing.go | 14 +- storage/sealing/states.go | 28 ++-- storage/sealing/types.go | 3 +- .../sealmgr/advmgr/lotus_storage_manager.go | 102 ++++++++++++ storage/sealmgr/simple.go | 74 +++++---- storage/sealmgr/types.go | 72 ++------- storage/sealmgr/worker_local.go | 65 +------- storage/sectorblocks/blocks.go | 5 +- 27 files changed, 459 insertions(+), 584 deletions(-) create mode 100644 node/config/storage.go create mode 100644 storage/sealmgr/advmgr/lotus_storage_manager.go diff --git a/api/api_storage.go b/api/api_storage.go index 0988d3bce..e8712b3b4 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -1,14 +1,13 @@ package api import ( + "bytes" "context" "github.com/ipfs/go-cid" "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" - - "github.com/filecoin-project/lotus/storage/sealmgr" ) // alias because cbor-gen doesn't like non-alias types @@ -136,8 +135,8 @@ type SectorInfo struct { CommR *cid.Cid Proof []byte Deals []abi.DealID - Ticket sealmgr.SealTicket - Seed sealmgr.SealSeed + Ticket SealTicket + Seed SealSeed Retries uint64 LastErr string @@ -154,3 +153,21 @@ type SealedRef struct { type SealedRefs struct { Refs []SealedRef } + +type SealTicket struct { + Value abi.SealRandomness + Epoch abi.ChainEpoch +} + +type SealSeed struct { + Value abi.InteractiveSealRandomness + Epoch abi.ChainEpoch +} + +func (st *SealTicket) Equals(ost *SealTicket) bool { + return bytes.Equal(st.Value, ost.Value) && st.Epoch == ost.Epoch +} + +func (st *SealSeed) Equals(ost *SealSeed) bool { + return bytes.Equal(st.Value, ost.Value) && st.Epoch == ost.Epoch +} \ No newline at end of file diff --git a/cmd/lotus-seed/main.go b/cmd/lotus-seed/main.go index 2096ac0ff..2daa83325 100644 --- a/cmd/lotus-seed/main.go +++ b/cmd/lotus-seed/main.go @@ -6,18 +6,12 @@ import ( "fmt" "io/ioutil" "os" - "path/filepath" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - badger "github.com/ipfs/go-ds-badger2" logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" - "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/abi/big" @@ -38,7 +32,6 @@ func main() { preSealCmd, aggregateManifestsCmd, - aggregateSectorDirsCmd, } app := &cli.App{ @@ -174,149 +167,6 @@ var aggregateManifestsCmd = &cli.Command{ }, } -var aggregateSectorDirsCmd = &cli.Command{ - Name: "aggregate-sector-dirs", - Usage: "aggregate a set of preseal manifests into a single file", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "miner", - Usage: "Specify address of miner to aggregate sectorbuilders for", - }, - &cli.StringFlag{ - Name: "dest", - Usage: "specify directory to create aggregate sector store in", - }, - &cli.Uint64Flag{ - Name: "sector-size", - Usage: "specify size of sectors to aggregate", - Value: 32 * 1024 * 1024 * 1024, - }, - }, - Action: func(cctx *cli.Context) error { - if cctx.String("miner") == "" { - return fmt.Errorf("must specify miner address with --miner") - } - if cctx.String("dest") == "" { - return fmt.Errorf("must specify dest directory with --dest") - } - - maddr, err := address.NewFromString(cctx.String("miner")) - if err != nil { - return err - } - - destdir, err := homedir.Expand(cctx.String("dest")) - if err != nil { - return err - } - - if err := os.MkdirAll(destdir, 0755); err != nil { - return err - } - - agmds, err := badger.NewDatastore(filepath.Join(destdir, "badger"), nil) - if err != nil { - return err - } - defer agmds.Close() - - ssize := abi.SectorSize(cctx.Uint64("sector-size")) - - ppt, spt, err := lapi.ProofTypeFromSectorSize(abi.SectorSize(cctx.Uint64("sector-size"))) - if err != nil { - return err - } - - agsb, err := sectorbuilder.New(§orbuilder.Config{ - Miner: maddr, - SealProofType: spt, - PoStProofType: ppt, - Paths: sectorbuilder.SimplePath(destdir), - WorkerThreads: 2, - }, namespace.Wrap(agmds, datastore.NewKey("/sectorbuilder"))) - if err != nil { - return err - } - - var aggrGenMiner genesis.Miner - var highestSectorID abi.SectorNumber - for _, dir := range cctx.Args().Slice() { - dir, err := homedir.Expand(dir) - if err != nil { - return xerrors.Errorf("failed to expand %q: %w", dir, err) - } - - st, err := os.Stat(dir) - if err != nil { - return err - } - if !st.IsDir() { - return fmt.Errorf("%q was not a directory", dir) - } - - fi, err := os.Open(filepath.Join(dir, "pre-seal-"+maddr.String()+".json")) - if err != nil { - return err - } - - var genmm map[string]genesis.Miner - if err := json.NewDecoder(fi).Decode(&genmm); err != nil { - return err - } - - genm, ok := genmm[maddr.String()] - if !ok { - return xerrors.Errorf("input data did not have our miner in it (%s)", maddr) - } - - if genm.SectorSize != ssize { - return xerrors.Errorf("sector size mismatch in %q (%d != %d)", dir) - } - - for _, s := range genm.Sectors { - if s.SectorID > highestSectorID { - highestSectorID = s.SectorID - } - } - - aggrGenMiner = mergeGenMiners(aggrGenMiner, genm) - - opts := badger.DefaultOptions - opts.ReadOnly = true - mds, err := badger.NewDatastore(filepath.Join(dir, "badger"), &opts) - if err != nil { - return err - } - defer mds.Close() - - sb, err := sectorbuilder.New(§orbuilder.Config{ - Miner: maddr, - SealProofType: spt, - PoStProofType: ppt, - Paths: sectorbuilder.SimplePath(dir), - WorkerThreads: 2, - }, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder"))) - if err != nil { - return err - } - - if err := agsb.ImportFrom(sb, false); err != nil { - return xerrors.Errorf("importing sectors from %q failed: %w", dir, err) - } - } - - if err := agsb.SetLastSectorNum(highestSectorID); err != nil { - return err - } - - if err := seed.WriteGenesisMiner(maddr, destdir, &aggrGenMiner, nil); err != nil { - return err - } - - return nil - }, -} - func mergeGenMiners(a, b genesis.Miner) genesis.Miner { if a.SectorSize != b.SectorSize { panic("sector sizes mismatch") diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index 8af65e3bb..b35d1cbc2 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -11,25 +11,23 @@ import ( "os" "path/filepath" + "github.com/google/uuid" + badger "github.com/ipfs/go-ds-badger2" + logging "github.com/ipfs/go-log/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/go-sectorbuilder/fs" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/abi/big" "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/specs-actors/actors/crypto" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - badger "github.com/ipfs/go-ds-badger2" - logging "github.com/ipfs/go-log/v2" - "github.com/multiformats/go-multihash" - "golang.org/x/xerrors" - - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" - - "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/node/config" ) var log = logging.Logger("preseal") @@ -46,12 +44,9 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum } cfg := §orbuilder.Config{ - Miner: maddr, - SealProofType: spt, - PoStProofType: ppt, - FallbackLastNum: offset, - Paths: sectorbuilder.SimplePath(sbroot), - WorkerThreads: 2, + Miner: maddr, + SealProofType: spt, + PoStProofType: ppt, } if err := os.MkdirAll(sbroot, 0775); err != nil { @@ -63,7 +58,13 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum return nil, nil, err } - sb, err := sectorbuilder.New(cfg, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder"))) + sbfs := &fs.Basic{ + Miner: maddr, + NextID: offset, + Root: sbroot, + } + + sb, err := sectorbuilder.New(sbfs, cfg) if err != nil { return nil, nil, err } @@ -75,7 +76,7 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum var sealedSectors []*genesis.PreSeal for i := 0; i < sectors; i++ { - sid, err := sb.AcquireSectorNumber() + sid, err := sbfs.AcquireSectorNumber() if err != nil { return nil, nil, err } @@ -90,12 +91,17 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum fmt.Printf("sector-id: %d, piece info: %v\n", sid, pi) - scid, ucid, err := sb.SealPreCommit(context.TODO(), sid, ticket, []abi.PieceInfo{pi}) + in2, err := sb.SealPreCommit1(context.TODO(), sid, ticket, []abi.PieceInfo{pi}) if err != nil { return nil, nil, xerrors.Errorf("commit: %w", err) } - if err := sb.TrimCache(context.TODO(), sid); err != nil { + scid, ucid, err := sb.SealPreCommit2(context.TODO(), sid, in2) + if err != nil { + return nil, nil, xerrors.Errorf("commit: %w", err) + } + + if err := sb.FinalizeSector(context.TODO(), sid); err != nil { return nil, nil, xerrors.Errorf("trim cache: %w", err) } @@ -138,6 +144,22 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum return nil, nil, xerrors.Errorf("closing datastore: %w", err) } + { + b, err := json.Marshal(&config.StorageMeta{ + ID: uuid.New().String(), + Weight: 0, // read-only + CanCommit: false, + CanStore: false, + }) + if err != nil { + return nil, nil, xerrors.Errorf("marshaling storage config: %w", err) + } + + if err := ioutil.WriteFile(filepath.Join(sbroot, "storage.json"), b, 0644); err != nil { + return nil, nil, xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(sbroot, "storage.json"), err) + } + } + return miner, &minerAddr.KeyInfo, nil } @@ -172,19 +194,6 @@ func WriteGenesisMiner(maddr address.Address, sbroot string, gm *genesis.Miner, return nil } -func commDCID(commd []byte) cid.Cid { - d, err := cid.Prefix{ - Version: 1, - Codec: cid.Raw, - MhType: multihash.IDENTITY, - MhLength: len(commd), - }.Sum(commd) - if err != nil { - panic(err) - } - return d -} - func createDeals(m *genesis.Miner, k *wallet.Key, maddr address.Address, ssize abi.SectorSize) error { for _, sector := range m.Sectors { proposal := &market.DealProposal{ diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 711234529..d3f13ad2c 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -72,7 +72,8 @@ var infoCmd = &cli.Command{ float64(10000*uint64(len(faults))/secCounts.Pset)/100.) } - // TODO: indicate whether the post worker is in use + panic("todo") + /*// TODO: indicate whether the post worker is in use wstat, err := nodeApi.WorkerStats(ctx) if err != nil { return err @@ -86,7 +87,7 @@ var infoCmd = &cli.Command{ fmt.Printf("\tAddPiece: %d\n", wstat.AddPieceWait) fmt.Printf("\tPreCommit: %d\n", wstat.PreCommitWait) fmt.Printf("\tCommit: %d\n", wstat.CommitWait) - fmt.Printf("\tUnseal: %d\n", wstat.UnsealWait) + fmt.Printf("\tUnseal: %d\n", wstat.UnsealWait)*/ ps, err := api.StateMinerPostState(ctx, maddr, types.EmptyTSK) if err != nil { diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index c1e24717b..bfd26572b 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -7,7 +7,6 @@ import ( "fmt" "io/ioutil" "os" - "path/filepath" "strconv" "github.com/filecoin-project/specs-actors/actors/builtin" @@ -19,11 +18,8 @@ import ( "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" paramfetch "github.com/filecoin-project/go-paramfetch" - "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - badger "github.com/ipfs/go-ds-badger2" "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" "github.com/mitchellh/go-homedir" @@ -38,11 +34,11 @@ import ( "github.com/filecoin-project/lotus/genesis" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/config" - "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sealing" + "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" ) var initCmd = &cli.Command{ @@ -77,7 +73,7 @@ var initCmd = &cli.Command{ Usage: "specify sector size to use", Value: uint64(build.SectorSizes[0]), }, - &cli.StringFlag{ + &cli.StringSliceFlag{ Name: "pre-sealed-sectors", Usage: "specify set of presealed sectors for starting as a genesis miner", }, @@ -159,60 +155,33 @@ var initCmd = &cli.Command{ return err } - if pssb := cctx.String("pre-sealed-sectors"); pssb != "" { - pssb, err := homedir.Expand(pssb) - if err != nil { - return err - } + if pssb := cctx.StringSlice("pre-sealed-sectors"); len(pssb) != 0 { + log.Infof("Setting up storage config with presealed sector", pssb) - log.Infof("moving pre-sealed-sectors from %s into newly created storage miner repo", pssb) lr, err := r.Lock(repo.StorageMiner) if err != nil { return err } - mds, err := lr.Datastore("/metadata") + sc, err := lr.GetStorage() if err != nil { - return err + return xerrors.Errorf("get storage config: %w", err) } - bopts := badger.DefaultOptions - bopts.ReadOnly = true - oldmds, err := badger.NewDatastore(filepath.Join(pssb, "badger"), &bopts) - if err != nil { - return err + for _, psp := range pssb { + psp, err := homedir.Expand(psp) + if err != nil { + return err + } + sc.StoragePaths = append(sc.StoragePaths, config.LocalPath{ + Path: psp, + }) } - ppt, spt, err := lapi.ProofTypeFromSectorSize(ssize) - if err != nil { - return err + if err := lr.SetStorage(sc); err != nil { + return xerrors.Errorf("set storage config: %w", err) } - oldsb, err := sectorbuilder.New(§orbuilder.Config{ - SealProofType: spt, - PoStProofType: ppt, - WorkerThreads: 2, - Paths: sectorbuilder.SimplePath(pssb), - }, namespace.Wrap(oldmds, datastore.NewKey("/sectorbuilder"))) - if err != nil { - return xerrors.Errorf("failed to open up preseal sectorbuilder: %w", err) - } - - nsb, err := sectorbuilder.New(§orbuilder.Config{ - SealProofType: spt, - PoStProofType: ppt, - WorkerThreads: 2, - Paths: sectorbuilder.SimplePath(lr.Path()), - }, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder"))) - if err != nil { - return xerrors.Errorf("failed to open up sectorbuilder: %w", err) - } - - if err := nsb.ImportFrom(oldsb, symlink); err != nil { - return err - } - if err := lr.Close(); err != nil { - return xerrors.Errorf("unlocking repo after preseal migration: %w", err) - } + panic("persist last sector id somehow") } if err := storageMinerInit(ctx, cctx, api, r); err != nil { @@ -374,30 +343,8 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, return err } - c, err := lr.Config() - if err != nil { - return err - } - - cfg, ok := c.(*config.StorageMiner) - if !ok { - return xerrors.Errorf("invalid config from repo, got: %T", c) - } - - scfg := sectorbuilder.SimplePath(lr.Path()) - if len(cfg.SectorBuilder.Storage) > 0 { - scfg = cfg.SectorBuilder.Storage - } - - sbcfg, err := modules.SectorBuilderConfig(scfg, 2, false, false)(mds, api) - if err != nil { - return xerrors.Errorf("getting genesis miner sector builder config: %w", err) - } - sb, err := sectorbuilder.New(sbcfg, mds) - if err != nil { - return xerrors.Errorf("failed to set up sectorbuilder for genesis mining: %w", err) - } - epp := storage.NewElectionPoStProver(sb) + smgr := advmgr.New(lr) + epp := storage.NewElectionPoStProver(smgr) m := miner.NewMiner(api, epp) { diff --git a/documentation/en/local-dev-net.md b/documentation/en/local-dev-net.md index 2bf2b35c3..b992d9bff 100644 --- a/documentation/en/local-dev-net.md +++ b/documentation/en/local-dev-net.md @@ -6,15 +6,15 @@ Build the Lotus Binaries in debug mode, This enables the use of 1024 byte sector make debug ``` -Download the 1024 byte parameters: +Download the 2048 byte parameters: ```sh -./lotus fetch-params --proving-params 1024 +./lotus fetch-params --proving-params 2048 ``` Pre-seal some sectors: ```sh -./lotus-seed pre-seal --sector-size 1024 --num-sectors 2 +./lotus-seed pre-seal --sector-size 2048 --num-sectors 2 ``` Create the genesis block and start up the first node: @@ -34,7 +34,7 @@ Then, in another console, import the genesis miner key: Set up the genesis miner: ```sh -./lotus-storage-miner init --genesis-miner --actor=t01000 --sector-size=1024 --pre-sealed-sectors=~/.genesis-sectors --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t0101.json --nosync +./lotus-storage-miner init --genesis-miner --actor=t01000 --sector-size=2048 --pre-sealed-sectors=~/.genesis-sectors --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t0101.json --nosync ``` Now, finally, start up the miner: diff --git a/markets/retrievaladapter/provider.go b/markets/retrievaladapter/provider.go index ec7505b66..a586cf01d 100644 --- a/markets/retrievaladapter/provider.go +++ b/markets/retrievaladapter/provider.go @@ -12,18 +12,19 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/lotus/storage/sealmgr" ) type retrievalProviderNode struct { - miner *storage.Miner - sb sectorbuilder.Interface - full api.FullNode + miner *storage.Miner + sealer sealmgr.Manager + full api.FullNode } // NewRetrievalProviderNode returns a new node adapter for a retrieval provider that talks to the // Lotus Node -func NewRetrievalProviderNode(miner *storage.Miner, sb sectorbuilder.Interface, full api.FullNode) retrievalmarket.RetrievalProviderNode { - return &retrievalProviderNode{miner, sb, full} +func NewRetrievalProviderNode(miner *storage.Miner, sealer sealmgr.Manager, full api.FullNode) retrievalmarket.RetrievalProviderNode { + return &retrievalProviderNode{miner, sealer, full} } func (rpn *retrievalProviderNode) UnsealSector(ctx context.Context, sectorID uint64, offset uint64, length uint64) (io.ReadCloser, error) { @@ -31,7 +32,7 @@ func (rpn *retrievalProviderNode) UnsealSector(ctx context.Context, sectorID uin if err != nil { return nil, err } - return rpn.sb.ReadPieceFromSealedSector(ctx, abi.SectorNumber(sectorID), sectorbuilder.UnpaddedByteIndex(offset), abi.UnpaddedPieceSize(length), si.Ticket.Value, *si.CommD) + return rpn.sealer.ReadPieceFromSealedSector(ctx, abi.SectorNumber(sectorID), sectorbuilder.UnpaddedByteIndex(offset), abi.UnpaddedPieceSize(length), si.Ticket.Value, *si.CommD) } func (rpn *retrievalProviderNode) SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *paych.SignedVoucher, proof []byte, expectedAmount abi.TokenAmount) (abi.TokenAmount, error) { diff --git a/node/builder.go b/node/builder.go index 6d153ba10..4a17ef7bf 100644 --- a/node/builder.go +++ b/node/builder.go @@ -253,7 +253,6 @@ func Online() Option { // Storage miner ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner }, - Override(new(sectorbuilder.Interface), modules.SectorBuilder), Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), Override(new(sealing.TicketFn), modules.SealTicketGen), Override(new(*storage.Miner), modules.StorageMiner), @@ -346,30 +345,13 @@ func ConfigFullNode(c interface{}) Option { ) } -func ConfigStorageMiner(c interface{}, lr repo.LockedRepo) Option { +func ConfigStorageMiner(c interface{}) Option { cfg, ok := c.(*config.StorageMiner) if !ok { return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) } - scfg := sectorbuilder.SimplePath(lr.Path()) - if cfg.SectorBuilder.Path == "" { - if len(cfg.SectorBuilder.Storage) > 0 { - scfg = cfg.SectorBuilder.Storage - } - } else { - scfg = sectorbuilder.SimplePath(cfg.SectorBuilder.Path) - log.Warn("LEGACY SectorBuilder.Path FOUND IN CONFIG. Please use the new storage config") - } - - return Options( - ConfigCommon(&cfg.Common), - - Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig(scfg, - cfg.SectorBuilder.WorkerCount, - cfg.SectorBuilder.DisableLocalPreCommit, - cfg.SectorBuilder.DisableLocalCommit)), - ) + return Options(ConfigCommon(&cfg.Common)) } func Repo(r repo.Repo) Option { @@ -387,7 +369,7 @@ func Repo(r repo.Repo) Option { Override(new(repo.LockedRepo), modules.LockedRepo(lr)), // module handles closing ApplyIf(isType(repo.FullNode), ConfigFullNode(c)), - ApplyIf(isType(repo.StorageMiner), ConfigStorageMiner(c, lr)), + ApplyIf(isType(repo.StorageMiner), ConfigStorageMiner(c)), Override(new(dtypes.MetadataDS), modules.Datastore), Override(new(dtypes.ChainBlockstore), modules.ChainBlockstore), diff --git a/node/config/def.go b/node/config/def.go index 43007b684..dafef3f4d 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -3,8 +3,6 @@ package config import ( "encoding" "time" - - "github.com/filecoin-project/go-sectorbuilder/fs" ) // Common is common config between full node and miner @@ -25,7 +23,7 @@ type FullNode struct { type StorageMiner struct { Common - SectorBuilder SectorBuilder + Storage Storage } // API contains configs for API endpoint @@ -54,14 +52,8 @@ type Metrics struct { } // // Storage Miner +type Storage struct { -type SectorBuilder struct { - Path string // TODO: remove // FORK (-ish) - Storage []fs.PathConfig - WorkerCount uint - - DisableLocalPreCommit bool - DisableLocalCommit bool } func defCommon() Common { @@ -95,9 +87,7 @@ func DefaultStorageMiner() *StorageMiner { cfg := &StorageMiner{ Common: defCommon(), - SectorBuilder: SectorBuilder{ - WorkerCount: 5, - }, + Storage: Storage{}, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" return cfg diff --git a/node/config/storage.go b/node/config/storage.go new file mode 100644 index 000000000..57bf80bff --- /dev/null +++ b/node/config/storage.go @@ -0,0 +1,68 @@ +package config + +import ( + "encoding/json" + "io" + "io/ioutil" + "os" + + "golang.org/x/xerrors" +) + +type LocalPath struct { + Path string +} + +// .lotusstorage/storage.json +type StorageConfig struct { + StoragePaths []LocalPath +} + +// [path]/metadata.json +type StorageMeta struct { + ID string + Weight int // 0 = readonly + + CanCommit bool + CanStore bool +} + +func StorageFromFile(path string, def *StorageConfig) (*StorageConfig, error) { + file, err := os.Open(path) + switch { + case os.IsNotExist(err): + if def == nil { + return nil, xerrors.Errorf("couldn't load storage config: %w", err) + } + return def, nil + case err != nil: + return nil, err + } + + defer file.Close() //nolint:errcheck // The file is RO + return StorageFromReader(file, *def) +} + +func StorageFromReader(reader io.Reader, def StorageConfig) (*StorageConfig, error) { + cfg := def + err := json.NewDecoder(reader).Decode(&cfg) + if err != nil { + return nil, err + } + + return &cfg, nil +} + +func WriteStorageFile(path string, config StorageConfig) error { + b, err := json.Marshal(config) + if err != nil { + return xerrors.Errorf("marshaling storage config: %w", err) + } + + if err := ioutil.WriteFile(path, b, 0644); err != nil { + return xerrors.Errorf("persisting storage config (%s): %w", path, err) + } + + return nil +} + diff --git a/node/impl/storminer.go b/node/impl/storminer.go index e115c089e..e16510d34 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -3,25 +3,18 @@ package impl import ( "context" "encoding/json" - "io" - "mime" "net/http" - "os" "strconv" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/gorilla/mux" - files "github.com/ipfs/go-ipfs-files" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/go-sectorbuilder/fs" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/gorilla/mux" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apistruct" - "github.com/filecoin-project/lotus/lib/tarutil" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" @@ -31,7 +24,7 @@ type StorageMinerAPI struct { CommonAPI SectorBuilderConfig *sectorbuilder.Config - SectorBuilder sectorbuilder.Interface + //SectorBuilder sectorbuilder.Interface SectorBlocks *sectorblocks.SectorBlocks Miner *storage.Miner @@ -57,7 +50,8 @@ func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { } func (sm *StorageMinerAPI) remoteGetSector(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) + panic("todo") +/* vars := mux.Vars(r) id, err := strconv.ParseUint(vars["id"], 10, 64) if err != nil { @@ -98,11 +92,12 @@ func (sm *StorageMinerAPI) remoteGetSector(w http.ResponseWriter, r *http.Reques if _, err := io.Copy(w, rd); err != nil { log.Error(err) return - } + }*/ } func (sm *StorageMinerAPI) remotePutSector(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) + panic("todo") +/* vars := mux.Vars(r) id, err := strconv.ParseUint(vars["id"], 10, 64) if err != nil { @@ -159,13 +154,13 @@ func (sm *StorageMinerAPI) remotePutSector(w http.ResponseWriter, r *http.Reques w.WriteHeader(200) - log.Infof("received %s sector (%s): %d bytes", vars["type"], vars["sname"], r.ContentLength) + log.Infof("received %s sector (%s): %d bytes", vars["type"], vars["sname"], r.ContentLength)*/ } - +/* func (sm *StorageMinerAPI) WorkerStats(context.Context) (sectorbuilder.WorkerStats, error) { stat := sm.SectorBuilder.WorkerStats() return stat, nil -} +}*/ func (sm *StorageMinerAPI) ActorAddress(context.Context) (address.Address, error) { return sm.SectorBuilderConfig.Miner, nil @@ -252,7 +247,7 @@ func (sm *StorageMinerAPI) SectorsRefs(context.Context) (map[string][]api.Sealed func (sm *StorageMinerAPI) SectorsUpdate(ctx context.Context, id abi.SectorNumber, state api.SectorState) error { return sm.Miner.ForceSectorState(ctx, id, state) } - +/* func (sm *StorageMinerAPI) WorkerQueue(ctx context.Context, cfg sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) { return sm.SectorBuilder.AddWorker(ctx, cfg) } @@ -260,5 +255,5 @@ func (sm *StorageMinerAPI) WorkerQueue(ctx context.Context, cfg sectorbuilder.Wo func (sm *StorageMinerAPI) WorkerDone(ctx context.Context, task uint64, res sectorbuilder.SealRes) error { return sm.SectorBuilder.TaskDone(ctx, task, res) } - +*/ var _ api.StorageMiner = &StorageMinerAPI{} diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 81b98f00b..896d954c7 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -2,11 +2,8 @@ package modules import ( "context" - "math" "reflect" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/go-address" dtgraphsync "github.com/filecoin-project/go-data-transfer/impl/graphsync" piecefilestore "github.com/filecoin-project/go-fil-markets/filestore" @@ -20,7 +17,6 @@ import ( smnet "github.com/filecoin-project/go-fil-markets/storagemarket/network" paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/go-sectorbuilder/fs" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/crypto" @@ -37,10 +33,12 @@ import ( "github.com/ipfs/go-merkledag" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/routing" - "github.com/mitchellh/go-homedir" "go.uber.org/fx" "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/storage/sealmgr" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/gen" @@ -75,51 +73,32 @@ func GetParams(sbc *sectorbuilder.Config) error { return nil } -func SectorBuilderConfig(storage []fs.PathConfig, threads uint, noprecommit, nocommit bool) func(dtypes.MetadataDS, api.FullNode) (*sectorbuilder.Config, error) { - return func(ds dtypes.MetadataDS, fnapi api.FullNode) (*sectorbuilder.Config, error) { - minerAddr, err := minerAddrFromDS(ds) - if err != nil { - return nil, err - } - - ssize, err := fnapi.StateMinerSectorSize(context.TODO(), minerAddr, types.EmptyTSK) - if err != nil { - return nil, err - } - - for i := range storage { - storage[i].Path, err = homedir.Expand(storage[i].Path) - if err != nil { - return nil, err - } - } - - if threads > math.MaxUint8 { - return nil, xerrors.Errorf("too many sectorbuilder threads specified: %d, max allowed: %d", threads, math.MaxUint8) - } - - ppt, spt, err := api.ProofTypeFromSectorSize(ssize) - if err != nil { - return nil, xerrors.Errorf("bad sector size: %w", err) - } - - sb := §orbuilder.Config{ - Miner: minerAddr, - SealProofType: spt, - PoStProofType: ppt, - - WorkerThreads: uint8(threads), - NoPreCommit: noprecommit, - NoCommit: nocommit, - - Paths: storage, - } - - return sb, nil +func SectorBuilderConfig(ds dtypes.MetadataDS, fnapi api.FullNode) (*sectorbuilder.Config, error) { + minerAddr, err := minerAddrFromDS(ds) + if err != nil { + return nil, err } + + ssize, err := fnapi.StateMinerSectorSize(context.TODO(), minerAddr, types.EmptyTSK) + if err != nil { + return nil, err + } + + ppt, spt, err := api.ProofTypeFromSectorSize(ssize) + if err != nil { + return nil, xerrors.Errorf("bad sector size: %w", err) + } + + sb := §orbuilder.Config{ + Miner: minerAddr, + SealProofType: spt, + PoStProofType: ppt, + } + + return sb, nil } -func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sb sectorbuilder.Interface, tktFn sealing.TicketFn) (*storage.Miner, error) { +func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sealer sealmgr.Manager, tktFn sealing.TicketFn) (*storage.Miner, error) { maddr, err := minerAddrFromDS(ds) if err != nil { return nil, err @@ -132,9 +111,9 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h return nil, err } - fps := storage.NewFPoStScheduler(api, sb, maddr, worker) + fps := storage.NewFPoStScheduler(api, sealer, maddr, worker) - sm, err := storage.NewMiner(api, maddr, worker, h, ds, sb, tktFn) + sm, err := storage.NewMiner(api, maddr, worker, h, ds, sealer, tktFn) if err != nil { return nil, err } @@ -271,15 +250,6 @@ func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api api.FullNode, return m, nil } -func SectorBuilder(cfg *sectorbuilder.Config, ds dtypes.MetadataDS) (*sectorbuilder.SectorBuilder, error) { - sb, err := sectorbuilder.New(cfg, namespace.Wrap(ds, datastore.NewKey("/sectorbuilder"))) - if err != nil { - return nil, err - } - - return sb, nil -} - func SealTicketGen(fapi api.FullNode) sealing.TicketFn { return func(ctx context.Context) (*api.SealTicket, error) { ts, err := fapi.ChainHead(ctx) @@ -333,8 +303,8 @@ func StorageProvider(ctx helpers.MetricsCtx, fapi api.FullNode, h host.Host, ds } // RetrievalProvider creates a new retrieval provider attached to the provider blockstore -func RetrievalProvider(h host.Host, miner *storage.Miner, sb sectorbuilder.Interface, full api.FullNode, ds dtypes.MetadataDS, pieceStore dtypes.ProviderPieceStore, ibs dtypes.StagingBlockstore) (retrievalmarket.RetrievalProvider, error) { - adapter := retrievaladapter.NewRetrievalProviderNode(miner, sb, full) +func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sealmgr.Manager, full api.FullNode, ds dtypes.MetadataDS, pieceStore dtypes.ProviderPieceStore, ibs dtypes.StagingBlockstore) (retrievalmarket.RetrievalProvider, error) { + adapter := retrievaladapter.NewRetrievalProviderNode(miner, sealer, full) address, err := minerAddrFromDS(ds) if err != nil { return nil, err diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index ceab72f0a..881acad63 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -28,6 +28,7 @@ const ( fsAPI = "api" fsAPIToken = "token" fsConfig = "config.toml" + fsStorageConfig = "storage.json" fsDatastore = "datastore" fsLock = "repo.lock" fsKeystore = "keystore" @@ -87,7 +88,7 @@ func (fsr *FsRepo) Exists() (bool, error) { func (fsr *FsRepo) Init(t RepoType) error { exist, err := fsr.Exists() - if err != nil { + if err != nil{ return err } if exist { @@ -276,6 +277,18 @@ func (fsr *fsLockedRepo) Config() (interface{}, error) { return config.FromFile(fsr.join(fsConfig), defConfForType(fsr.repoType)) } +func (fsr *fsLockedRepo) GetStorage() (config.StorageConfig, error) { + c, err := config.StorageFromFile(fsr.join(fsStorageConfig), nil) + if err != nil { + return config.StorageConfig{}, err + } + return *c, nil +} + +func (fsr *fsLockedRepo) SetStorage(c config.StorageConfig) error { + return config.WriteStorageFile(fsr.join(fsStorageConfig), c) +} + func (fsr *fsLockedRepo) SetAPIEndpoint(ma multiaddr.Multiaddr) error { if err := fsr.stillValid(); err != nil { return err diff --git a/node/repo/interface.go b/node/repo/interface.go index b4c2b0950..58d6a7fdd 100644 --- a/node/repo/interface.go +++ b/node/repo/interface.go @@ -7,6 +7,7 @@ import ( "github.com/multiformats/go-multiaddr" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/config" ) var ( @@ -37,6 +38,9 @@ type LockedRepo interface { // Returns config in this repo Config() (interface{}, error) + GetStorage() (config.StorageConfig, error) + SetStorage(config.StorageConfig) error + // SetAPIEndpoint sets the endpoint of the current API // so it can be read by API clients SetAPIEndpoint(multiaddr.Multiaddr) error diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index 6cc54ddb9..d1b491831 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -12,6 +12,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/config" ) type MemRepo struct { @@ -38,6 +39,14 @@ type lockedMemRepo struct { token *byte } +func (lmem *lockedMemRepo) GetStorage() (config.StorageConfig, error) { + panic("implement me") +} + +func (lmem *lockedMemRepo) SetStorage(config.StorageConfig) error { + panic("implement me") +} + func (lmem *lockedMemRepo) Path() string { t, err := ioutil.TempDir(os.TempDir(), "lotus-memrepo-temp-") if err != nil { @@ -169,6 +178,10 @@ func (lmem *lockedMemRepo) Config() (interface{}, error) { return lmem.mem.configF(lmem.t), nil } +func (lmem *lockedMemRepo) Storage() (config.StorageConfig, error) { + panic("implement me") +} + func (lmem *lockedMemRepo) SetAPIEndpoint(ma multiaddr.Multiaddr) error { if err := lmem.checkToken(); err != nil { return err diff --git a/storage/fpost_run.go b/storage/fpost_run.go index 1ac99a3e7..5110d00b8 100644 --- a/storage/fpost_run.go +++ b/storage/fpost_run.go @@ -90,7 +90,9 @@ func (s *FPoStScheduler) declareFaults(ctx context.Context, fc uint64, params *m } func (s *FPoStScheduler) checkFaults(ctx context.Context, ssi []abi.SectorNumber) ([]abi.SectorNumber, error) { - faults := s.sb.Scrub(ssi) + //faults := s.sb.Scrub(ssi) + log.Warnf("Stub checkFaults") + var faults []struct{SectorNum abi.SectorNumber; Err error} declaredFaults := map[abi.SectorNumber]struct{}{} diff --git a/storage/fpost_sched.go b/storage/fpost_sched.go index 27eb7eb11..506092015 100644 --- a/storage/fpost_sched.go +++ b/storage/fpost_sched.go @@ -22,7 +22,7 @@ const StartConfidence = 4 // TODO: config type FPoStScheduler struct { api storageMinerApi - sb sectorbuilder.Interface + sb sectorbuilder.Prover actor address.Address worker address.Address @@ -37,7 +37,7 @@ type FPoStScheduler struct { failLk sync.Mutex } -func NewFPoStScheduler(api storageMinerApi, sb sectorbuilder.Interface, actor address.Address, worker address.Address) *FPoStScheduler { +func NewFPoStScheduler(api storageMinerApi, sb sectorbuilder.Prover, actor address.Address, worker address.Address) *FPoStScheduler { return &FPoStScheduler{api: api, sb: sb, actor: actor, worker: worker} } diff --git a/storage/miner.go b/storage/miner.go index 41088908d..32ef250bb 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -13,6 +13,8 @@ import ( "golang.org/x/xerrors" ffi "github.com/filecoin-project/filecoin-ffi" + "github.com/filecoin-project/lotus/storage/sealmgr" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/builtin/miner" @@ -30,11 +32,11 @@ import ( var log = logging.Logger("storageminer") type Miner struct { - api storageMinerApi - h host.Host - sb sectorbuilder.Interface - ds datastore.Batching - tktFn sealing.TicketFn + api storageMinerApi + h host.Host + sealer sealmgr.Manager + ds datastore.Batching + tktFn sealing.TicketFn maddr address.Address worker address.Address @@ -71,13 +73,13 @@ type storageMinerApi interface { WalletHas(context.Context, address.Address) (bool, error) } -func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sb sectorbuilder.Interface, tktFn sealing.TicketFn) (*Miner, error) { +func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sealer sealmgr.Manager, tktFn sealing.TicketFn) (*Miner, error) { m := &Miner{ - api: api, - h: h, - sb: sb, - ds: ds, - tktFn: tktFn, + api: api, + h: h, + sealer: sealer, + ds: ds, + tktFn: tktFn, maddr: maddr, worker: worker, @@ -92,7 +94,7 @@ func (m *Miner) Run(ctx context.Context) error { } evts := events.NewEvents(ctx, m.api) - m.sealing = sealing.New(m.api, evts, m.maddr, m.worker, m.ds, m.sb, m.tktFn) + m.sealing = sealing.New(m.api, evts, m.maddr, m.worker, m.ds, m.sealer, m.tktFn) go m.sealing.Run(ctx) @@ -119,10 +121,10 @@ func (m *Miner) runPreflightChecks(ctx context.Context) error { } type SectorBuilderEpp struct { - sb sectorbuilder.Interface + prover sectorbuilder.Prover } -func NewElectionPoStProver(sb sectorbuilder.Interface) *SectorBuilderEpp { +func NewElectionPoStProver(sb sectorbuilder.Prover) *SectorBuilderEpp { return &SectorBuilderEpp{sb} } @@ -132,7 +134,7 @@ func (epp *SectorBuilderEpp) GenerateCandidates(ctx context.Context, ssi []abi.S start := time.Now() var faults []abi.SectorNumber // TODO - cds, err := epp.sb.GenerateEPostCandidates(ssi, rand, faults) + cds, err := epp.prover.GenerateEPostCandidates(ssi, rand, faults) if err != nil { return nil, xerrors.Errorf("failed to generate candidates: %w", err) } @@ -152,7 +154,7 @@ func (epp *SectorBuilderEpp) ComputeProof(ctx context.Context, ssi []abi.SectorI } start := time.Now() - proof, err := epp.sb.ComputeElectionPoSt(ssi, rand, owins) + proof, err := epp.prover.ComputeElectionPoSt(ssi, rand, owins) if err != nil { return nil, err } diff --git a/storage/sealing/garbage.go b/storage/sealing/garbage.go index 660a2604a..42d1bc80a 100644 --- a/storage/sealing/garbage.go +++ b/storage/sealing/garbage.go @@ -25,7 +25,7 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorNumber, e out := make([]Piece, len(sizes)) for i, size := range sizes { - ppi, err := m.sb.AddPiece(ctx, size, sectorID, m.pledgeReader(size), existingPieceSizes) + ppi, err := m.sealer.AddPiece(ctx, size, sectorID, m.pledgeReader(size), existingPieceSizes) if err != nil { return nil, xerrors.Errorf("add piece: %w", err) } @@ -47,15 +47,15 @@ func (m *Sealing) PledgeSector() error { // this, as we run everything here async, and it's cancelled when the // command exits - size := abi.PaddedPieceSize(m.sb.SectorSize()).Unpadded() + size := abi.PaddedPieceSize(m.sealer.SectorSize()).Unpadded() - rt, _, err := api.ProofTypeFromSectorSize(m.sb.SectorSize()) + rt, _, err := api.ProofTypeFromSectorSize(m.sealer.SectorSize()) if err != nil { log.Error(err) return } - sid, err := m.sb.AcquireSectorNumber() + sid, err := m.sealer.NewSector() if err != nil { log.Errorf("%+v", err) return diff --git a/storage/sealing/sealing.go b/storage/sealing/sealing.go index cd2f6ff5f..a7d11586a 100644 --- a/storage/sealing/sealing.go +++ b/storage/sealing/sealing.go @@ -12,7 +12,6 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-padreader" - "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/crypto" @@ -22,6 +21,7 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/statemachine" + "github.com/filecoin-project/lotus/storage/sealmgr" ) const SectorStorePrefix = "/sectors" @@ -65,19 +65,19 @@ type Sealing struct { maddr address.Address worker address.Address - sb sectorbuilder.Interface + sealer sealmgr.Manager sectors *statemachine.StateGroup tktFn TicketFn } -func New(api sealingApi, events *events.Events, maddr address.Address, worker address.Address, ds datastore.Batching, sb sectorbuilder.Interface, tktFn TicketFn) *Sealing { +func New(api sealingApi, events *events.Events, maddr address.Address, worker address.Address, ds datastore.Batching, sealer sealmgr.Manager, tktFn TicketFn) *Sealing { s := &Sealing{ api: api, events: events, maddr: maddr, worker: worker, - sb: sb, + sealer: sealer, tktFn: tktFn, } @@ -104,7 +104,7 @@ func (m *Sealing) AllocatePiece(size abi.UnpaddedPieceSize) (sectorID abi.Sector return 0, 0, xerrors.Errorf("cannot allocate unpadded piece") } - sid, err := m.sb.AcquireSectorNumber() // TODO: Put more than one thing in a sector + sid, err := m.sealer.NewSector() // TODO: Put more than one thing in a sector if err != nil { return 0, 0, xerrors.Errorf("acquiring sector ID: %w", err) } @@ -116,12 +116,12 @@ func (m *Sealing) AllocatePiece(size abi.UnpaddedPieceSize) (sectorID abi.Sector func (m *Sealing) SealPiece(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, sectorID abi.SectorNumber, dealID abi.DealID) error { log.Infof("Seal piece for deal %d", dealID) - ppi, err := m.sb.AddPiece(ctx, size, sectorID, r, []abi.UnpaddedPieceSize{}) + ppi, err := m.sealer.AddPiece(ctx, size, sectorID, r, []abi.UnpaddedPieceSize{}) if err != nil { return xerrors.Errorf("adding piece to sector: %w", err) } - _, rt, err := api.ProofTypeFromSectorSize(m.sb.SectorSize()) + _, rt, err := api.ProofTypeFromSectorSize(m.sealer.SectorSize()) if err != nil { return xerrors.Errorf("bad sector size: %w", err) } diff --git a/storage/sealing/states.go b/storage/sealing/states.go index b3f033a9d..d7966a373 100644 --- a/storage/sealing/states.go +++ b/storage/sealing/states.go @@ -5,7 +5,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/crypto" - "github.com/filecoin-project/go-sectorbuilder/fs" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/miner" @@ -26,7 +25,7 @@ func (m *Sealing) handlePacking(ctx statemachine.Context, sector SectorInfo) err allocated += piece.Size } - ubytes := abi.PaddedPieceSize(m.sb.SectorSize()).Unpadded() + ubytes := abi.PaddedPieceSize(m.sealer.SectorSize()).Unpadded() if allocated > ubytes { return xerrors.Errorf("too much data in sector: %d > %d", allocated, ubytes) @@ -70,7 +69,12 @@ func (m *Sealing) handleUnsealed(ctx statemachine.Context, sector SectorInfo) er return ctx.Send(SectorSealFailed{xerrors.Errorf("getting ticket failed: %w", err)}) } - sealed, unsealed, err := m.sb.SealPreCommit(ctx.Context(), sector.SectorID, ticket.Value, sector.pieceInfos()) + pc1o, err := m.sealer.SealPreCommit1(ctx.Context(), sector.SectorID, ticket.Value, sector.pieceInfos()) + if err != nil { + return ctx.Send(SectorSealFailed{xerrors.Errorf("seal pre commit failed: %w", err)}) + } + + sealed, unsealed, err := m.sealer.SealPreCommit2(ctx.Context(), sector.SectorID, pc1o) if err != nil { return ctx.Send(SectorSealFailed{xerrors.Errorf("seal pre commit failed: %w", err)}) } @@ -180,7 +184,12 @@ func (m *Sealing) handleCommitting(ctx statemachine.Context, sector SectorInfo) log.Infof("KOMIT %d %x(%d); %x(%d); %v; r:%x; d:%x", sector.SectorID, sector.Ticket.Value, sector.Ticket.Epoch, sector.Seed.Value, sector.Seed.Epoch, sector.pieceInfos(), sector.CommR, sector.CommD) - proof, err := m.sb.SealCommit(ctx.Context(), sector.SectorID, sector.Ticket.Value, sector.Seed.Value, sector.pieceInfos(), *sector.CommR, *sector.CommD) + c2in, err := m.sealer.SealCommit1(ctx.Context(), sector.SectorID, sector.Ticket.Value, sector.Seed.Value, sector.pieceInfos(), *sector.CommR, *sector.CommD) + if err != nil { + return ctx.Send(SectorComputeProofFailed{xerrors.Errorf("computing seal proof failed: %w", err)}) + } + + proof, err := m.sealer.SealCommit2(ctx.Context(), sector.SectorID, c2in) if err != nil { return ctx.Send(SectorComputeProofFailed{xerrors.Errorf("computing seal proof failed: %w", err)}) } @@ -241,15 +250,8 @@ func (m *Sealing) handleCommitWait(ctx statemachine.Context, sector SectorInfo) func (m *Sealing) handleFinalizeSector(ctx statemachine.Context, sector SectorInfo) error { // TODO: Maybe wait for some finality - if err := m.sb.FinalizeSector(ctx.Context(), sector.SectorID); err != nil { - if !xerrors.Is(err, fs.ErrNoSuitablePath) { - return ctx.Send(SectorFinalizeFailed{xerrors.Errorf("finalize sector: %w", err)}) - } - log.Warnf("finalize sector: %v", err) - } - - if err := m.sb.DropStaged(ctx.Context(), sector.SectorID); err != nil { - return ctx.Send(SectorFinalizeFailed{xerrors.Errorf("drop staged: %w", err)}) + if err := m.sealer.FinalizeSector(ctx.Context(), sector.SectorID); err != nil { + return ctx.Send(SectorFinalizeFailed{xerrors.Errorf("finalize sector: %w", err)}) } return ctx.Send(SectorFinalized{}) diff --git a/storage/sealing/types.go b/storage/sealing/types.go index b25a43ef8..4e9e0dccc 100644 --- a/storage/sealing/types.go +++ b/storage/sealing/types.go @@ -1,9 +1,10 @@ package sealing import ( - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lotus/api" ) type Piece struct { diff --git a/storage/sealmgr/advmgr/lotus_storage_manager.go b/storage/sealmgr/advmgr/lotus_storage_manager.go new file mode 100644 index 000000000..bd6407a0e --- /dev/null +++ b/storage/sealmgr/advmgr/lotus_storage_manager.go @@ -0,0 +1,102 @@ +package advmgr + +import ( + "context" + "io" + "sync" + + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/specs-actors/actors/abi" + + ffi "github.com/filecoin-project/filecoin-ffi" + "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/storage/sealmgr" +) + +type LocalStorage interface { + GetStorage() (config.StorageConfig, error) + SetStorage(config.StorageConfig) error +} + +type Path struct { + ID string + Weight uint64 + + LocalPath string + + CanSeal bool + CanStore bool +} + +type Worker interface { + sealmgr.Worker + + Paths() []Path +} + +type Manager struct { + workers []sealmgr.Worker + + localLk sync.RWMutex + localStorage LocalStorage +} + +func (m Manager) SectorSize() abi.SectorSize { + panic("implement me") +} + +func (m Manager) NewSector() (abi.SectorNumber, error) { + panic("implement me") +} + +func (m Manager) ReadPieceFromSealedSector(context.Context, abi.SectorNumber, sectorbuilder.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error) { + panic("implement me") +} + +func (m Manager) AddPiece(context.Context, abi.UnpaddedPieceSize, abi.SectorNumber, io.Reader, []abi.UnpaddedPieceSize) (abi.PieceInfo, error) { + panic("implement me") +} + +func (m Manager) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out []byte, err error) { + panic("implement me") +} + +func (m Manager) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { + panic("implement me") +} + +func (m Manager) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output []byte, err error) { + panic("implement me") +} + +func (m Manager) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (proof []byte, err error) { + panic("implement me") +} + +func (m Manager) FinalizeSector(context.Context, abi.SectorNumber) error { + panic("implement me") +} + +func (m Manager) GenerateEPostCandidates(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]ffi.PoStCandidateWithTicket, error) { + panic("implement me") +} + +func (m Manager) GenerateFallbackPoSt(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]ffi.PoStCandidateWithTicket, []abi.PoStProof, error) { + panic("implement me") +} + +func (m Manager) ComputeElectionPoSt(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, winners []abi.PoStCandidate) ([]abi.PoStProof, error) { + panic("implement me") +} + +func New(ls LocalStorage) *Manager { + return &Manager{ + workers: nil, + + localStorage: ls, + } +} + +var _ sealmgr.Manager = &Manager{} diff --git a/storage/sealmgr/simple.go b/storage/sealmgr/simple.go index 7d8f13298..4d0292e65 100644 --- a/storage/sealmgr/simple.go +++ b/storage/sealmgr/simple.go @@ -26,15 +26,13 @@ type Simple struct { worker Worker } -func NewSimpleManager(sc *storedcounter.StoredCounter, maddr address.Address, sb sectorbuilder.Basic) (*Simple, error) { - mid, err := address.IDFromAddress(maddr) - if err != nil { - return nil, xerrors.Errorf("get miner id: %w", err) - } +func (s *Simple) SectorSize() abi.SectorSize { + panic("implement me") +} +func NewSimpleManager(sc *storedcounter.StoredCounter, maddr address.Address, sb sectorbuilder.Basic) (*Simple, error) { w := &LocalWorker{ - sealer: sb, - mid: abi.ActorID(mid), + sb, } return &Simple{ @@ -44,49 +42,71 @@ func NewSimpleManager(sc *storedcounter.StoredCounter, maddr address.Address, sb }, nil } -func (s *Simple) NewSector() (SectorInfo, error) { +func (s *Simple) NewSector() (abi.SectorNumber, error) { n, err := s.sc.Next() if err != nil { - return SectorInfo{}, xerrors.Errorf("acquire sector number: %w", err) + return 0, xerrors.Errorf("acquire sector number: %w", err) } - mid, err := address.IDFromAddress(s.maddr) - if err != nil { - return SectorInfo{}, xerrors.Errorf("get miner id: %w", err) - } - - return SectorInfo{ - ID: abi.SectorID{ - Miner: abi.ActorID(mid), - Number: abi.SectorNumber(n), - }, - }, nil + return abi.SectorNumber(n), nil } -func (s *Simple) AddPiece(ctx context.Context, si SectorInfo, sz abi.UnpaddedPieceSize, r io.Reader) (cid.Cid, SectorInfo, error) { +func (s *Simple) AddPiece(ctx context.Context, sz abi.UnpaddedPieceSize, sectorNum abi.SectorNumber, r io.Reader, existingPieces []abi.UnpaddedPieceSize) (abi.PieceInfo, error) { s.rateLimiter.Lock() defer s.rateLimiter.Unlock() - return s.worker.AddPiece(ctx, si, sz, r) + return s.worker.AddPiece(ctx, sz, sectorNum, r, existingPieces) } -func (s *Simple) RunSeal(ctx context.Context, task TaskType, si SectorInfo) (SectorInfo, error) { +func (s *Simple) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out []byte, err error) { s.rateLimiter.Lock() defer s.rateLimiter.Unlock() - return s.worker.Run(ctx, task, si) + return s.worker.SealPreCommit1(ctx, sectorNum, ticket, pieces) +} + +func (s *Simple) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { + s.rateLimiter.Lock() + defer s.rateLimiter.Unlock() + + return s.worker.SealPreCommit2(ctx, sectorNum, phase1Out) +} + +func (s *Simple) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output []byte, err error) { + s.rateLimiter.Lock() + defer s.rateLimiter.Unlock() + + return s.worker.SealCommit1(ctx, sectorNum, ticket, seed, pieces, sealedCID, unsealedCID) +} + +func (s *Simple) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (proof []byte, err error) { + s.rateLimiter.Lock() + defer s.rateLimiter.Unlock() + + return s.worker.SealCommit2(ctx, sectorNum, phase1Out) +} + +func (s *Simple) FinalizeSector(ctx context.Context, sectorNum abi.SectorNumber) error { + s.rateLimiter.Lock() + defer s.rateLimiter.Unlock() + + return s.worker.FinalizeSector(ctx, sectorNum) } func (s *Simple) GenerateEPostCandidates(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]ffi.PoStCandidateWithTicket, error) { - return s.worker.(*LocalWorker).sealer.GenerateEPostCandidates(sectorInfo, challengeSeed, faults) + return s.worker.GenerateEPostCandidates(sectorInfo, challengeSeed, faults) } func (s *Simple) GenerateFallbackPoSt(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]ffi.PoStCandidateWithTicket, []abi.PoStProof, error) { - return s.worker.(*LocalWorker).sealer.GenerateFallbackPoSt(sectorInfo, challengeSeed, faults) + return s.worker.GenerateFallbackPoSt(sectorInfo, challengeSeed, faults) } func (s *Simple) ComputeElectionPoSt(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, winners []abi.PoStCandidate) ([]abi.PoStProof, error) { - return s.worker.(*LocalWorker).sealer.ComputeElectionPoSt(sectorInfo, challengeSeed, winners) + return s.worker.ComputeElectionPoSt(sectorInfo, challengeSeed, winners) +} + +func (s *Simple) ReadPieceFromSealedSector(context.Context, abi.SectorNumber, sectorbuilder.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error) { + panic("todo") } var _ Manager = &Simple{} diff --git a/storage/sealmgr/types.go b/storage/sealmgr/types.go index a81346a97..52da12c71 100644 --- a/storage/sealmgr/types.go +++ b/storage/sealmgr/types.go @@ -1,80 +1,32 @@ package sealmgr import ( - "bytes" "context" "io" - "github.com/ipfs/go-cid" - "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/ipfs/go-cid" ) - -type SealTicket struct { - Value abi.SealRandomness - Epoch abi.ChainEpoch -} - -type SealSeed struct { - Value abi.InteractiveSealRandomness - Epoch abi.ChainEpoch -} - -func (st *SealTicket) Equals(ost *SealTicket) bool { - return bytes.Equal(st.Value, ost.Value) && st.Epoch == ost.Epoch -} - -func (st *SealSeed) Equals(ost *SealSeed) bool { - return bytes.Equal(st.Value, ost.Value) && st.Epoch == ost.Epoch -} - -// SectorInfo holds all sector-related metadata -type SectorInfo struct { - ID abi.SectorID - - Pieces []abi.PieceInfo - - Ticket SealTicket - Seed SealSeed - - PreCommit1Out []byte - - Sealed *cid.Cid - Unsealed *cid.Cid - - CommitInput []byte - Proof []byte -} - -func (si SectorInfo) PieceSizes() []abi.UnpaddedPieceSize { - out := make([]abi.UnpaddedPieceSize, len(si.Pieces)) - for i := range out { - out[i] = si.Pieces[i].Size.Unpadded() - } - - return nil -} - type Worker interface { - AddPiece(context.Context, SectorInfo, abi.UnpaddedPieceSize, io.Reader) (cid.Cid, SectorInfo, error) - Run(context.Context, TaskType, SectorInfo) (SectorInfo, error) + sectorbuilder.Sealer + sectorbuilder.Prover } type Manager interface { + SectorSize() abi.SectorSize + // NewSector allocates staging area for data - NewSector() (SectorInfo, error) + // Storage manager forwards proof-related calls + NewSector() (abi.SectorNumber, error) - // AddPiece appends the piece to the specified sector. Returns PieceCID, and - // mutated sector info - // - // Note: The passed reader can support other transfer mechanisms, making - // it possible to move the data between data transfer module and workers - AddPiece(context.Context, SectorInfo, abi.UnpaddedPieceSize, io.Reader) (cid.Cid, SectorInfo, error) + // TODO: Can[Pre]Commit[1,2] + // TODO: Scrub() []Faults - RunSeal(ctx context.Context, task TaskType, si SectorInfo) (SectorInfo, error) + // TODO: Separate iface + ReadPieceFromSealedSector(context.Context, abi.SectorNumber, sectorbuilder.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error) - // Storage manager forwards proving calls + sectorbuilder.Sealer sectorbuilder.Prover } diff --git a/storage/sealmgr/worker_local.go b/storage/sealmgr/worker_local.go index 05435967b..67636766a 100644 --- a/storage/sealmgr/worker_local.go +++ b/storage/sealmgr/worker_local.go @@ -1,74 +1,11 @@ package sealmgr import ( - "context" - "io" - - "github.com/ipfs/go-cid" - "golang.org/x/xerrors" - "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/specs-actors/actors/abi" ) type LocalWorker struct { - sealer sectorbuilder.Basic - mid abi.ActorID -} - -func (w *LocalWorker) Run(ctx context.Context, task TaskType, si SectorInfo) (SectorInfo, error) { - if si.ID.Miner != w.mid { - return si, xerrors.Errorf("received a task with wrong actor id; worker for %d, task for %d", w.mid, si.ID.Miner) - } - - switch task { - case TTPreCommit1: - pco, err := w.sealer.SealPreCommit1(ctx, si.ID.Number, si.Ticket.Value, si.Pieces) - if err != nil { - return si, xerrors.Errorf("calling sealer: %w", err) - } - si.PreCommit1Out = pco - case TTPreCommit2: - sealed, unsealed, err := w.sealer.SealPreCommit2(ctx, si.ID.Number, si.PreCommit1Out) - if err != nil { - return si, xerrors.Errorf("calling sealer (precommit2): %w", err) - } - - si.Sealed = &sealed - si.Unsealed = &unsealed - - // We also call Commit1 here as it only grabs some inputs for the snark, - // which is very fast (<1s), and it doesn't really make sense to have a separate - // task type for it - - c2in, err := w.sealer.SealCommit1(ctx, si.ID.Number, si.Ticket.Value, si.Seed.Value, si.Pieces, *si.Sealed, *si.Unsealed) - if err != nil { - return si, xerrors.Errorf("calling sealer (commit1): %w", err) - } - - si.CommitInput = c2in - case TTCommit2: - proof, err := w.sealer.SealCommit2(ctx, si.ID.Number, si.CommitInput) - if err != nil { - return SectorInfo{}, xerrors.Errorf("calling sealer: %w", err) - } - - si.Proof = proof - default: - return si, xerrors.Errorf("unknown task type '%s'", task) - } - - return si, nil -} - -func (w *LocalWorker) AddPiece(ctx context.Context, si SectorInfo, sz abi.UnpaddedPieceSize, r io.Reader) (cid.Cid, SectorInfo, error) { - pi, err := w.sealer.AddPiece(ctx, sz, si.ID.Number, r, si.PieceSizes()) - if err != nil { - return cid.Cid{}, SectorInfo{}, xerrors.Errorf("addPiece on local worker: %w", err) - } - - si.Pieces = append(si.Pieces, pi) - return pi.PieceCID, si, nil + sectorbuilder.Basic } var _ Worker = &LocalWorker{} diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index f485e2860..b8561fffa 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -9,7 +9,6 @@ import ( "sync" "github.com/filecoin-project/go-padreader" - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" @@ -51,16 +50,14 @@ func DsKeyToDealID(key datastore.Key) (uint64, error) { type SectorBlocks struct { *storage.Miner - sb sectorbuilder.Interface keys datastore.Batching keyLk sync.Mutex } -func NewSectorBlocks(miner *storage.Miner, ds dtypes.MetadataDS, sb sectorbuilder.Interface) *SectorBlocks { +func NewSectorBlocks(miner *storage.Miner, ds dtypes.MetadataDS) *SectorBlocks { sbc := &SectorBlocks{ Miner: miner, - sb: sb, keys: namespace.Wrap(ds, dsPrefix), } From 12d870e27423ac64ed70fb4f87bfc6c9b99fd403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 4 Mar 2020 03:24:08 +0100 Subject: [PATCH 10/24] sealmgr: Read only multi-path file manager --- cmd/lotus-seed/seed/seed.go | 14 +- cmd/lotus-storage-miner/init.go | 38 ++++- documentation/en/local-dev-net.md | 2 +- go.mod | 5 +- go.sum | 4 + node/config/storage.go | 8 +- .../{lotus_storage_manager.go => manager.go} | 54 +++--- storage/sealmgr/advmgr/roprov.go | 28 ++++ storage/sealmgr/advmgr/storage.go | 157 ++++++++++++++++++ 9 files changed, 258 insertions(+), 52 deletions(-) rename storage/sealmgr/advmgr/{lotus_storage_manager.go => manager.go} (73%) create mode 100644 storage/sealmgr/advmgr/roprov.go create mode 100644 storage/sealmgr/advmgr/storage.go diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index b35d1cbc2..be8dac380 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -12,7 +12,6 @@ import ( "path/filepath" "github.com/google/uuid" - badger "github.com/ipfs/go-ds-badger2" logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" @@ -53,11 +52,6 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum return nil, nil, err } - mds, err := badger.NewDatastore(filepath.Join(sbroot, "badger"), nil) - if err != nil { - return nil, nil, err - } - sbfs := &fs.Basic{ Miner: maddr, NextID: offset, @@ -140,17 +134,13 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum return nil, nil, xerrors.Errorf("creating deals: %w", err) } - if err := mds.Close(); err != nil { - return nil, nil, xerrors.Errorf("closing datastore: %w", err) - } - { - b, err := json.Marshal(&config.StorageMeta{ + b, err := json.MarshalIndent(&config.StorageMeta{ ID: uuid.New().String(), Weight: 0, // read-only CanCommit: false, CanStore: false, - }) + }, "", " ") if err != nil { return nil, nil, xerrors.Errorf("marshaling storage config: %w", err) } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index bfd26572b..2708b14de 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -3,12 +3,14 @@ package main import ( "context" "crypto/rand" + "encoding/binary" "encoding/json" "fmt" "io/ioutil" "os" "strconv" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/market" miner2 "github.com/filecoin-project/specs-actors/actors/builtin/miner" @@ -162,10 +164,7 @@ var initCmd = &cli.Command{ if err != nil { return err } - sc, err := lr.GetStorage() - if err != nil { - return xerrors.Errorf("get storage config: %w", err) - } + var sc config.StorageConfig for _, psp := range pssb { psp, err := homedir.Expand(psp) @@ -181,10 +180,12 @@ var initCmd = &cli.Command{ return xerrors.Errorf("set storage config: %w", err) } - panic("persist last sector id somehow") + if err := lr.Close(); err != nil { + return err + } } - if err := storageMinerInit(ctx, cctx, api, r); err != nil { + if err := storageMinerInit(ctx, cctx, api, r, ssize); err != nil { log.Errorf("Failed to initialize lotus-storage-miner: %+v", err) path, err := homedir.Expand(repoPath) if err != nil { @@ -220,6 +221,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, metadata string, return xerrors.Errorf("unmarshaling preseal metadata: %w", err) } + maxSectorID := abi.SectorNumber(0) for _, sector := range meta.Sectors { sectorKey := datastore.NewKey(sealing.SectorStorePrefix).ChildString(fmt.Sprint(sector.SectorID)) @@ -258,6 +260,10 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, metadata string, return err } + if sector.SectorID > maxSectorID { + maxSectorID = sector.SectorID + } + /* // TODO: Import deals into market pnd, err := cborutil.AsIpld(sector.Deal) if err != nil { @@ -285,7 +291,9 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, metadata string, }*/ } - return nil + buf := make([]byte, binary.MaxVarintLen64) + size := binary.PutUvarint(buf, uint64(maxSectorID+1)) + return mds.Put(datastore.NewKey("/storage/nextid"), buf[:size]) } func findMarketDealID(ctx context.Context, api lapi.FullNode, deal market.DealProposal) (abi.DealID, error) { @@ -307,7 +315,7 @@ func findMarketDealID(ctx context.Context, api lapi.FullNode, deal market.DealPr return 0, xerrors.New("deal not found") } -func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, r repo.Repo) error { +func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, r repo.Repo, ssize abi.SectorSize) error { lr, err := r.Lock(repo.StorageMiner) if err != nil { return err @@ -343,7 +351,19 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, return err } - smgr := advmgr.New(lr) + ppt, spt, err := lapi.ProofTypeFromSectorSize(ssize) + if err != nil { + return err + } + + smgr, err := advmgr.New(lr, §orbuilder.Config{ + SealProofType: spt, + PoStProofType: ppt, + Miner: a, + }) + if err != nil { + return err + } epp := storage.NewElectionPoStProver(smgr) m := miner.NewMiner(api, epp) diff --git a/documentation/en/local-dev-net.md b/documentation/en/local-dev-net.md index b992d9bff..b1c4130cc 100644 --- a/documentation/en/local-dev-net.md +++ b/documentation/en/local-dev-net.md @@ -34,7 +34,7 @@ Then, in another console, import the genesis miner key: Set up the genesis miner: ```sh -./lotus-storage-miner init --genesis-miner --actor=t01000 --sector-size=2048 --pre-sealed-sectors=~/.genesis-sectors --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t0101.json --nosync +./lotus-storage-miner init --genesis-miner --actor=t01000 --sector-size=2048 --pre-sealed-sectors=~/.genesis-sectors --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t01000.json --nosync ``` Now, finally, start up the miner: diff --git a/go.mod b/go.mod index 8a767e7cd..4b6e8b86b 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/coreos/go-systemd/v22 v22.0.0 github.com/docker/go-units v0.4.0 github.com/filecoin-project/chain-validation v0.0.3 - github.com/filecoin-project/filecoin-ffi v0.0.0-20200226205820-4da0bccccefb + github.com/filecoin-project/filecoin-ffi v0.0.0-20200226231125-fc253ccb5294 github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 @@ -23,7 +23,8 @@ require ( github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200229022239-442fe78a3168 github.com/filecoin-project/go-statestore v0.1.0 - github.com/filecoin-project/specs-actors v0.0.0-20200229011003-1d726e3afd04 + github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf + github.com/filecoin-project/specs-storage v0.0.0-20200303230804-16c9a38030eb github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-ole/go-ole v1.2.4 // indirect github.com/google/uuid v1.1.1 diff --git a/go.sum b/go.sum index 54768a163..07a700d64 100644 --- a/go.sum +++ b/go.sum @@ -130,6 +130,10 @@ github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.m github.com/filecoin-project/specs-actors v0.0.0-20200226200336-94c9b92b2775/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= github.com/filecoin-project/specs-actors v0.0.0-20200229011003-1d726e3afd04 h1:O343OeQLkLWLj5ZqQ5nhevAGBTeB5LioiA53ddScqdY= github.com/filecoin-project/specs-actors v0.0.0-20200229011003-1d726e3afd04/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= +github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf h1:3IojVqJAD5IXMxvZ+WYx+LRbfSB/rOXpYBuHh6o3XkY= +github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= +github.com/filecoin-project/specs-storage v0.0.0-20200303230804-16c9a38030eb h1:oHB9hKaD7g75NFulnfh+SCYS5bSl8hB6Eanf8A6l5tw= +github.com/filecoin-project/specs-storage v0.0.0-20200303230804-16c9a38030eb/go.mod h1:sC2Ck2l1G8hXI5Do/3sp0yxbMRMnukbFwP9KF1CRFLw= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0= diff --git a/node/config/storage.go b/node/config/storage.go index 57bf80bff..faf0c65ff 100644 --- a/node/config/storage.go +++ b/node/config/storage.go @@ -40,11 +40,11 @@ func StorageFromFile(path string, def *StorageConfig) (*StorageConfig, error) { } defer file.Close() //nolint:errcheck // The file is RO - return StorageFromReader(file, *def) + return StorageFromReader(file) } -func StorageFromReader(reader io.Reader, def StorageConfig) (*StorageConfig, error) { - cfg := def +func StorageFromReader(reader io.Reader) (*StorageConfig, error) { + var cfg StorageConfig err := json.NewDecoder(reader).Decode(&cfg) if err != nil { return nil, err @@ -54,7 +54,7 @@ func StorageFromReader(reader io.Reader, def StorageConfig) (*StorageConfig, err } func WriteStorageFile(path string, config StorageConfig) error { - b, err := json.Marshal(config) + b, err := json.MarshalIndent(config, "", " ") if err != nil { return xerrors.Errorf("marshaling storage config: %w", err) } diff --git a/storage/sealmgr/advmgr/lotus_storage_manager.go b/storage/sealmgr/advmgr/manager.go similarity index 73% rename from storage/sealmgr/advmgr/lotus_storage_manager.go rename to storage/sealmgr/advmgr/manager.go index bd6407a0e..061ea77e7 100644 --- a/storage/sealmgr/advmgr/lotus_storage_manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -3,14 +3,14 @@ package advmgr import ( "context" "io" - "sync" + "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" + "golang.org/x/xerrors" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" - ffi "github.com/filecoin-project/filecoin-ffi" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/storage/sealmgr" ) @@ -39,8 +39,34 @@ type Worker interface { type Manager struct { workers []sealmgr.Worker - localLk sync.RWMutex - localStorage LocalStorage + sectorbuilder.Prover +} + +func New(ls LocalStorage, cfg *sectorbuilder.Config) (*Manager, error) { + stor := &storage{ + localStorage: ls, + } + if err := stor.open(); err != nil { + return nil, err + } + + mid, err := address.IDFromAddress(cfg.Miner) + if err != nil { + return nil, xerrors.Errorf("getting miner id: %w", err) + } + + prover, err := sectorbuilder.New(&readonlyProvider{stor: stor, miner: abi.ActorID(mid)}, cfg) + if err != nil { + return nil, xerrors.Errorf("creating prover instance: %w", err) + } + + m := &Manager{ + workers: nil, + + Prover: prover, + } + + return m, nil } func (m Manager) SectorSize() abi.SectorSize { @@ -79,24 +105,4 @@ func (m Manager) FinalizeSector(context.Context, abi.SectorNumber) error { panic("implement me") } -func (m Manager) GenerateEPostCandidates(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]ffi.PoStCandidateWithTicket, error) { - panic("implement me") -} - -func (m Manager) GenerateFallbackPoSt(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]ffi.PoStCandidateWithTicket, []abi.PoStProof, error) { - panic("implement me") -} - -func (m Manager) ComputeElectionPoSt(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, winners []abi.PoStCandidate) ([]abi.PoStProof, error) { - panic("implement me") -} - -func New(ls LocalStorage) *Manager { - return &Manager{ - workers: nil, - - localStorage: ls, - } -} - var _ sealmgr.Manager = &Manager{} diff --git a/storage/sealmgr/advmgr/roprov.go b/storage/sealmgr/advmgr/roprov.go new file mode 100644 index 000000000..cb3e6854c --- /dev/null +++ b/storage/sealmgr/advmgr/roprov.go @@ -0,0 +1,28 @@ +package advmgr + +import ( + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/specs-actors/actors/abi" + "golang.org/x/xerrors" +) + +type readonlyProvider struct { + miner abi.ActorID + stor *storage +} + +func (l *readonlyProvider) AcquireSectorNumber() (abi.SectorNumber, error) { + return 0, xerrors.New("read-only provider") +} + +func (l *readonlyProvider) FinalizeSector(abi.SectorNumber) error { + return xerrors.New("read-only provider") +} + +func (l *readonlyProvider) AcquireSector(id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { + if allocate != 0 { + return sectorbuilder.SectorPaths{}, nil, xerrors.New("read-only storage") + } + + return l.stor.acquireSector(l.miner, id, existing, allocate, sealing) +} diff --git a/storage/sealmgr/advmgr/storage.go b/storage/sealmgr/advmgr/storage.go new file mode 100644 index 000000000..3214a022a --- /dev/null +++ b/storage/sealmgr/advmgr/storage.go @@ -0,0 +1,157 @@ +package advmgr + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "path/filepath" + "sync" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/lotus/node/config" +) + +const metaFile = "storage.json" +var pathTypes = []sectorbuilder.SectorFileType{sectorbuilder.FTUnsealed, sectorbuilder.FTSealed, sectorbuilder.FTCache} + +type storage struct { + localLk sync.RWMutex + localStorage LocalStorage + + paths []path +} + +type path struct { + meta config.StorageMeta + local string + + sectors map[abi.SectorID]sectorbuilder.SectorFileType +} + +func openPath(p string, meta config.StorageMeta) (path, error) { + out := path{ + meta: meta, + local: p, + sectors: map[abi.SectorID]sectorbuilder.SectorFileType{}, + } + + for _, t := range pathTypes { + ents, err := ioutil.ReadDir(filepath.Join(p, t.String())) + if err != nil { + return path{}, xerrors.Errorf("listing %s: %w", filepath.Join(p, t.String()), err) + } + + for _, ent := range ents { + sid, err := parseSectorID(ent.Name()) + if err != nil { + return path{}, xerrors.Errorf("parse sector id %s: %w", ent.Name(), err) + } + + out.sectors[sid] |= t + } + } + + return out, nil +} + +func (st *storage) open() error { + st.localLk.Lock() + defer st.localLk.Unlock() + + cfg, err := st.localStorage.GetStorage() + if err != nil { + return xerrors.Errorf("getting local storage config: %w", err) + } + + if len(cfg.StoragePaths) == 0 { + return xerrors.New("no local storage paths configured") + } + + for _, path := range cfg.StoragePaths { + mb, err := ioutil.ReadFile(filepath.Join(path.Path, metaFile)) + if err != nil { + return xerrors.Errorf("reading storage metadata for %s: %w", path.Path, err) + } + + var meta config.StorageMeta + if err := json.Unmarshal(mb, &meta); err != nil { + return xerrors.Errorf("unmarshalling storage metadata for %s: %w", path.Path, err) + } + + pi, err := openPath(path.Path, meta) + if err != nil { + return xerrors.Errorf("opening path %s: %w", path.Path, err) + } + + st.paths = append(st.paths, pi) + } + + return nil +} + +func (st *storage) acquireSector(mid abi.ActorID, id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { + st.localLk.RLock() + + if allocate != 0 { + st.localLk.RUnlock() + return sectorbuilder.SectorPaths{}, nil, xerrors.New("acquire alloc todo") + } + + var out sectorbuilder.SectorPaths + + for _, fileType := range pathTypes { + if fileType & existing == 0 { + continue + } + + for _, p := range st.paths { + s, ok := p.sectors[abi.SectorID{ + Miner: mid, + Number: id, + }] + if !ok { + continue + } + if s & fileType == 0 { + continue + } + + spath := filepath.Join(p.local, fileType.String(), fmt.Sprintf("s-t0%d-%d", mid, id)) + + switch fileType { + case sectorbuilder.FTUnsealed: + out.Unsealed = spath + case sectorbuilder.FTSealed: + out.Sealed = spath + case sectorbuilder.FTCache: + out.Cache = spath + } + + existing ^= fileType + } + } + + return out, st.localLk.RUnlock, nil +} + +func parseSectorID(baseName string) (abi.SectorID, error) { + var n abi.SectorNumber + var mid abi.ActorID + read, err := fmt.Sscanf(baseName, "s-t0%d-%d", &mid, &n) + if err != nil { + return abi.SectorID{}, xerrors.Errorf(": %w", err) + } + + if read != 2 { + return abi.SectorID{}, xerrors.Errorf("parseSectorID expected to scan 2 values, got %d", read) + } + + return abi.SectorID{ + Miner: mid, + Number: n, + }, nil +} \ No newline at end of file From f50e26e5dd65c1ba92ea1dc294a32f5e63839ee1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 4 Mar 2020 06:32:13 +0100 Subject: [PATCH 11/24] update sectorbuilder --- go.mod | 2 +- go.sum | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 411c96696..b9bbd22ef 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/filecoin-project/go-fil-markets v0.0.0-20200303015849-1159079679ca github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 - github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200229022239-442fe78a3168 + github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200304050010-2cfac00a93e7 github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf github.com/filecoin-project/specs-storage v0.0.0-20200303230804-16c9a38030eb diff --git a/go.sum b/go.sum index 973880722..2c102b49d 100644 --- a/go.sum +++ b/go.sum @@ -124,6 +124,8 @@ github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200226210935-4739f8749f5 github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200226210935-4739f8749f56/go.mod h1:tzTc9BxxSbjlIzhFwm5h9oBkXKkRuLxeiWspntwnKyw= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200229022239-442fe78a3168 h1:6fBZezecouqHNRi4Q4YlH7QqgJ3yZZ7adMBpUBAx4F8= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200229022239-442fe78a3168/go.mod h1:gEQJVRVqQX8Vx02IosTdC2UA4l2MgHfG3POJEI2GIpc= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200304050010-2cfac00a93e7 h1:pGvvPZgIfBXAda+OGc943zFcM8Wt4GG9UVymVeyfx6c= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200304050010-2cfac00a93e7/go.mod h1:gEQJVRVqQX8Vx02IosTdC2UA4l2MgHfG3POJEI2GIpc= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 h1:k9qVR9ItcziSB2rxtlkN/MDWNlbsI6yzec+zjUatLW0= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= @@ -131,11 +133,11 @@ github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZO github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200226200336-94c9b92b2775/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= github.com/filecoin-project/specs-actors v0.0.0-20200226200336-94c9b92b2775/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= -github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf h1:3IojVqJAD5IXMxvZ+WYx+LRbfSB/rOXpYBuHh6o3XkY= -github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= github.com/filecoin-project/specs-actors v0.0.0-20200229011003-1d726e3afd04 h1:O343OeQLkLWLj5ZqQ5nhevAGBTeB5LioiA53ddScqdY= github.com/filecoin-project/specs-actors v0.0.0-20200229011003-1d726e3afd04/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf h1:3IojVqJAD5IXMxvZ+WYx+LRbfSB/rOXpYBuHh6o3XkY= +github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf h1:3IojVqJAD5IXMxvZ+WYx+LRbfSB/rOXpYBuHh6o3XkY= +github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= github.com/filecoin-project/specs-storage v0.0.0-20200303230804-16c9a38030eb h1:oHB9hKaD7g75NFulnfh+SCYS5bSl8hB6Eanf8A6l5tw= github.com/filecoin-project/specs-storage v0.0.0-20200303230804-16c9a38030eb/go.mod h1:sC2Ck2l1G8hXI5Do/3sp0yxbMRMnukbFwP9KF1CRFLw= From 1d3877fab3e9d7cc28dd8d154e70627cefa2f964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 4 Mar 2020 06:32:33 +0100 Subject: [PATCH 12/24] sync: Use correct epost randomness --- chain/sync.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/chain/sync.go b/chain/sync.go index 91f74accc..c52f6f4a3 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -2,6 +2,7 @@ package chain import ( "context" + "crypto/sha256" "errors" "fmt" "sync" @@ -722,8 +723,9 @@ func (syncer *Syncer) VerifyElectionPoStProof(ctx context.Context, h *types.Bloc // TODO: why do we need this here? challengeCount := sectorbuilder.ElectionPostChallengeCount(uint64(len(sectorInfo)), 0) + hvrf := sha256.Sum256(h.EPostProof.PostRand) pvi := abi.PoStVerifyInfo{ - Randomness: h.EPostProof.PostRand, + Randomness: hvrf[:], Candidates: candidates, Proofs: h.EPostProof.Proofs, EligibleSectors: sectorInfo, @@ -737,6 +739,7 @@ func (syncer *Syncer) VerifyElectionPoStProof(ctx context.Context, h *types.Bloc } if !ok { + log.Errorf("invalid election post (%x; %v)", pvi.Randomness, candidates) return xerrors.Errorf("election post was invalid") } From e483383a4499e2e1031477dbc3e7b31c4f912b54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 4 Mar 2020 20:42:49 +0100 Subject: [PATCH 13/24] node: wire up new storage manager --- node/builder.go | 9 +++++++++ node/options.go | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/node/builder.go b/node/builder.go index 4a17ef7bf..06508eacf 100644 --- a/node/builder.go +++ b/node/builder.go @@ -56,6 +56,8 @@ import ( "github.com/filecoin-project/lotus/paychmgr" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sealing" + "github.com/filecoin-project/lotus/storage/sealmgr" + "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -253,6 +255,13 @@ func Online() Option { // Storage miner ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner }, + Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig), + Override(new(advmgr.LocalStorage), From(new(repo.LockedRepo))), + Override(new(*advmgr.Manager), advmgr.New), + + Override(new(sealmgr.Manager), From(new(*advmgr.Manager))), + Override(new(sectorbuilder.Prover), From(new(sealmgr.Manager))), + Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), Override(new(sealing.TicketFn), modules.SealTicketGen), Override(new(*storage.Miner), modules.StorageMiner), diff --git a/node/options.go b/node/options.go index 75a57cb9b..a99dc5bbc 100644 --- a/node/options.go +++ b/node/options.go @@ -84,6 +84,15 @@ func Unset(typ interface{}) Option { } } +// From(*T) -> func(t T) T {return t} +func From(typ interface{}) interface{} { + rt := []reflect.Type{reflect.TypeOf(typ).Elem()} + ft := reflect.FuncOf(rt, rt, false) + return reflect.MakeFunc(ft, func(args []reflect.Value) (results []reflect.Value) { + return args + }).Interface() +} + // from go-ipfs // as casts input constructor to a given interface (if a value is given, it // wraps it into a constructor). From 76698fe2db987ff922907e81efa45fd92649c277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Mar 2020 01:34:29 +0100 Subject: [PATCH 14/24] jsonrpc: Don't try to reconnect on the server side --- lib/jsonrpc/websocket.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/jsonrpc/websocket.go b/lib/jsonrpc/websocket.go index e28a0d96d..066056204 100644 --- a/lib/jsonrpc/websocket.go +++ b/lib/jsonrpc/websocket.go @@ -480,12 +480,16 @@ func (c *wsConn) handleWsConn(ctx context.Context) { c.closeChans() c.incoming = make(chan io.Reader) // listen again for responses go func() { + if c.connFactory == nil { // likely the server side, don't try to reconnect + return + } + var conn *websocket.Conn for conn == nil { time.Sleep(c.reconnectInterval) var err error if conn, err = c.connFactory(); err != nil { - log.Debugw("websocket connection retried failed", "error", err) + log.Debugw("websocket connection retry failed", "error", err) } } From 7db1dd52bd0b549f8ae7fd88a538028965e0dec6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Mar 2020 01:38:07 +0100 Subject: [PATCH 15/24] storagemgr: Wire up most of AddPiece logic --- cmd/lotus-seed/seed/seed.go | 8 +-- cmd/lotus-storage-miner/init.go | 2 +- node/builder.go | 1 + node/config/storage.go | 4 +- node/modules/storageminer.go | 16 +++++ storage/sealmgr/advmgr/manager.go | 111 ++++++++++++++++++++++++------ storage/sealmgr/advmgr/storage.go | 102 +++++++++++++++++++++++++-- storage/sealmgr/task.go | 1 + 8 files changed, 213 insertions(+), 32 deletions(-) diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index be8dac380..3a55eeac3 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -136,10 +136,10 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum { b, err := json.MarshalIndent(&config.StorageMeta{ - ID: uuid.New().String(), - Weight: 0, // read-only - CanCommit: false, - CanStore: false, + ID: uuid.New().String(), + Weight: 0, // read-only + CanSeal: false, + CanStore: false, }, "", " ") if err != nil { return nil, nil, xerrors.Errorf("marshaling storage config: %w", err) diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 303a24adb..bb88e5427 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -363,7 +363,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, SealProofType: spt, PoStProofType: ppt, Miner: a, - }) + }, nil) if err != nil { return err } diff --git a/node/builder.go b/node/builder.go index 06508eacf..6785abf12 100644 --- a/node/builder.go +++ b/node/builder.go @@ -257,6 +257,7 @@ func Online() Option { ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner }, Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig), Override(new(advmgr.LocalStorage), From(new(repo.LockedRepo))), + Override(new(advmgr.SectorIDCounter), modules.SectorIDCounter), Override(new(*advmgr.Manager), advmgr.New), Override(new(sealmgr.Manager), From(new(*advmgr.Manager))), diff --git a/node/config/storage.go b/node/config/storage.go index faf0c65ff..3761a0b96 100644 --- a/node/config/storage.go +++ b/node/config/storage.go @@ -21,9 +21,9 @@ type StorageConfig struct { // [path]/metadata.json type StorageMeta struct { ID string - Weight int // 0 = readonly + Weight uint64 // 0 = readonly - CanCommit bool + CanSeal bool CanStore bool } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 896d954c7..55a7af93c 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -15,6 +15,7 @@ import ( deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl" smnet "github.com/filecoin-project/go-fil-markets/storagemarket/network" + "github.com/filecoin-project/go-fil-markets/storedcounter" paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-statestore" @@ -38,6 +39,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/storage/sealmgr" + "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -98,6 +100,20 @@ func SectorBuilderConfig(ds dtypes.MetadataDS, fnapi api.FullNode) (*sectorbuild return sb, nil } +type sidsc struct { + sc *storedcounter.StoredCounter +} + +func (s *sidsc) Next() (abi.SectorNumber, error) { + i, err := s.sc.Next() + return abi.SectorNumber(i), err +} + +func SectorIDCounter(ds dtypes.MetadataDS) advmgr.SectorIDCounter { + sc := storedcounter.New(ds, datastore.NewKey("/storage/nextid")) + return &sidsc{sc} +} + func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sealer sealmgr.Manager, tktFn sealing.TicketFn) (*storage.Miner, error) { maddr, err := minerAddrFromDS(ds) if err != nil { diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index 061ea77e7..1207b786f 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -4,10 +4,10 @@ import ( "context" "io" - "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" @@ -15,6 +15,10 @@ import ( "github.com/filecoin-project/lotus/storage/sealmgr" ) +type SectorIDCounter interface { + Next() (abi.SectorNumber, error) +} + type LocalStorage interface { GetStorage() (config.StorageConfig, error) SetStorage(config.StorageConfig) error @@ -31,18 +35,23 @@ type Path struct { } type Worker interface { - sealmgr.Worker + sectorbuilder.Sealer + TaskTypes() map[sealmgr.TaskType]struct{} Paths() []Path } type Manager struct { - workers []sealmgr.Worker + workers []Worker + scfg *sectorbuilder.Config + sc SectorIDCounter + + storage *storage sectorbuilder.Prover } -func New(ls LocalStorage, cfg *sectorbuilder.Config) (*Manager, error) { +func New(ls LocalStorage, cfg *sectorbuilder.Config, sc SectorIDCounter) (*Manager, error) { stor := &storage{ localStorage: ls, } @@ -62,6 +71,10 @@ func New(ls LocalStorage, cfg *sectorbuilder.Config) (*Manager, error) { m := &Manager{ workers: nil, + scfg: cfg, + sc: sc, + + storage: stor, Prover: prover, } @@ -69,40 +82,98 @@ func New(ls LocalStorage, cfg *sectorbuilder.Config) (*Manager, error) { return m, nil } -func (m Manager) SectorSize() abi.SectorSize { +func (m *Manager) SectorSize() abi.SectorSize { + sz, _ := m.scfg.SealProofType.SectorSize() + return sz +} + +func (m *Manager) NewSector() (abi.SectorNumber, error) { + return m.sc.Next() +} + +func (m *Manager) ReadPieceFromSealedSector(context.Context, abi.SectorNumber, sectorbuilder.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error) { panic("implement me") } -func (m Manager) NewSector() (abi.SectorNumber, error) { +func (m *Manager) AddPiece(ctx context.Context, sz abi.UnpaddedPieceSize, sn abi.SectorNumber, r io.Reader, existingPieces []abi.UnpaddedPieceSize) (abi.PieceInfo, error) { + // TODO: consider multiple paths vs workers when initially allocating + + var best []config.StorageMeta + var err error + if len(existingPieces) == 0 { // new + best, err = m.storage.findBestAllocStorage(sectorbuilder.FTUnsealed, true) + } else { // append to existing + best, err = m.storage.findSector(m.minerID(), sn, sectorbuilder.FTUnsealed) + } + if err != nil { + return abi.PieceInfo{}, xerrors.Errorf("finding sector path: %w", err) + } + + var candidateWorkers []Worker + bestPaths := map[int]config.StorageMeta{} + + for i, worker := range m.workers { + if _, ok := worker.TaskTypes()[sealmgr.TTAddPiece]; !ok { + continue + } + + // check if the worker has access to the path we selected + var st *config.StorageMeta + for _, p := range worker.Paths() { + for _, m := range best { + if p.ID == m.ID { + if st != nil && st.Weight > p.Weight { + continue + } + + p := m // copy + st = &p + } + } + } + if st == nil { + continue + } + + bestPaths[i] = *st + candidateWorkers = append(candidateWorkers, worker) + } + + if len(candidateWorkers) == 0 { + return abi.PieceInfo{}, xerrors.New("no worker selected") + } + + // TODO: schedule(addpiece, ..., ) + // TODO: remove sectorbuilder abstraction, pass path directly + return candidateWorkers[0].AddPiece(ctx, sz, sn, r, existingPieces) +} + +func (m *Manager) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out []byte, err error) { panic("implement me") } -func (m Manager) ReadPieceFromSealedSector(context.Context, abi.SectorNumber, sectorbuilder.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error) { +func (m *Manager) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { panic("implement me") } -func (m Manager) AddPiece(context.Context, abi.UnpaddedPieceSize, abi.SectorNumber, io.Reader, []abi.UnpaddedPieceSize) (abi.PieceInfo, error) { +func (m *Manager) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output []byte, err error) { panic("implement me") } -func (m Manager) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out []byte, err error) { +func (m *Manager) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (proof []byte, err error) { panic("implement me") } -func (m Manager) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { +func (m *Manager) FinalizeSector(context.Context, abi.SectorNumber) error { panic("implement me") } -func (m Manager) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output []byte, err error) { - panic("implement me") -} - -func (m Manager) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (proof []byte, err error) { - panic("implement me") -} - -func (m Manager) FinalizeSector(context.Context, abi.SectorNumber) error { - panic("implement me") +func (m *Manager) minerID() abi.ActorID { + mid, err := address.IDFromAddress(m.scfg.Miner) + if err != nil { + panic(err) + } + return abi.ActorID(mid) } var _ sealmgr.Manager = &Manager{} diff --git a/storage/sealmgr/advmgr/storage.go b/storage/sealmgr/advmgr/storage.go index 3214a022a..c593c7a20 100644 --- a/storage/sealmgr/advmgr/storage.go +++ b/storage/sealmgr/advmgr/storage.go @@ -94,13 +94,12 @@ func (st *storage) open() error { } func (st *storage) acquireSector(mid abi.ActorID, id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { - st.localLk.RLock() - - if allocate != 0 { - st.localLk.RUnlock() - return sectorbuilder.SectorPaths{}, nil, xerrors.New("acquire alloc todo") + if existing | allocate != existing ^ allocate { + return sectorbuilder.SectorPaths{}, nil, xerrors.New("can't both find and allocate a sector") } + st.localLk.RLock() + var out sectorbuilder.SectorPaths for _, fileType := range pathTypes { @@ -135,9 +134,102 @@ func (st *storage) acquireSector(mid abi.ActorID, id abi.SectorNumber, existing } } + for _, fileType := range pathTypes { + if fileType & allocate == 0 { + continue + } + + var best string + + for _, p := range st.paths { + if sealing && !p.meta.CanSeal { + continue + } + if !sealing && !p.meta.CanStore { + continue + } + + s, ok := p.sectors[abi.SectorID{ + Miner: mid, + Number: id, + }] + if !ok { + continue + } + if s & fileType == 0 { + continue + } + + // TODO: Check free space + // TODO: Calc weights + + best = filepath.Join(p.local, fileType.String(), fmt.Sprintf("s-t0%d-%d", mid, id)) + break // todo: the first path won't always be the best + } + + if best == "" { + st.localLk.RUnlock() + return sectorbuilder.SectorPaths{}, nil, xerrors.Errorf("couldn't find a suitable path for a sector") + } + + switch fileType { + case sectorbuilder.FTUnsealed: + out.Unsealed = best + case sectorbuilder.FTSealed: + out.Sealed = best + case sectorbuilder.FTCache: + out.Cache = best + } + + allocate ^= fileType + } + return out, st.localLk.RUnlock, nil } +func (st *storage) findBestAllocStorage(allocate sectorbuilder.SectorFileType, sealing bool) ([]config.StorageMeta, error) { + var out []config.StorageMeta + + for _, p := range st.paths { + if sealing && !p.meta.CanSeal { + continue + } + if !sealing && !p.meta.CanStore { + continue + } + + // TODO: filter out of space + + out = append(out, p.meta) + } + + if len(out) == 0 { + return nil, xerrors.New("no good path found") + } + + // todo: sort by some kind of preference + return out, nil +} + +func (st *storage) findSector(mid abi.ActorID, sn abi.SectorNumber, typ sectorbuilder.SectorFileType) ([]config.StorageMeta, error) { + var out []config.StorageMeta + for _, p := range st.paths { + t := p.sectors[abi.SectorID{ + Miner: mid, + Number: sn, + }] + if t | typ == 0 { + continue + } + out = append(out, p.meta) + } + if len(out) == 0 { + return nil, xerrors.Errorf("sector %s/s-t0%d-%d not found", typ, mid, sn) + } + + return out, nil +} + func parseSectorID(baseName string) (abi.SectorID, error) { var n abi.SectorNumber var mid abi.ActorID diff --git a/storage/sealmgr/task.go b/storage/sealmgr/task.go index cbe7b5344..10b804724 100644 --- a/storage/sealmgr/task.go +++ b/storage/sealmgr/task.go @@ -3,6 +3,7 @@ package sealmgr type TaskType string const ( + TTAddPiece TaskType = "seal/v0/addpiece" TTPreCommit1 TaskType = "seal/v0/precommit/1" TTPreCommit2 TaskType = "seal/v0/precommit/2" // Commit1 is called here too TTCommit2 TaskType = "seal/v0/commit/2" From c91d970c41346595df4415ebdf2eb9a18918bd05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Mar 2020 02:12:52 +0100 Subject: [PATCH 16/24] storageminer: Init with default local storage --- cmd/lotus-seed/seed/seed.go | 2 +- cmd/lotus-storage-miner/init.go | 50 ++++++++++++++++++++++++------- storage/sealmgr/advmgr/manager.go | 2 +- storage/sealmgr/advmgr/storage.go | 10 ++++++- 4 files changed, 51 insertions(+), 13 deletions(-) diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index 3a55eeac3..74804d938 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -145,7 +145,7 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum return nil, nil, xerrors.Errorf("marshaling storage config: %w", err) } - if err := ioutil.WriteFile(filepath.Join(sbroot, "storage.json"), b, 0644); err != nil { + if err := ioutil.WriteFile(filepath.Join(sbroot, "sectorstore.json"), b, 0644); err != nil { return nil, nil, xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(sbroot, "storage.json"), err) } } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index bb88e5427..c2781f714 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -8,6 +8,7 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" "strconv" "github.com/filecoin-project/go-sectorbuilder" @@ -16,6 +17,7 @@ import ( miner2 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/builtin/power" crypto2 "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/google/uuid" "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" @@ -91,6 +93,10 @@ var initCmd = &cli.Command{ Name: "symlink-imported-sectors", Usage: "attempt to symlink to presealed sectors instead of copying them into place", }, + &cli.BoolFlag{ + Name: "no-local-storage", + Usage: "don't use storageminer repo for sector storage", + }, }, Action: func(cctx *cli.Context) error { log.Info("Initializing lotus storage miner") @@ -157,25 +163,48 @@ var initCmd = &cli.Command{ return err } - if pssb := cctx.StringSlice("pre-sealed-sectors"); len(pssb) != 0 { - log.Infof("Setting up storage config with presealed sector", pssb) - + { lr, err := r.Lock(repo.StorageMiner) if err != nil { return err } + var sc config.StorageConfig - for _, psp := range pssb { - psp, err := homedir.Expand(psp) - if err != nil { - return err + if pssb := cctx.StringSlice("pre-sealed-sectors"); len(pssb) != 0 { + log.Infof("Setting up storage config with presealed sector", pssb) + + for _, psp := range pssb { + psp, err := homedir.Expand(psp) + if err != nil { + return err + } + sc.StoragePaths = append(sc.StoragePaths, config.LocalPath{ + Path: psp, + }) } - sc.StoragePaths = append(sc.StoragePaths, config.LocalPath{ - Path: psp, - }) } + if !cctx.Bool("no-local-storage") { + b, err := json.MarshalIndent(&config.StorageMeta{ + ID: uuid.New().String(), + Weight: 10, + CanSeal: true, + CanStore: true, + }, "", " ") + if err != nil { + return xerrors.Errorf("marshaling storage config: %w", err) + } + + if err := ioutil.WriteFile(filepath.Join(lr.Path(), "sectorstore.json"), b, 0644); err != nil { + return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "storage.json"), err) + } + } + + sc.StoragePaths = append(sc.StoragePaths, config.LocalPath{ + Path: lr.Path(), + }) + if err := lr.SetStorage(sc); err != nil { return xerrors.Errorf("set storage config: %w", err) } @@ -185,6 +214,7 @@ var initCmd = &cli.Command{ } } + if err := storageMinerInit(ctx, cctx, api, r, ssize); err != nil { log.Errorf("Failed to initialize lotus-storage-miner: %+v", err) path, err := homedir.Expand(repoPath) diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index 1207b786f..712c65a8c 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -140,7 +140,7 @@ func (m *Manager) AddPiece(ctx context.Context, sz abi.UnpaddedPieceSize, sn abi } if len(candidateWorkers) == 0 { - return abi.PieceInfo{}, xerrors.New("no worker selected") + return abi.PieceInfo{}, xerrors.New("no worker found") } // TODO: schedule(addpiece, ..., ) diff --git a/storage/sealmgr/advmgr/storage.go b/storage/sealmgr/advmgr/storage.go index c593c7a20..e392c1b3b 100644 --- a/storage/sealmgr/advmgr/storage.go +++ b/storage/sealmgr/advmgr/storage.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "os" "path/filepath" "sync" @@ -15,7 +16,7 @@ import ( "github.com/filecoin-project/lotus/node/config" ) -const metaFile = "storage.json" +const metaFile = "sectorstore.json" var pathTypes = []sectorbuilder.SectorFileType{sectorbuilder.FTUnsealed, sectorbuilder.FTSealed, sectorbuilder.FTCache} type storage struct { @@ -42,6 +43,13 @@ func openPath(p string, meta config.StorageMeta) (path, error) { for _, t := range pathTypes { ents, err := ioutil.ReadDir(filepath.Join(p, t.String())) if err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(filepath.Join(p, t.String()), 0755); err != nil { + return path{}, xerrors.Errorf("mkdir '%s': %w", filepath.Join(p, t.String()), err) + } + + continue + } return path{}, xerrors.Errorf("listing %s: %w", filepath.Join(p, t.String()), err) } From 4beed065b61d3d356730d2f167ae9a05c4f9cc9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Mar 2020 03:18:22 +0100 Subject: [PATCH 17/24] sealmgr: Working local addpiece --- storage/sealmgr/advmgr/local.go | 89 +++++++++++++++++++++++++++++++ storage/sealmgr/advmgr/manager.go | 4 +- storage/sealmgr/advmgr/storage.go | 30 +++++++---- storage/sealmgr/simple.go | 7 +++ storage/sealmgr/worker_local.go | 11 ---- 5 files changed, 118 insertions(+), 23 deletions(-) create mode 100644 storage/sealmgr/advmgr/local.go delete mode 100644 storage/sealmgr/worker_local.go diff --git a/storage/sealmgr/advmgr/local.go b/storage/sealmgr/advmgr/local.go new file mode 100644 index 000000000..9f54d664a --- /dev/null +++ b/storage/sealmgr/advmgr/local.go @@ -0,0 +1,89 @@ +package advmgr + +import ( + "context" + "io" + + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/lotus/storage/sealmgr" +) + +type localWorker struct { + scfg *sectorbuilder.Config + storage *storage +} + +type localWorkerPathProvider struct { + w *localWorker +} + +func (l *localWorkerPathProvider) AcquireSectorNumber() (abi.SectorNumber, error) { + return 0, xerrors.Errorf("unsupported") +} + +func (l *localWorkerPathProvider) FinalizeSector(abi.SectorNumber) error { + return xerrors.Errorf("unsupported") +} + +func (l *localWorkerPathProvider) AcquireSector(id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { + mid, err := address.IDFromAddress(l.w.scfg.Miner) + if err != nil { + return sectorbuilder.SectorPaths{}, nil, xerrors.Errorf("get miner ID: %w", err) + } + + return l.w.storage.acquireSector(abi.ActorID(mid), id, existing, allocate, sealing) +} + +func (l *localWorker) sb() (sectorbuilder.Basic, error) { + return sectorbuilder.New(&localWorkerPathProvider{w:l}, l.scfg) +} + +func (l *localWorker) AddPiece(ctx context.Context, sz abi.UnpaddedPieceSize, sn abi.SectorNumber, r io.Reader, epcs []abi.UnpaddedPieceSize) (abi.PieceInfo, error) { + sb, err := l.sb() + if err != nil { + return abi.PieceInfo{}, err + } + + return sb.AddPiece(ctx, sz, sn, r, epcs) +} + +func (l *localWorker) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out []byte, err error) { + panic("implement me") +} + +func (l *localWorker) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { + panic("implement me") +} + +func (l *localWorker) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output []byte, err error) { + panic("implement me") +} + +func (l *localWorker) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (proof []byte, err error) { + panic("implement me") +} + +func (l *localWorker) FinalizeSector(context.Context, abi.SectorNumber) error { + panic("implement me") +} + +func (l *localWorker) TaskTypes() map[sealmgr.TaskType]struct{} { + return map[sealmgr.TaskType]struct{}{ + sealmgr.TTAddPiece: {}, + sealmgr.TTPreCommit1: {}, + sealmgr.TTPreCommit2: {}, + sealmgr.TTCommit2: {}, + } +} + +func (l *localWorker) Paths() []Path { + return l.storage.local() +} + +var _ Worker = &localWorker{} \ No newline at end of file diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index 712c65a8c..a086c8940 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -70,7 +70,9 @@ func New(ls LocalStorage, cfg *sectorbuilder.Config, sc SectorIDCounter) (*Manag } m := &Manager{ - workers: nil, + workers: []Worker{ + &localWorker{scfg: cfg, storage: stor}, + }, scfg: cfg, sc: sc, diff --git a/storage/sealmgr/advmgr/storage.go b/storage/sealmgr/advmgr/storage.go index e392c1b3b..eef4489c1 100644 --- a/storage/sealmgr/advmgr/storage.go +++ b/storage/sealmgr/advmgr/storage.go @@ -157,17 +157,6 @@ func (st *storage) acquireSector(mid abi.ActorID, id abi.SectorNumber, existing continue } - s, ok := p.sectors[abi.SectorID{ - Miner: mid, - Number: id, - }] - if !ok { - continue - } - if s & fileType == 0 { - continue - } - // TODO: Check free space // TODO: Calc weights @@ -238,6 +227,25 @@ func (st *storage) findSector(mid abi.ActorID, sn abi.SectorNumber, typ sectorbu return out, nil } +func (st *storage) local() []Path { + var out []Path + for _, p := range st.paths { + if p.local == "" { + continue + } + + out = append(out, Path{ + ID: p.meta.ID, + Weight: p.meta.Weight, + LocalPath: p.local, + CanSeal: p.meta.CanSeal, + CanStore: p.meta.CanStore, + }) + } + + return out +} + func parseSectorID(baseName string) (abi.SectorID, error) { var n abi.SectorNumber var mid abi.ActorID diff --git a/storage/sealmgr/simple.go b/storage/sealmgr/simple.go index 4d0292e65..a9e122586 100644 --- a/storage/sealmgr/simple.go +++ b/storage/sealmgr/simple.go @@ -16,6 +16,13 @@ import ( ffi "github.com/filecoin-project/filecoin-ffi" ) + +type LocalWorker struct { + sectorbuilder.Basic +} + +var _ Worker = &LocalWorker{} + // Simple implements a very basic storage manager which has one local worker, // running one thing locally type Simple struct { diff --git a/storage/sealmgr/worker_local.go b/storage/sealmgr/worker_local.go deleted file mode 100644 index 67636766a..000000000 --- a/storage/sealmgr/worker_local.go +++ /dev/null @@ -1,11 +0,0 @@ -package sealmgr - -import ( - "github.com/filecoin-project/go-sectorbuilder" -) - -type LocalWorker struct { - sectorbuilder.Basic -} - -var _ Worker = &LocalWorker{} From f06bf6721e272c48258fca807d9761fe42387803 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Mar 2020 19:18:33 +0100 Subject: [PATCH 18/24] sealmgr: Implement all sealing steps --- storage/sealing/garbage.go | 2 +- storage/sealmgr/advmgr/local.go | 37 ++++++++-- storage/sealmgr/advmgr/manager.go | 109 ++++++++++++++++++++++-------- storage/sealmgr/advmgr/storage.go | 4 ++ 4 files changed, 117 insertions(+), 35 deletions(-) diff --git a/storage/sealing/garbage.go b/storage/sealing/garbage.go index 42d1bc80a..47634eb5d 100644 --- a/storage/sealing/garbage.go +++ b/storage/sealing/garbage.go @@ -49,7 +49,7 @@ func (m *Sealing) PledgeSector() error { size := abi.PaddedPieceSize(m.sealer.SectorSize()).Unpadded() - rt, _, err := api.ProofTypeFromSectorSize(m.sealer.SectorSize()) + _, rt, err := api.ProofTypeFromSectorSize(m.sealer.SectorSize()) if err != nil { log.Error(err) return diff --git a/storage/sealmgr/advmgr/local.go b/storage/sealmgr/advmgr/local.go index 9f54d664a..005fade01 100644 --- a/storage/sealmgr/advmgr/local.go +++ b/storage/sealmgr/advmgr/local.go @@ -54,23 +54,48 @@ func (l *localWorker) AddPiece(ctx context.Context, sz abi.UnpaddedPieceSize, sn } func (l *localWorker) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out []byte, err error) { - panic("implement me") + sb, err := l.sb() + if err != nil { + return nil, err + } + + return sb.SealPreCommit1(ctx, sectorNum, ticket, pieces) } func (l *localWorker) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { - panic("implement me") + sb, err := l.sb() + if err != nil { + return cid.Undef, cid.Undef, err + } + + return sb.SealPreCommit2(ctx, sectorNum, phase1Out) } func (l *localWorker) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output []byte, err error) { - panic("implement me") + sb, err := l.sb() + if err != nil { + return nil, err + } + + return sb.SealCommit1(ctx, sectorNum, ticket, seed, pieces, sealedCID, unsealedCID) } func (l *localWorker) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (proof []byte, err error) { - panic("implement me") + sb, err := l.sb() + if err != nil { + return nil, err + } + + return sb.SealCommit2(ctx, sectorNum, phase1Out) } -func (l *localWorker) FinalizeSector(context.Context, abi.SectorNumber) error { - panic("implement me") +func (l *localWorker) FinalizeSector(ctx context.Context, sectorNum abi.SectorNumber) error { + sb, err := l.sb() + if err != nil { + return err + } + + return sb.FinalizeSector(ctx, sectorNum) } func (l *localWorker) TaskTypes() map[sealmgr.TaskType]struct{} { diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index a086c8940..33a736534 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -97,32 +97,19 @@ func (m *Manager) ReadPieceFromSealedSector(context.Context, abi.SectorNumber, s panic("implement me") } -func (m *Manager) AddPiece(ctx context.Context, sz abi.UnpaddedPieceSize, sn abi.SectorNumber, r io.Reader, existingPieces []abi.UnpaddedPieceSize) (abi.PieceInfo, error) { - // TODO: consider multiple paths vs workers when initially allocating - - var best []config.StorageMeta - var err error - if len(existingPieces) == 0 { // new - best, err = m.storage.findBestAllocStorage(sectorbuilder.FTUnsealed, true) - } else { // append to existing - best, err = m.storage.findSector(m.minerID(), sn, sectorbuilder.FTUnsealed) - } - if err != nil { - return abi.PieceInfo{}, xerrors.Errorf("finding sector path: %w", err) - } - - var candidateWorkers []Worker - bestPaths := map[int]config.StorageMeta{} +func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []config.StorageMeta) ([]Worker, map[int]config.StorageMeta) { + var workers []Worker + paths := map[int]config.StorageMeta{} for i, worker := range m.workers { - if _, ok := worker.TaskTypes()[sealmgr.TTAddPiece]; !ok { + if _, ok := worker.TaskTypes()[task]; !ok { continue } // check if the worker has access to the path we selected var st *config.StorageMeta for _, p := range worker.Paths() { - for _, m := range best { + for _, m := range inPaths { if p.ID == m.ID { if st != nil && st.Weight > p.Weight { continue @@ -137,37 +124,103 @@ func (m *Manager) AddPiece(ctx context.Context, sz abi.UnpaddedPieceSize, sn abi continue } - bestPaths[i] = *st - candidateWorkers = append(candidateWorkers, worker) + paths[i] = *st + workers = append(workers, worker) } + return workers, paths +} + +func (m *Manager) AddPiece(ctx context.Context, sz abi.UnpaddedPieceSize, sn abi.SectorNumber, r io.Reader, existingPieces []abi.UnpaddedPieceSize) (abi.PieceInfo, error) { + // TODO: consider multiple paths vs workers when initially allocating + + var best []config.StorageMeta + var err error + if len(existingPieces) == 0 { // new + best, err = m.storage.findBestAllocStorage(sectorbuilder.FTUnsealed, true) + } else { // append to existing + best, err = m.storage.findSector(m.minerID(), sn, sectorbuilder.FTUnsealed) + } + if err != nil { + return abi.PieceInfo{}, xerrors.Errorf("finding sector path: %w", err) + } + + candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTAddPiece, best) + if len(candidateWorkers) == 0 { return abi.PieceInfo{}, xerrors.New("no worker found") } - // TODO: schedule(addpiece, ..., ) - // TODO: remove sectorbuilder abstraction, pass path directly + // TODO: select(candidateWorkers, ...) + // TODO: remove the sectorbuilder abstraction, pass path directly return candidateWorkers[0].AddPiece(ctx, sz, sn, r, existingPieces) } func (m *Manager) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out []byte, err error) { - panic("implement me") + // TODO: also consider where the unsealed data sits + + best, err := m.storage.findBestAllocStorage(sectorbuilder.FTCache | sectorbuilder.FTSealed, true) + if err != nil { + return nil, xerrors.Errorf("finding path for sector sealing: %w", err) + } + + candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTPreCommit1, best) + + // TODO: select(candidateWorkers, ...) + // TODO: remove the sectorbuilder abstraction, pass path directly + return candidateWorkers[0].SealPreCommit1(ctx, sectorNum, ticket, pieces) } func (m *Manager) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { - panic("implement me") + // TODO: allow workers to fetch the sectors + + best, err := m.storage.findSector(m.minerID(), sectorNum, sectorbuilder.FTCache | sectorbuilder.FTSealed) + if err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("finding path for sector sealing: %w", err) + } + + candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTPreCommit2, best) + + // TODO: select(candidateWorkers, ...) + // TODO: remove the sectorbuilder abstraction, pass path directly + return candidateWorkers[0].SealPreCommit2(ctx, sectorNum, phase1Out) } func (m *Manager) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output []byte, err error) { - panic("implement me") + best, err := m.storage.findSector(m.minerID(), sectorNum, sectorbuilder.FTCache | sectorbuilder.FTSealed) + if err != nil { + return nil, xerrors.Errorf("finding path for sector sealing: %w", err) + } + + candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTPreCommit2, best) + + // TODO: select(candidateWorkers, ...) + // TODO: remove the sectorbuilder abstraction, pass path directly + return candidateWorkers[0].SealCommit1(ctx, sectorNum, ticket, seed, pieces, sealedCID, unsealedCID) } func (m *Manager) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (proof []byte, err error) { - panic("implement me") + for _, worker := range m.workers { + if _, ok := worker.TaskTypes()[sealmgr.TTCommit2]; !ok { + continue + } + + return worker.SealCommit2(ctx, sectorNum, phase1Out) + } + + return nil, xerrors.New("no worker found") } -func (m *Manager) FinalizeSector(context.Context, abi.SectorNumber) error { - panic("implement me") +func (m *Manager) FinalizeSector(ctx context.Context, sectorNum abi.SectorNumber) error { + best, err := m.storage.findSector(m.minerID(), sectorNum, sectorbuilder.FTCache | sectorbuilder.FTSealed | sectorbuilder.FTUnsealed) + if err != nil { + return xerrors.Errorf("finding sealed sector: %w", err) + } + + candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTPreCommit2, best) // find last worker with the sector + + // TODO: Move the sector to long-term storage + return candidateWorkers[0].FinalizeSector(ctx, sectorNum) } func (m *Manager) minerID() abi.ActorID { diff --git a/storage/sealmgr/advmgr/storage.go b/storage/sealmgr/advmgr/storage.go index eef4489c1..159159071 100644 --- a/storage/sealmgr/advmgr/storage.go +++ b/storage/sealmgr/advmgr/storage.go @@ -156,6 +156,10 @@ func (st *storage) acquireSector(mid abi.ActorID, id abi.SectorNumber, existing if !sealing && !p.meta.CanStore { continue } + p.sectors[abi.SectorID{ + Miner: mid, + Number: id, + }] |= fileType // TODO: Check free space // TODO: Calc weights From 7d95e134eb8001afceaf4bbebf5dcaf4d9fc2f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Mar 2020 19:26:05 +0100 Subject: [PATCH 19/24] storageminer: Print what we can in info cmd --- cmd/lotus-storage-miner/info.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index d3f13ad2c..43c788b08 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -72,7 +72,6 @@ var infoCmd = &cli.Command{ float64(10000*uint64(len(faults))/secCounts.Pset)/100.) } - panic("todo") /*// TODO: indicate whether the post worker is in use wstat, err := nodeApi.WorkerStats(ctx) if err != nil { From a5e5918fc5f3caac0e58bcbaaff178f39105d784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Mar 2020 20:21:06 +0100 Subject: [PATCH 20/24] mod tidy; gofmt --- api/api_storage.go | 4 +- api/apistruct/struct.go | 8 +- cmd/lotus-storage-miner/init.go | 25 +++-- go.sum | 10 -- node/config/def.go | 1 - node/config/storage.go | 3 +- node/impl/storminer.go | 172 +++++++++++++++--------------- node/repo/fsrepo.go | 14 +-- storage/fpost_run.go | 5 +- storage/sealmgr/advmgr/local.go | 10 +- storage/sealmgr/advmgr/manager.go | 20 ++-- storage/sealmgr/advmgr/roprov.go | 2 +- storage/sealmgr/advmgr/storage.go | 17 +-- storage/sealmgr/simple.go | 1 - 14 files changed, 142 insertions(+), 150 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index 6c729f63c..ba6efc93c 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -116,7 +116,7 @@ type StorageMiner interface { WorkerQueue(context.Context, sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) WorkerDone(ctx context.Context, task uint64, res sectorbuilder.SealRes) error -*/ + */ MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error) MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error) @@ -184,4 +184,4 @@ func (st *SealTicket) Equals(ost *SealTicket) bool { func (st *SealSeed) Equals(ost *SealSeed) bool { return bytes.Equal(st.Value, ost.Value) && st.Epoch == ost.Epoch -} \ No newline at end of file +} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 16b02412c..8fb9ddfa8 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -171,11 +171,11 @@ type StorageMinerStruct struct { SectorsRefs func(context.Context) (map[string][]api.SealedRef, error) `perm:"read"` SectorsUpdate func(context.Context, abi.SectorNumber, api.SectorState) error `perm:"write"` -/* WorkerStats func(context.Context) (sectorbuilder.WorkerStats, error) `perm:"read"` + /* WorkerStats func(context.Context) (sectorbuilder.WorkerStats, error) `perm:"read"` - WorkerQueue func(ctx context.Context, cfg sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) `perm:"admin"` // TODO: worker perm - WorkerDone func(ctx context.Context, task uint64, res sectorbuilder.SealRes) error `perm:"admin"` -*/ + WorkerQueue func(ctx context.Context, cfg sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) `perm:"admin"` // TODO: worker perm + WorkerDone func(ctx context.Context, task uint64, res sectorbuilder.SealRes) error `perm:"admin"` + */ SetPrice func(context.Context, types.BigInt) error `perm:"admin"` DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index c2781f714..f4662670e 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -186,19 +186,19 @@ var initCmd = &cli.Command{ } if !cctx.Bool("no-local-storage") { - b, err := json.MarshalIndent(&config.StorageMeta{ - ID: uuid.New().String(), - Weight: 10, - CanSeal: true, - CanStore: true, - }, "", " ") - if err != nil { - return xerrors.Errorf("marshaling storage config: %w", err) - } + b, err := json.MarshalIndent(&config.StorageMeta{ + ID: uuid.New().String(), + Weight: 10, + CanSeal: true, + CanStore: true, + }, "", " ") + if err != nil { + return xerrors.Errorf("marshaling storage config: %w", err) + } - if err := ioutil.WriteFile(filepath.Join(lr.Path(), "sectorstore.json"), b, 0644); err != nil { - return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "storage.json"), err) - } + if err := ioutil.WriteFile(filepath.Join(lr.Path(), "sectorstore.json"), b, 0644); err != nil { + return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "storage.json"), err) + } } sc.StoragePaths = append(sc.StoragePaths, config.LocalPath{ @@ -214,7 +214,6 @@ var initCmd = &cli.Command{ } } - if err := storageMinerInit(ctx, cctx, api, r, ssize); err != nil { log.Errorf("Failed to initialize lotus-storage-miner: %+v", err) path, err := homedir.Expand(repoPath) diff --git a/go.sum b/go.sum index eb7eaac09..b0cf94939 100644 --- a/go.sum +++ b/go.sum @@ -119,8 +119,6 @@ github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6/go.m github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 h1:eYxi6vI5CyeXD15X1bB3bledDXbqKxqf0wQzTLgwYwA= github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200226210935-4739f8749f56/go.mod h1:tzTc9BxxSbjlIzhFwm5h9oBkXKkRuLxeiWspntwnKyw= -github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200229022239-442fe78a3168 h1:6fBZezecouqHNRi4Q4YlH7QqgJ3yZZ7adMBpUBAx4F8= -github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200229022239-442fe78a3168/go.mod h1:gEQJVRVqQX8Vx02IosTdC2UA4l2MgHfG3POJEI2GIpc= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200304050010-2cfac00a93e7 h1:pGvvPZgIfBXAda+OGc943zFcM8Wt4GG9UVymVeyfx6c= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200304050010-2cfac00a93e7/go.mod h1:gEQJVRVqQX8Vx02IosTdC2UA4l2MgHfG3POJEI2GIpc= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 h1:k9qVR9ItcziSB2rxtlkN/MDWNlbsI6yzec+zjUatLW0= @@ -129,14 +127,6 @@ github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIi github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.0.0-20200226200336-94c9b92b2775/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= -github.com/filecoin-project/specs-actors v0.0.0-20200229011003-1d726e3afd04 h1:O343OeQLkLWLj5ZqQ5nhevAGBTeB5LioiA53ddScqdY= -github.com/filecoin-project/specs-actors v0.0.0-20200229011003-1d726e3afd04/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= -github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf h1:3IojVqJAD5IXMxvZ+WYx+LRbfSB/rOXpYBuHh6o3XkY= -github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf h1:3IojVqJAD5IXMxvZ+WYx+LRbfSB/rOXpYBuHh6o3XkY= -github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= -github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= -github.com/filecoin-project/specs-storage v0.0.0-20200303230804-16c9a38030eb h1:oHB9hKaD7g75NFulnfh+SCYS5bSl8hB6Eanf8A6l5tw= -github.com/filecoin-project/specs-storage v0.0.0-20200303230804-16c9a38030eb/go.mod h1:sC2Ck2l1G8hXI5Do/3sp0yxbMRMnukbFwP9KF1CRFLw= github.com/filecoin-project/specs-actors v0.0.0-20200304210626-21ee86aadcb9 h1:5/XkV9N7Zlidi2RYY/04BToD/XeQrudUseI7Gx6owl8= github.com/filecoin-project/specs-actors v0.0.0-20200304210626-21ee86aadcb9/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= diff --git a/node/config/def.go b/node/config/def.go index dafef3f4d..69355aadd 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -53,7 +53,6 @@ type Metrics struct { // // Storage Miner type Storage struct { - } func defCommon() Common { diff --git a/node/config/storage.go b/node/config/storage.go index 3761a0b96..3e6c9d2e2 100644 --- a/node/config/storage.go +++ b/node/config/storage.go @@ -20,7 +20,7 @@ type StorageConfig struct { // [path]/metadata.json type StorageMeta struct { - ID string + ID string Weight uint64 // 0 = readonly CanSeal bool @@ -65,4 +65,3 @@ func WriteStorageFile(path string, config StorageConfig) error { return nil } - diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 4f91670cd..372211738 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -29,7 +29,7 @@ type StorageMinerAPI struct { SectorBuilderConfig *sectorbuilder.Config //SectorBuilder sectorbuilder.Interface - SectorBlocks *sectorblocks.SectorBlocks + SectorBlocks *sectorblocks.SectorBlocks StorageProvider storagemarket.StorageProvider Miner *storage.Miner @@ -56,111 +56,112 @@ func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { func (sm *StorageMinerAPI) remoteGetSector(w http.ResponseWriter, r *http.Request) { panic("todo") -/* vars := mux.Vars(r) + /* vars := mux.Vars(r) - id, err := strconv.ParseUint(vars["id"], 10, 64) - if err != nil { - log.Error("parsing sector id: ", err) - w.WriteHeader(500) - return - } - - path, err := sm.SectorBuilder.SectorPath(fs.DataType(vars["type"]), abi.SectorNumber(id)) - if err != nil { - log.Error(err) - w.WriteHeader(500) - return - } - - stat, err := os.Stat(string(path)) - if err != nil { - log.Error(err) - w.WriteHeader(500) - return - } - - var rd io.Reader - if stat.IsDir() { - rd, err = tarutil.TarDirectory(string(path)) - w.Header().Set("Content-Type", "application/x-tar") - } else { - rd, err = os.OpenFile(string(path), os.O_RDONLY, 0644) - w.Header().Set("Content-Type", "application/octet-stream") - } - if err != nil { - log.Error(err) - w.WriteHeader(500) - return - } - - w.WriteHeader(200) - if _, err := io.Copy(w, rd); err != nil { - log.Error(err) - return - }*/ -} - -func (sm *StorageMinerAPI) remotePutSector(w http.ResponseWriter, r *http.Request) { - panic("todo") -/* vars := mux.Vars(r) - - id, err := strconv.ParseUint(vars["id"], 10, 64) - if err != nil { - log.Error("parsing sector id: ", err) - w.WriteHeader(500) - return - } - - // This is going to get better with worker-to-worker transfers - - path, err := sm.SectorBuilder.SectorPath(fs.DataType(vars["type"]), abi.SectorNumber(id)) - if err != nil { - if err != fs.ErrNotFound { - log.Error(err) + id, err := strconv.ParseUint(vars["id"], 10, 64) + if err != nil { + log.Error("parsing sector id: ", err) w.WriteHeader(500) return } - path, err = sm.SectorBuilder.AllocSectorPath(fs.DataType(vars["type"]), abi.SectorNumber(id), true) + path, err := sm.SectorBuilder.SectorPath(fs.DataType(vars["type"]), abi.SectorNumber(id)) if err != nil { log.Error(err) w.WriteHeader(500) return } - } - mediatype, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) - if err != nil { - log.Error(err) - w.WriteHeader(500) - return - } - - if err := os.RemoveAll(string(path)); err != nil { - log.Error(err) - w.WriteHeader(500) - return - } - - switch mediatype { - case "application/x-tar": - if err := tarutil.ExtractTar(r.Body, string(path)); err != nil { + stat, err := os.Stat(string(path)) + if err != nil { log.Error(err) w.WriteHeader(500) return } - default: - if err := files.WriteTo(files.NewReaderFile(r.Body), string(path)); err != nil { + + var rd io.Reader + if stat.IsDir() { + rd, err = tarutil.TarDirectory(string(path)) + w.Header().Set("Content-Type", "application/x-tar") + } else { + rd, err = os.OpenFile(string(path), os.O_RDONLY, 0644) + w.Header().Set("Content-Type", "application/octet-stream") + } + if err != nil { log.Error(err) w.WriteHeader(500) return } - } - w.WriteHeader(200) - - log.Infof("received %s sector (%s): %d bytes", vars["type"], vars["sname"], r.ContentLength)*/ + w.WriteHeader(200) + if _, err := io.Copy(w, rd); err != nil { + log.Error(err) + return + }*/ } + +func (sm *StorageMinerAPI) remotePutSector(w http.ResponseWriter, r *http.Request) { + panic("todo") + /* vars := mux.Vars(r) + + id, err := strconv.ParseUint(vars["id"], 10, 64) + if err != nil { + log.Error("parsing sector id: ", err) + w.WriteHeader(500) + return + } + + // This is going to get better with worker-to-worker transfers + + path, err := sm.SectorBuilder.SectorPath(fs.DataType(vars["type"]), abi.SectorNumber(id)) + if err != nil { + if err != fs.ErrNotFound { + log.Error(err) + w.WriteHeader(500) + return + } + + path, err = sm.SectorBuilder.AllocSectorPath(fs.DataType(vars["type"]), abi.SectorNumber(id), true) + if err != nil { + log.Error(err) + w.WriteHeader(500) + return + } + } + + mediatype, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil { + log.Error(err) + w.WriteHeader(500) + return + } + + if err := os.RemoveAll(string(path)); err != nil { + log.Error(err) + w.WriteHeader(500) + return + } + + switch mediatype { + case "application/x-tar": + if err := tarutil.ExtractTar(r.Body, string(path)); err != nil { + log.Error(err) + w.WriteHeader(500) + return + } + default: + if err := files.WriteTo(files.NewReaderFile(r.Body), string(path)); err != nil { + log.Error(err) + w.WriteHeader(500) + return + } + } + + w.WriteHeader(200) + + log.Infof("received %s sector (%s): %d bytes", vars["type"], vars["sname"], r.ContentLength)*/ +} + /* func (sm *StorageMinerAPI) WorkerStats(context.Context) (sectorbuilder.WorkerStats, error) { stat := sm.SectorBuilder.WorkerStats() @@ -252,6 +253,7 @@ func (sm *StorageMinerAPI) SectorsRefs(context.Context) (map[string][]api.Sealed func (sm *StorageMinerAPI) SectorsUpdate(ctx context.Context, id abi.SectorNumber, state api.SectorState) error { return sm.Miner.ForceSectorState(ctx, id, state) } + /* func (sm *StorageMinerAPI) WorkerQueue(ctx context.Context, cfg sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) { return sm.SectorBuilder.AddWorker(ctx, cfg) diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 881acad63..0c1ff82ae 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -25,13 +25,13 @@ import ( ) const ( - fsAPI = "api" - fsAPIToken = "token" - fsConfig = "config.toml" + fsAPI = "api" + fsAPIToken = "token" + fsConfig = "config.toml" fsStorageConfig = "storage.json" - fsDatastore = "datastore" - fsLock = "repo.lock" - fsKeystore = "keystore" + fsDatastore = "datastore" + fsLock = "repo.lock" + fsKeystore = "keystore" ) type RepoType int @@ -88,7 +88,7 @@ func (fsr *FsRepo) Exists() (bool, error) { func (fsr *FsRepo) Init(t RepoType) error { exist, err := fsr.Exists() - if err != nil{ + if err != nil { return err } if exist { diff --git a/storage/fpost_run.go b/storage/fpost_run.go index 5110d00b8..6c53c2dd8 100644 --- a/storage/fpost_run.go +++ b/storage/fpost_run.go @@ -92,7 +92,10 @@ func (s *FPoStScheduler) declareFaults(ctx context.Context, fc uint64, params *m func (s *FPoStScheduler) checkFaults(ctx context.Context, ssi []abi.SectorNumber) ([]abi.SectorNumber, error) { //faults := s.sb.Scrub(ssi) log.Warnf("Stub checkFaults") - var faults []struct{SectorNum abi.SectorNumber; Err error} + var faults []struct { + SectorNum abi.SectorNumber + Err error + } declaredFaults := map[abi.SectorNumber]struct{}{} diff --git a/storage/sealmgr/advmgr/local.go b/storage/sealmgr/advmgr/local.go index 005fade01..81f8a6727 100644 --- a/storage/sealmgr/advmgr/local.go +++ b/storage/sealmgr/advmgr/local.go @@ -15,7 +15,7 @@ import ( ) type localWorker struct { - scfg *sectorbuilder.Config + scfg *sectorbuilder.Config storage *storage } @@ -41,7 +41,7 @@ func (l *localWorkerPathProvider) AcquireSector(id abi.SectorNumber, existing se } func (l *localWorker) sb() (sectorbuilder.Basic, error) { - return sectorbuilder.New(&localWorkerPathProvider{w:l}, l.scfg) + return sectorbuilder.New(&localWorkerPathProvider{w: l}, l.scfg) } func (l *localWorker) AddPiece(ctx context.Context, sz abi.UnpaddedPieceSize, sn abi.SectorNumber, r io.Reader, epcs []abi.UnpaddedPieceSize) (abi.PieceInfo, error) { @@ -100,10 +100,10 @@ func (l *localWorker) FinalizeSector(ctx context.Context, sectorNum abi.SectorNu func (l *localWorker) TaskTypes() map[sealmgr.TaskType]struct{} { return map[sealmgr.TaskType]struct{}{ - sealmgr.TTAddPiece: {}, + sealmgr.TTAddPiece: {}, sealmgr.TTPreCommit1: {}, sealmgr.TTPreCommit2: {}, - sealmgr.TTCommit2: {}, + sealmgr.TTCommit2: {}, } } @@ -111,4 +111,4 @@ func (l *localWorker) Paths() []Path { return l.storage.local() } -var _ Worker = &localWorker{} \ No newline at end of file +var _ Worker = &localWorker{} diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index 33a736534..bbe241c1d 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -25,12 +25,12 @@ type LocalStorage interface { } type Path struct { - ID string + ID string Weight uint64 LocalPath string - CanSeal bool + CanSeal bool CanStore bool } @@ -43,8 +43,8 @@ type Worker interface { type Manager struct { workers []Worker - scfg *sectorbuilder.Config - sc SectorIDCounter + scfg *sectorbuilder.Config + sc SectorIDCounter storage *storage @@ -70,11 +70,11 @@ func New(ls LocalStorage, cfg *sectorbuilder.Config, sc SectorIDCounter) (*Manag } m := &Manager{ - workers: []Worker{ + workers: []Worker{ &localWorker{scfg: cfg, storage: stor}, }, scfg: cfg, - sc: sc, + sc: sc, storage: stor, @@ -159,7 +159,7 @@ func (m *Manager) AddPiece(ctx context.Context, sz abi.UnpaddedPieceSize, sn abi func (m *Manager) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out []byte, err error) { // TODO: also consider where the unsealed data sits - best, err := m.storage.findBestAllocStorage(sectorbuilder.FTCache | sectorbuilder.FTSealed, true) + best, err := m.storage.findBestAllocStorage(sectorbuilder.FTCache|sectorbuilder.FTSealed, true) if err != nil { return nil, xerrors.Errorf("finding path for sector sealing: %w", err) } @@ -174,7 +174,7 @@ func (m *Manager) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber func (m *Manager) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out []byte) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { // TODO: allow workers to fetch the sectors - best, err := m.storage.findSector(m.minerID(), sectorNum, sectorbuilder.FTCache | sectorbuilder.FTSealed) + best, err := m.storage.findSector(m.minerID(), sectorNum, sectorbuilder.FTCache|sectorbuilder.FTSealed) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("finding path for sector sealing: %w", err) } @@ -187,7 +187,7 @@ func (m *Manager) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber } func (m *Manager) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output []byte, err error) { - best, err := m.storage.findSector(m.minerID(), sectorNum, sectorbuilder.FTCache | sectorbuilder.FTSealed) + best, err := m.storage.findSector(m.minerID(), sectorNum, sectorbuilder.FTCache|sectorbuilder.FTSealed) if err != nil { return nil, xerrors.Errorf("finding path for sector sealing: %w", err) } @@ -212,7 +212,7 @@ func (m *Manager) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, p } func (m *Manager) FinalizeSector(ctx context.Context, sectorNum abi.SectorNumber) error { - best, err := m.storage.findSector(m.minerID(), sectorNum, sectorbuilder.FTCache | sectorbuilder.FTSealed | sectorbuilder.FTUnsealed) + best, err := m.storage.findSector(m.minerID(), sectorNum, sectorbuilder.FTCache|sectorbuilder.FTSealed|sectorbuilder.FTUnsealed) if err != nil { return xerrors.Errorf("finding sealed sector: %w", err) } diff --git a/storage/sealmgr/advmgr/roprov.go b/storage/sealmgr/advmgr/roprov.go index cb3e6854c..db7353c0c 100644 --- a/storage/sealmgr/advmgr/roprov.go +++ b/storage/sealmgr/advmgr/roprov.go @@ -8,7 +8,7 @@ import ( type readonlyProvider struct { miner abi.ActorID - stor *storage + stor *storage } func (l *readonlyProvider) AcquireSectorNumber() (abi.SectorNumber, error) { diff --git a/storage/sealmgr/advmgr/storage.go b/storage/sealmgr/advmgr/storage.go index 159159071..cb5533c21 100644 --- a/storage/sealmgr/advmgr/storage.go +++ b/storage/sealmgr/advmgr/storage.go @@ -17,17 +17,18 @@ import ( ) const metaFile = "sectorstore.json" + var pathTypes = []sectorbuilder.SectorFileType{sectorbuilder.FTUnsealed, sectorbuilder.FTSealed, sectorbuilder.FTCache} type storage struct { - localLk sync.RWMutex + localLk sync.RWMutex localStorage LocalStorage paths []path } type path struct { - meta config.StorageMeta + meta config.StorageMeta local string sectors map[abi.SectorID]sectorbuilder.SectorFileType @@ -102,7 +103,7 @@ func (st *storage) open() error { } func (st *storage) acquireSector(mid abi.ActorID, id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { - if existing | allocate != existing ^ allocate { + if existing|allocate != existing^allocate { return sectorbuilder.SectorPaths{}, nil, xerrors.New("can't both find and allocate a sector") } @@ -111,7 +112,7 @@ func (st *storage) acquireSector(mid abi.ActorID, id abi.SectorNumber, existing var out sectorbuilder.SectorPaths for _, fileType := range pathTypes { - if fileType & existing == 0 { + if fileType&existing == 0 { continue } @@ -123,7 +124,7 @@ func (st *storage) acquireSector(mid abi.ActorID, id abi.SectorNumber, existing if !ok { continue } - if s & fileType == 0 { + if s&fileType == 0 { continue } @@ -143,7 +144,7 @@ func (st *storage) acquireSector(mid abi.ActorID, id abi.SectorNumber, existing } for _, fileType := range pathTypes { - if fileType & allocate == 0 { + if fileType&allocate == 0 { continue } @@ -219,7 +220,7 @@ func (st *storage) findSector(mid abi.ActorID, sn abi.SectorNumber, typ sectorbu Miner: mid, Number: sn, }] - if t | typ == 0 { + if t|typ == 0 { continue } out = append(out, p.meta) @@ -266,4 +267,4 @@ func parseSectorID(baseName string) (abi.SectorID, error) { Miner: mid, Number: n, }, nil -} \ No newline at end of file +} diff --git a/storage/sealmgr/simple.go b/storage/sealmgr/simple.go index a9e122586..15b714ba5 100644 --- a/storage/sealmgr/simple.go +++ b/storage/sealmgr/simple.go @@ -16,7 +16,6 @@ import ( ffi "github.com/filecoin-project/filecoin-ffi" ) - type LocalWorker struct { sectorbuilder.Basic } From 1461e475da004efd5d44ade6161c5d2bef259686 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Mar 2020 23:02:01 +0100 Subject: [PATCH 21/24] storage: Support adding paths at runtime --- api/api_storage.go | 2 ++ api/apistruct/struct.go | 6 +++++ node/impl/storminer.go | 10 ++++++++ node/modules/services.go | 2 +- node/node_test.go | 41 ++++++++++-------------------- node/repo/memrepo.go | 42 ++++++++++++++++++++++++++++--- storage/sbmock/sbmock_test.go | 2 +- storage/sealmgr/advmgr/manager.go | 20 +++++++++++++++ storage/sealmgr/advmgr/storage.go | 38 +++++++++++++++------------- 9 files changed, 112 insertions(+), 51 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index ba6efc93c..6b3705b79 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -124,6 +124,8 @@ type StorageMiner interface { DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) + + StorageAddLocal(ctx context.Context, path string) error } type SealRes struct { diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 8fb9ddfa8..e11a89aab 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -180,6 +180,8 @@ type StorageMinerStruct struct { DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` + + StorageAddLocal func(ctx context.Context, path string) error `perm:"admin"` } } @@ -645,6 +647,10 @@ func (c *StorageMinerStruct) DealsList(ctx context.Context) ([]storagemarket.Sto return c.Internal.DealsList(ctx) } +func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) error { + return c.Internal.StorageAddLocal(ctx, path) +} + var _ api.Common = &CommonStruct{} var _ api.FullNode = &FullNodeStruct{} var _ api.StorageMiner = &StorageMinerStruct{} diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 372211738..150f25130 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -21,6 +21,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -35,6 +36,7 @@ type StorageMinerAPI struct { Miner *storage.Miner BlockMiner *miner.Miner Full api.FullNode + StorageMgr *advmgr.Manager `optional:"true"` } func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { @@ -300,4 +302,12 @@ func (sm *StorageMinerAPI) DealsImportData(ctx context.Context, deal cid.Cid, fn return sm.StorageProvider.ImportDataForDeal(ctx, deal, fi) } +func (sm *StorageMinerAPI) StorageAddLocal(ctx context.Context, path string) error { + if sm.StorageMgr == nil { + return xerrors.Errorf("no storage manager") + } + + return sm.StorageMgr.AddLocalStorage(path) +} + var _ api.StorageMiner = &StorageMinerAPI{} diff --git a/node/modules/services.go b/node/modules/services.go index 49bc4e5ae..cac24391b 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -71,7 +71,7 @@ func HandleIncomingBlocks(mctx helpers.MetricsCtx, lc fx.Lifecycle, ps *pubsub.P func HandleIncomingMessages(mctx helpers.MetricsCtx, lc fx.Lifecycle, ps *pubsub.PubSub, mpool *messagepool.MessagePool) { ctx := helpers.LifecycleCtx(mctx, lc) - msgsub, err := ps.Subscribe(build.BlocksTopic) + msgsub, err := ps.Subscribe(build.MessagesTopic) if err != nil { panic(err) } diff --git a/node/node_test.go b/node/node_test.go index b97809b2e..2b5ac072d 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -6,13 +6,10 @@ import ( "crypto/rand" "io/ioutil" "net/http/httptest" - "path/filepath" "testing" "time" "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - badger "github.com/ipfs/go-ds-badger2" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" @@ -39,11 +36,12 @@ import ( "github.com/filecoin-project/lotus/lib/jsonrpc" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node" - "github.com/filecoin-project/lotus/node/impl" "github.com/filecoin-project/lotus/node/modules" modtest "github.com/filecoin-project/lotus/node/modules/testing" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage/sbmock" + "github.com/filecoin-project/lotus/storage/sealmgr" + "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" ) func init() { @@ -240,40 +238,24 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te f := fulls[full] if _, err := f.FullNode.WalletImport(ctx, &keys[i].KeyInfo); err != nil { - return nil, nil + t.Fatal(err) } if err := f.FullNode.WalletSetDefault(ctx, keys[i].Address); err != nil { - return nil, nil + t.Fatal(err) } genMiner := maddrs[i] wa := genms[i].Worker storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options()) - - sma := storers[i].StorageMiner.(*impl.StorageMinerAPI) - - psd := presealDirs[i] - mds, err := badger.NewDatastore(filepath.Join(psd, "badger"), nil) - if err != nil { - t.Fatal(err) - } - - osb, err := sectorbuilder.New(§orbuilder.Config{ - SealProofType: abi.RegisteredProof_StackedDRG2KiBSeal, - PoStProofType: abi.RegisteredProof_StackedDRG2KiBPoSt, - WorkerThreads: 2, - Miner: genMiner, - Paths: sectorbuilder.SimplePath(psd), - }, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder"))) - if err != nil { - t.Fatal(err) - } - - if err := sma.SectorBuilder.(*sectorbuilder.SectorBuilder).ImportFrom(osb, false); err != nil { + if err := storers[i].StorageAddLocal(ctx, presealDirs[i]); err != nil { t.Fatal(err) } + /* + sma := storers[i].StorageMiner.(*impl.StorageMinerAPI) + psd := presealDirs[i] + */ } if err := mn.LinkAll(); err != nil { @@ -394,7 +376,10 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t wa := genms[i].Worker storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options( - node.Override(new(sectorbuilder.Interface), sbmock.NewMockSectorBuilder(5, build.SectorSizes[0])), + node.Override(new(sealmgr.Manager), func() (sealmgr.Manager, error) { + return sealmgr.NewSimpleManager(nil, genMiner, sbmock.NewMockSectorBuilder(5, build.SectorSizes[0])) + }), + node.Unset(new(*advmgr.Manager)), )) } diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index d1b491831..ddc283bee 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -1,10 +1,13 @@ package repo import ( + "encoding/json" "io/ioutil" "os" + "path/filepath" "sync" + "github.com/google/uuid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" dssync "github.com/ipfs/go-datastore/sync" @@ -40,7 +43,9 @@ type lockedMemRepo struct { } func (lmem *lockedMemRepo) GetStorage() (config.StorageConfig, error) { - panic("implement me") + return config.StorageConfig{StoragePaths: []config.LocalPath{ + {Path: lmem.Path()}, + }}, nil } func (lmem *lockedMemRepo) SetStorage(config.StorageConfig) error { @@ -48,14 +53,45 @@ func (lmem *lockedMemRepo) SetStorage(config.StorageConfig) error { } func (lmem *lockedMemRepo) Path() string { + lmem.Lock() + defer lmem.Unlock() + + if lmem.tempDir != "" { + return lmem.tempDir + } + t, err := ioutil.TempDir(os.TempDir(), "lotus-memrepo-temp-") if err != nil { panic(err) // only used in tests, probably fine } - lmem.Lock() + if lmem.t == StorageMiner { + if err := config.WriteStorageFile(filepath.Join(t, fsStorageConfig), config.StorageConfig{ + StoragePaths: []config.LocalPath{ + {Path: t}, + }}); err != nil { + panic(err) + } + + b, err := json.MarshalIndent(&config.StorageMeta{ + ID: uuid.New().String(), + Weight: 10, + CanSeal: true, + CanStore: true, + }, "", " ") + if err != nil { + panic(err) + } + + if err := ioutil.WriteFile(filepath.Join(t, "sectorstore.json"), b, 0644); err != nil { + panic(err) + } + } + + + + lmem.tempDir = t - lmem.Unlock() return t } diff --git a/storage/sbmock/sbmock_test.go b/storage/sbmock/sbmock_test.go index d76efa375..6bfe9e062 100644 --- a/storage/sbmock/sbmock_test.go +++ b/storage/sbmock/sbmock_test.go @@ -20,7 +20,7 @@ func TestOpFinish(t *testing.T) { finished := make(chan struct{}) go func() { - _, _, err := sb.SealPreCommit(ctx, sid, abi.SealRandomness{}, pieces) + _, err := sb.SealPreCommit1(ctx, sid, abi.SealRandomness{}, pieces) if err != nil { t.Error(err) return diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index bbe241c1d..cbc3dfb97 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -5,6 +5,7 @@ import ( "io" "github.com/ipfs/go-cid" + "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -84,6 +85,25 @@ func New(ls LocalStorage, cfg *sectorbuilder.Config, sc SectorIDCounter) (*Manag return m, nil } +func (m *Manager) AddLocalStorage(path string) error { + path, err := homedir.Expand(path) + if err != nil { + return xerrors.Errorf("expanding local path: %w", err) + } + + if err := m.storage.openPath(path); err != nil { + return xerrors.Errorf("opening local path: %w", err) + } + + sc, err := m.storage.localStorage.GetStorage() + if err != nil { + return xerrors.Errorf("get storage config: %w", err) + } + + sc.StoragePaths = append(sc.StoragePaths, config.LocalPath{Path: path}) + return nil +} + func (m *Manager) SectorSize() abi.SectorSize { sz, _ := m.scfg.SealProofType.SectorSize() return sz diff --git a/storage/sealmgr/advmgr/storage.go b/storage/sealmgr/advmgr/storage.go index cb5533c21..564935942 100644 --- a/storage/sealmgr/advmgr/storage.go +++ b/storage/sealmgr/advmgr/storage.go @@ -34,7 +34,19 @@ type path struct { sectors map[abi.SectorID]sectorbuilder.SectorFileType } -func openPath(p string, meta config.StorageMeta) (path, error) { +func (st *storage) openPath(p string) error { + mb, err := ioutil.ReadFile(filepath.Join(p, metaFile)) + if err != nil { + return xerrors.Errorf("reading storage metadata for %s: %w", p, err) + } + + var meta config.StorageMeta + if err := json.Unmarshal(mb, &meta); err != nil { + return xerrors.Errorf("unmarshalling storage metadata for %s: %w", p, err) + } + + // TODO: Check existing / dedupe + out := path{ meta: meta, local: p, @@ -46,25 +58,27 @@ func openPath(p string, meta config.StorageMeta) (path, error) { if err != nil { if os.IsNotExist(err) { if err := os.MkdirAll(filepath.Join(p, t.String()), 0755); err != nil { - return path{}, xerrors.Errorf("mkdir '%s': %w", filepath.Join(p, t.String()), err) + return xerrors.Errorf("mkdir '%s': %w", filepath.Join(p, t.String()), err) } continue } - return path{}, xerrors.Errorf("listing %s: %w", filepath.Join(p, t.String()), err) + return xerrors.Errorf("listing %s: %w", filepath.Join(p, t.String()), err) } for _, ent := range ents { sid, err := parseSectorID(ent.Name()) if err != nil { - return path{}, xerrors.Errorf("parse sector id %s: %w", ent.Name(), err) + return xerrors.Errorf("parse sector id %s: %w", ent.Name(), err) } out.sectors[sid] |= t } } - return out, nil + st.paths = append(st.paths, out) + + return nil } func (st *storage) open() error { @@ -81,22 +95,10 @@ func (st *storage) open() error { } for _, path := range cfg.StoragePaths { - mb, err := ioutil.ReadFile(filepath.Join(path.Path, metaFile)) - if err != nil { - return xerrors.Errorf("reading storage metadata for %s: %w", path.Path, err) - } - - var meta config.StorageMeta - if err := json.Unmarshal(mb, &meta); err != nil { - return xerrors.Errorf("unmarshalling storage metadata for %s: %w", path.Path, err) - } - - pi, err := openPath(path.Path, meta) + err := st.openPath(path.Path) if err != nil { return xerrors.Errorf("opening path %s: %w", path.Path, err) } - - st.paths = append(st.paths, pi) } return nil From 16cb27b5f92598aea9ef8f5b74b5d015455cf8be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Mar 2020 23:27:30 +0100 Subject: [PATCH 22/24] Get node_test to pass --- node/impl/storminer.go | 2 +- node/node_test.go | 3 ++- node/repo/memrepo.go | 3 --- storage/sealmgr/simple.go | 6 +++++- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 150f25130..f4bee869b 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -36,7 +36,7 @@ type StorageMinerAPI struct { Miner *storage.Miner BlockMiner *miner.Miner Full api.FullNode - StorageMgr *advmgr.Manager `optional:"true"` + StorageMgr *advmgr.Manager `optional:"true"` } func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { diff --git a/node/node_test.go b/node/node_test.go index 2b5ac072d..55bfd6eb2 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/filecoin-project/go-fil-markets/storedcounter" "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/crypto" @@ -377,7 +378,7 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options( node.Override(new(sealmgr.Manager), func() (sealmgr.Manager, error) { - return sealmgr.NewSimpleManager(nil, genMiner, sbmock.NewMockSectorBuilder(5, build.SectorSizes[0])) + return sealmgr.NewSimpleManager(storedcounter.New(datastore.NewMapDatastore(), datastore.NewKey("/potato")), genMiner, sbmock.NewMockSectorBuilder(5, build.SectorSizes[0])) }), node.Unset(new(*advmgr.Manager)), )) diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index ddc283bee..919c09376 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -88,9 +88,6 @@ func (lmem *lockedMemRepo) Path() string { } } - - - lmem.tempDir = t return t } diff --git a/storage/sealmgr/simple.go b/storage/sealmgr/simple.go index 15b714ba5..ff291b129 100644 --- a/storage/sealmgr/simple.go +++ b/storage/sealmgr/simple.go @@ -32,8 +32,12 @@ type Simple struct { worker Worker } +type sszgetter interface { + SectorSize() abi.SectorSize +} + func (s *Simple) SectorSize() abi.SectorSize { - panic("implement me") + return s.worker.(sszgetter).SectorSize() } func NewSimpleManager(sc *storedcounter.StoredCounter, maddr address.Address, sb sectorbuilder.Basic) (*Simple, error) { From d10b84d8cd4e9867f416ecb78852e67039b009b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Mar 2020 23:40:40 +0100 Subject: [PATCH 23/24] worker: Fix build --- cmd/lotus-seal-worker/main.go | 20 +++++--------------- cmd/lotus-seal-worker/sub.go | 3 ++- cmd/lotus-seal-worker/transfer.go | 3 ++- 3 files changed, 9 insertions(+), 17 deletions(-) diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index 206430050..a0fd0abc2 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -2,21 +2,11 @@ package main import ( "os" - "sync" - - paramfetch "github.com/filecoin-project/go-paramfetch" - "github.com/filecoin-project/go-sectorbuilder" - "github.com/mitchellh/go-homedir" logging "github.com/ipfs/go-log/v2" - "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" - manet "github.com/multiformats/go-multiaddr-net" - - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/lotuslog" "github.com/filecoin-project/lotus/node/repo" ) @@ -85,7 +75,7 @@ var runCmd = &cli.Command{ Name: "run", Usage: "Start lotus worker", Action: func(cctx *cli.Context) error { - if !cctx.Bool("enable-gpu-proving") { + /* if !cctx.Bool("enable-gpu-proving") { os.Setenv("BELLMAN_NO_GPU", "true") } @@ -138,12 +128,12 @@ var runCmd = &cli.Command{ return xerrors.Errorf("get params: %w", err) } - ppt, spt, err := api.ProofTypeFromSectorSize(ssize) + /*ppt, spt, err := api.ProofTypeFromSectorSize(ssize) if err != nil { return err } - sb, err := sectorbuilder.NewStandalone(§orbuilder.Config{ + /*sb, err := sectorbuilder.NewStandalone(§orbuilder.Config{ SealProofType: spt, PoStProofType: ppt, Miner: act, @@ -162,14 +152,14 @@ var runCmd = &cli.Command{ go func() { defer wg.Done() - if err := acceptJobs(ctx, nodeApi, sb, limiter, "http://"+storageAddr, ainfo.AuthHeader(), r, cctx.Bool("no-precommit"), cctx.Bool("no-commit")); err != nil { + /* if err := acceptJobs(ctx, nodeApi, sb, limiter, "http://"+storageAddr, ainfo.AuthHeader(), r, cctx.Bool("no-precommit"), cctx.Bool("no-commit")); err != nil { log.Warnf("%+v", err) return } }() } - wg.Wait() + wg.Wait()*/ return nil }, } diff --git a/cmd/lotus-seal-worker/sub.go b/cmd/lotus-seal-worker/sub.go index 8c5560d1d..31f28786b 100644 --- a/cmd/lotus-seal-worker/sub.go +++ b/cmd/lotus-seal-worker/sub.go @@ -1,5 +1,5 @@ package main - +/* import ( "context" "net/http" @@ -128,3 +128,4 @@ func (w *worker) processTask(ctx context.Context, task sectorbuilder.WorkerTask) func errRes(err error) sectorbuilder.SealRes { return sectorbuilder.SealRes{Err: err.Error(), GoErr: err} } +*/ \ No newline at end of file diff --git a/cmd/lotus-seal-worker/transfer.go b/cmd/lotus-seal-worker/transfer.go index 683071b5f..12505ae44 100644 --- a/cmd/lotus-seal-worker/transfer.go +++ b/cmd/lotus-seal-worker/transfer.go @@ -1,5 +1,5 @@ package main - +/* import ( "fmt" "io" @@ -174,3 +174,4 @@ func (w *worker) fetchSector(sectorID abi.SectorNumber, typ sectorbuilder.Worker } return nil } +*/ \ No newline at end of file From a5c6d83cd9c0319b4f6427dcb764e74a4893bc2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 5 Mar 2020 23:43:41 +0100 Subject: [PATCH 24/24] gofmt; mod tidy --- cmd/lotus-seal-worker/main.go | 154 +++++++++++++++--------------- cmd/lotus-seal-worker/sub.go | 3 +- cmd/lotus-seal-worker/transfer.go | 3 +- go.mod | 1 - 4 files changed, 81 insertions(+), 80 deletions(-) diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index a0fd0abc2..efa4aa015 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -75,91 +75,91 @@ var runCmd = &cli.Command{ Name: "run", Usage: "Start lotus worker", Action: func(cctx *cli.Context) error { - /* if !cctx.Bool("enable-gpu-proving") { - os.Setenv("BELLMAN_NO_GPU", "true") - } + /* if !cctx.Bool("enable-gpu-proving") { + os.Setenv("BELLMAN_NO_GPU", "true") + } - nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return xerrors.Errorf("getting miner api: %w", err) - } - defer closer() - ctx := lcli.ReqContext(cctx) + nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return xerrors.Errorf("getting miner api: %w", err) + } + defer closer() + ctx := lcli.ReqContext(cctx) - ainfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) - if err != nil { - return xerrors.Errorf("could not get api info: %w", err) - } - _, storageAddr, err := manet.DialArgs(ainfo.Addr) + ainfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) + if err != nil { + return xerrors.Errorf("could not get api info: %w", err) + } + _, storageAddr, err := manet.DialArgs(ainfo.Addr) - r, err := homedir.Expand(cctx.String("repo")) - if err != nil { - return err - } + r, err := homedir.Expand(cctx.String("repo")) + if err != nil { + return err + } - v, err := nodeApi.Version(ctx) - if err != nil { - return err - } - if v.APIVersion != build.APIVersion { - return xerrors.Errorf("lotus-storage-miner API version doesn't match: local: ", api.Version{APIVersion: build.APIVersion}) - } + v, err := nodeApi.Version(ctx) + if err != nil { + return err + } + if v.APIVersion != build.APIVersion { + return xerrors.Errorf("lotus-storage-miner API version doesn't match: local: ", api.Version{APIVersion: build.APIVersion}) + } - go func() { - <-ctx.Done() - log.Warn("Shutting down..") - }() - - limiter := &limits{ - workLimit: make(chan struct{}, workers), - transferLimit: make(chan struct{}, transfers), - } - - act, err := nodeApi.ActorAddress(ctx) - if err != nil { - return err - } - ssize, err := nodeApi.ActorSectorSize(ctx, act) - if err != nil { - return err - } - - if err := paramfetch.GetParams(build.ParametersJson(), uint64(ssize)); err != nil { - return xerrors.Errorf("get params: %w", err) - } - - /*ppt, spt, err := api.ProofTypeFromSectorSize(ssize) - if err != nil { - return err - } - - /*sb, err := sectorbuilder.NewStandalone(§orbuilder.Config{ - SealProofType: spt, - PoStProofType: ppt, - Miner: act, - WorkerThreads: workers, - Paths: sectorbuilder.SimplePath(r), - }) - if err != nil { - return err - } - - nQueues := workers + transfers - var wg sync.WaitGroup - wg.Add(nQueues) - - for i := 0; i < nQueues; i++ { go func() { - defer wg.Done() - - /* if err := acceptJobs(ctx, nodeApi, sb, limiter, "http://"+storageAddr, ainfo.AuthHeader(), r, cctx.Bool("no-precommit"), cctx.Bool("no-commit")); err != nil { - log.Warnf("%+v", err) - return - } + <-ctx.Done() + log.Warn("Shutting down..") }() - } - wg.Wait()*/ + limiter := &limits{ + workLimit: make(chan struct{}, workers), + transferLimit: make(chan struct{}, transfers), + } + + act, err := nodeApi.ActorAddress(ctx) + if err != nil { + return err + } + ssize, err := nodeApi.ActorSectorSize(ctx, act) + if err != nil { + return err + } + + if err := paramfetch.GetParams(build.ParametersJson(), uint64(ssize)); err != nil { + return xerrors.Errorf("get params: %w", err) + } + + /*ppt, spt, err := api.ProofTypeFromSectorSize(ssize) + if err != nil { + return err + } + + /*sb, err := sectorbuilder.NewStandalone(§orbuilder.Config{ + SealProofType: spt, + PoStProofType: ppt, + Miner: act, + WorkerThreads: workers, + Paths: sectorbuilder.SimplePath(r), + }) + if err != nil { + return err + } + + nQueues := workers + transfers + var wg sync.WaitGroup + wg.Add(nQueues) + + for i := 0; i < nQueues; i++ { + go func() { + defer wg.Done() + + /* if err := acceptJobs(ctx, nodeApi, sb, limiter, "http://"+storageAddr, ainfo.AuthHeader(), r, cctx.Bool("no-precommit"), cctx.Bool("no-commit")); err != nil { + log.Warnf("%+v", err) + return + } + }() + } + + wg.Wait()*/ return nil }, } diff --git a/cmd/lotus-seal-worker/sub.go b/cmd/lotus-seal-worker/sub.go index 31f28786b..d11df2acc 100644 --- a/cmd/lotus-seal-worker/sub.go +++ b/cmd/lotus-seal-worker/sub.go @@ -1,4 +1,5 @@ package main + /* import ( "context" @@ -128,4 +129,4 @@ func (w *worker) processTask(ctx context.Context, task sectorbuilder.WorkerTask) func errRes(err error) sectorbuilder.SealRes { return sectorbuilder.SealRes{Err: err.Error(), GoErr: err} } -*/ \ No newline at end of file +*/ diff --git a/cmd/lotus-seal-worker/transfer.go b/cmd/lotus-seal-worker/transfer.go index 12505ae44..fe4a07dba 100644 --- a/cmd/lotus-seal-worker/transfer.go +++ b/cmd/lotus-seal-worker/transfer.go @@ -1,4 +1,5 @@ package main + /* import ( "fmt" @@ -174,4 +175,4 @@ func (w *worker) fetchSector(sectorID abi.SectorNumber, typ sectorbuilder.Worker } return nil } -*/ \ No newline at end of file +*/ diff --git a/go.mod b/go.mod index 7e3a60ac1..482296b7b 100644 --- a/go.mod +++ b/go.mod @@ -105,7 +105,6 @@ require ( golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 golang.org/x/tools v0.0.0-20200108195415-316d2f248479 // indirect golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 - gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 gotest.tools v2.2.0+incompatible launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect