From e3edab66e3544e9c9ebbf87f9a7653991c901608 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 10 Feb 2022 17:33:38 +0100 Subject: [PATCH 01/13] refactor RepoType --- chain/gen/gen.go | 2 +- chain/store/store_test.go | 2 +- cli/auth.go | 2 +- cli/client_retr.go | 4 +- cli/pprof.go | 2 +- cli/util/api.go | 46 +++--- cmd/lotus-miner/actor_test.go | 2 +- cmd/lotus-miner/allinfo_test.go | 2 +- cmd/lotus-miner/backup.go | 2 +- cmd/lotus-miner/config.go | 2 +- cmd/lotus-miner/init.go | 6 +- cmd/lotus-miner/init_restore.go | 4 +- cmd/lotus-miner/main.go | 6 +- cmd/lotus-miner/run.go | 2 +- cmd/lotus-seal-worker/main.go | 12 +- cmd/lotus-shed/balances.go | 4 +- cmd/lotus-shed/datastore.go | 16 +-- cmd/lotus-shed/export-car.go | 2 +- cmd/lotus-shed/export.go | 2 +- cmd/lotus-shed/import-car.go | 4 +- cmd/lotus-shed/keyinfo.go | 2 +- cmd/lotus-shed/market.go | 2 +- cmd/lotus-shed/migrations.go | 2 +- cmd/lotus-shed/miner-peerid.go | 2 +- cmd/lotus-shed/miner-types.go | 2 +- cmd/lotus-shed/pruning.go | 2 +- cmd/lotus-shed/rpc.go | 8 +- cmd/lotus-shed/splitstore.go | 4 +- cmd/lotus-shed/terminations.go | 2 +- cmd/lotus-sim/simulation/node.go | 2 +- cmd/lotus-wallet/main.go | 4 +- cmd/lotus/backup.go | 4 +- cmd/lotus/config.go | 2 +- cmd/lotus/daemon.go | 4 +- cmd/lotus/main.go | 2 +- documentation/en/architecture/architecture.md | 6 +- extern/sector-storage/stores/remote_test.go | 4 +- itests/deals_concurrent_test.go | 2 +- itests/kit/ensemble.go | 2 +- itests/sector_finalize_early_test.go | 2 +- itests/sector_miner_collateral_test.go | 2 +- node/builder.go | 18 ++- node/builder_chain.go | 2 +- node/builder_miner.go | 6 +- node/modules/storageminer_dagstore.go | 132 ++++++++---------- node/repo/fsrepo.go | 110 ++++++++++----- node/repo/fsrepo_test.go | 2 +- node/repo/memrepo.go | 2 +- node/repo/repo_test.go | 8 +- testplans/lotus-soup/testkit/role_miner.go | 6 +- 50 files changed, 248 insertions(+), 224 deletions(-) diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 4153830dc..51289cdf8 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -121,7 +121,7 @@ func NewGeneratorWithSectorsAndUpgradeSchedule(numSectors int, us stmgr.UpgradeS policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) mr := repo.NewMemory(nil) - lr, err := mr.Lock(repo.StorageMiner) + lr, err := mr.Lock(repo.StorageMinerRepoType{}) if err != nil { return nil, xerrors.Errorf("taking mem-repo lock failed: %w", err) } diff --git a/chain/store/store_test.go b/chain/store/store_test.go index a759a48a8..d75917017 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -48,7 +48,7 @@ func BenchmarkGetRandomness(b *testing.B) { b.Fatal(err) } - lr, err := r.Lock(repo.FullNode) + lr, err := r.Lock(repo.FullNodeRepoType{}) if err != nil { b.Fatal(err) } diff --git a/cli/auth.go b/cli/auth.go index e6eeace89..f5a29a41e 100644 --- a/cli/auth.go +++ b/cli/auth.go @@ -114,7 +114,7 @@ var AuthApiInfoToken = &cli.Command{ ti, ok := cctx.App.Metadata["repoType"] if !ok { log.Errorf("unknown repo type, are you sure you want to use GetCommonAPI?") - ti = repo.FullNode + ti = repo.FullNodeRepoType{} } t, ok := ti.(repo.RepoType) if !ok { diff --git a/cli/client_retr.go b/cli/client_retr.go index 9b195a5d8..1b7061ad9 100644 --- a/cli/client_retr.go +++ b/cli/client_retr.go @@ -403,7 +403,7 @@ var clientRetrieveCatCmd = &cli.Command{ return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments")) } - ainfo, err := GetAPIInfo(cctx, repo.FullNode) + ainfo, err := GetAPIInfo(cctx, repo.FullNodeRepoType{}) if err != nil { return xerrors.Errorf("could not get API info: %w", err) } @@ -482,7 +482,7 @@ var clientRetrieveLsCmd = &cli.Command{ return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments")) } - ainfo, err := GetAPIInfo(cctx, repo.FullNode) + ainfo, err := GetAPIInfo(cctx, repo.FullNodeRepoType{}) if err != nil { return xerrors.Errorf("could not get API info: %w", err) } diff --git a/cli/pprof.go b/cli/pprof.go index ae4016e11..81eb03448 100644 --- a/cli/pprof.go +++ b/cli/pprof.go @@ -26,7 +26,7 @@ var PprofGoroutines = &cli.Command{ ti, ok := cctx.App.Metadata["repoType"] if !ok { log.Errorf("unknown repo type, are you sure you want to use GetAPI?") - ti = repo.FullNode + ti = repo.FullNodeRepoType{} } t, ok := ti.(repo.RepoType) if !ok { diff --git a/cli/util/api.go b/cli/util/api.go index d87817bb3..e8bf4af2d 100644 --- a/cli/util/api.go +++ b/cli/util/api.go @@ -32,14 +32,14 @@ const ( // of the API server (only used by the tests), in the order of precedence they // should be applied for the requested kind of node. func flagsForAPI(t repo.RepoType) []string { - switch t { - case repo.FullNode: + switch t.Type() { + case "FullNode": return []string{"api-url"} - case repo.StorageMiner: + case "StorageMiner": return []string{"miner-api-url"} - case repo.Worker: + case "Worker": return []string{"worker-api-url"} - case repo.Markets: + case "Markets": // support split markets-miner and monolith deployments. return []string{"markets-api-url", "miner-api-url"} default: @@ -48,14 +48,14 @@ func flagsForAPI(t repo.RepoType) []string { } func flagsForRepo(t repo.RepoType) []string { - switch t { - case repo.FullNode: + switch t.Type() { + case "FullNode": return []string{"repo"} - case repo.StorageMiner: + case "StorageMiner": return []string{"miner-repo"} - case repo.Worker: + case "Worker": return []string{"worker-repo"} - case repo.Markets: + case "Markets": // support split markets-miner and monolith deployments. return []string{"markets-repo", "miner-repo"} default: @@ -69,15 +69,15 @@ func flagsForRepo(t repo.RepoType) []string { // It returns the current variables and deprecated ones separately, so that // the user can log a warning when deprecated ones are found to be in use. func EnvsForAPIInfos(t repo.RepoType) (primary string, fallbacks []string, deprecated []string) { - switch t { - case repo.FullNode: + switch t.Type() { + case "FullNode": return "FULLNODE_API_INFO", nil, nil - case repo.StorageMiner: + case "StorageMiner": // TODO remove deprecated deprecation period return "MINER_API_INFO", nil, []string{"STORAGE_API_INFO"} - case repo.Worker: + case "Worker": return "WORKER_API_INFO", nil, nil - case repo.Markets: + case "Markets": // support split markets-miner and monolith deployments. return "MARKETS_API_INFO", []string{"MINER_API_INFO"}, nil default: @@ -200,7 +200,7 @@ func GetCommonAPI(ctx *cli.Context) (api.CommonNet, jsonrpc.ClientCloser, error) ti, ok := ctx.App.Metadata["repoType"] if !ok { log.Errorf("unknown repo type, are you sure you want to use GetCommonAPI?") - ti = repo.FullNode + ti = repo.FullNodeRepoType{} } t, ok := ti.(repo.RepoType) if !ok { @@ -232,7 +232,7 @@ func GetFullNodeAPI(ctx *cli.Context) (v0api.FullNode, jsonrpc.ClientCloser, err return &v0api.WrapperV1Full{FullNode: tn.(v1api.FullNode)}, func() {}, nil } - addr, headers, err := GetRawAPI(ctx, repo.FullNode, "v0") + addr, headers, err := GetRawAPI(ctx, repo.FullNodeRepoType{}, "v0") if err != nil { return nil, nil, err } @@ -249,7 +249,7 @@ func GetFullNodeAPIV1(ctx *cli.Context) (v1api.FullNode, jsonrpc.ClientCloser, e return tn.(v1api.FullNode), func() {}, nil } - addr, headers, err := GetRawAPI(ctx, repo.FullNode, "v1") + addr, headers, err := GetRawAPI(ctx, repo.FullNodeRepoType{}, "v1") if err != nil { return nil, nil, err } @@ -293,7 +293,7 @@ func GetStorageMinerAPI(ctx *cli.Context, opts ...GetStorageMinerOption) (api.St return tn.(api.StorageMiner), func() {}, nil } - addr, headers, err := GetRawAPI(ctx, repo.StorageMiner, "v0") + addr, headers, err := GetRawAPI(ctx, repo.StorageMinerRepoType{}, "v0") if err != nil { return nil, nil, err } @@ -322,7 +322,7 @@ func GetStorageMinerAPI(ctx *cli.Context, opts ...GetStorageMinerOption) (api.St } func GetWorkerAPI(ctx *cli.Context) (api.Worker, jsonrpc.ClientCloser, error) { - addr, headers, err := GetRawAPI(ctx, repo.Worker, "v0") + addr, headers, err := GetRawAPI(ctx, repo.WorkerRepoType{}, "v0") if err != nil { return nil, nil, err } @@ -340,7 +340,7 @@ func GetMarketsAPI(ctx *cli.Context) (api.StorageMiner, jsonrpc.ClientCloser, er return tn.(api.StorageMiner), func() {}, nil } - addr, headers, err := GetRawAPI(ctx, repo.Markets, "v0") + addr, headers, err := GetRawAPI(ctx, repo.MarketsRepoType{}, "v0") if err != nil { return nil, nil, err } @@ -356,7 +356,7 @@ func GetMarketsAPI(ctx *cli.Context) (api.StorageMiner, jsonrpc.ClientCloser, er } func GetGatewayAPI(ctx *cli.Context) (api.Gateway, jsonrpc.ClientCloser, error) { - addr, headers, err := GetRawAPI(ctx, repo.FullNode, "v1") + addr, headers, err := GetRawAPI(ctx, repo.FullNodeRepoType{}, "v1") if err != nil { return nil, nil, err } @@ -369,7 +369,7 @@ func GetGatewayAPI(ctx *cli.Context) (api.Gateway, jsonrpc.ClientCloser, error) } func GetGatewayAPIV0(ctx *cli.Context) (v0api.Gateway, jsonrpc.ClientCloser, error) { - addr, headers, err := GetRawAPI(ctx, repo.FullNode, "v0") + addr, headers, err := GetRawAPI(ctx, repo.FullNodeRepoType{}, "v0") if err != nil { return nil, nil, err } diff --git a/cmd/lotus-miner/actor_test.go b/cmd/lotus-miner/actor_test.go index 5650a9ac5..d9f7297b9 100644 --- a/cmd/lotus-miner/actor_test.go +++ b/cmd/lotus-miner/actor_test.go @@ -40,7 +40,7 @@ func TestWorkerKeyChange(t *testing.T) { run := func(cmd *cli.Command, args ...string) error { app := cli.NewApp() app.Metadata = map[string]interface{}{ - "repoType": repo.StorageMiner, + "repoType": repo.StorageMinerRepoType{}, "testnode-full": client1, "testnode-storage": miner, } diff --git a/cmd/lotus-miner/allinfo_test.go b/cmd/lotus-miner/allinfo_test.go index 5f30b4fec..c85e5282c 100644 --- a/cmd/lotus-miner/allinfo_test.go +++ b/cmd/lotus-miner/allinfo_test.go @@ -36,7 +36,7 @@ func TestMinerAllInfo(t *testing.T) { run := func(t *testing.T) { app := cli.NewApp() app.Metadata = map[string]interface{}{ - "repoType": repo.StorageMiner, + "repoType": repo.StorageMinerRepoType{}, "testnode-full": client, "testnode-storage": miner, } diff --git a/cmd/lotus-miner/backup.go b/cmd/lotus-miner/backup.go index cf8c9f912..8fe087561 100644 --- a/cmd/lotus-miner/backup.go +++ b/cmd/lotus-miner/backup.go @@ -9,6 +9,6 @@ import ( "github.com/filecoin-project/lotus/node/repo" ) -var backupCmd = lcli.BackupCmd(FlagMinerRepo, repo.StorageMiner, func(cctx *cli.Context) (lcli.BackupAPI, jsonrpc.ClientCloser, error) { +var backupCmd = lcli.BackupCmd(FlagMinerRepo, repo.StorageMinerRepoType{}, func(cctx *cli.Context) (lcli.BackupAPI, jsonrpc.ClientCloser, error) { return lcli.GetStorageMinerAPI(cctx) }) diff --git a/cmd/lotus-miner/config.go b/cmd/lotus-miner/config.go index 652426583..42bb4fdef 100644 --- a/cmd/lotus-miner/config.go +++ b/cmd/lotus-miner/config.go @@ -66,7 +66,7 @@ var configUpdateCmd = &cli.Command{ return xerrors.Errorf("repo not initialized") } - lr, err := r.LockRO(repo.StorageMiner) + lr, err := r.LockRO(repo.StorageMinerRepoType{}) if err != nil { return xerrors.Errorf("locking repo: %w", err) } diff --git a/cmd/lotus-miner/init.go b/cmd/lotus-miner/init.go index 59ea75b10..30a2d6844 100644 --- a/cmd/lotus-miner/init.go +++ b/cmd/lotus-miner/init.go @@ -203,12 +203,12 @@ var initCmd = &cli.Command{ log.Info("Initializing repo") - if err := r.Init(repo.StorageMiner); err != nil { + if err := r.Init(repo.StorageMinerRepoType{}); err != nil { return err } { - lr, err := r.Lock(repo.StorageMiner) + lr, err := r.Lock(repo.StorageMinerRepoType{}) if err != nil { return err } @@ -410,7 +410,7 @@ func findMarketDealID(ctx context.Context, api v1api.FullNode, deal market2.Deal } func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode, r repo.Repo, ssize abi.SectorSize, gasPrice types.BigInt) error { - lr, err := r.Lock(repo.StorageMiner) + lr, err := r.Lock(repo.StorageMinerRepoType{}) if err != nil { return err } diff --git a/cmd/lotus-miner/init_restore.go b/cmd/lotus-miner/init_restore.go index 1aaa7909a..077ac8011 100644 --- a/cmd/lotus-miner/init_restore.go +++ b/cmd/lotus-miner/init_restore.go @@ -159,11 +159,11 @@ func restore(ctx context.Context, cctx *cli.Context, targetPath string, strConfi log.Info("Initializing repo") - if err := r.Init(repo.StorageMiner); err != nil { + if err := r.Init(repo.StorageMinerRepoType{}); err != nil { return err } - lr, err := r.Lock(repo.StorageMiner) + lr, err := r.Lock(repo.StorageMinerRepoType{}) if err != nil { return err } diff --git a/cmd/lotus-miner/main.go b/cmd/lotus-miner/main.go index 57b5d8a3e..01a931a5f 100644 --- a/cmd/lotus-miner/main.go +++ b/cmd/lotus-miner/main.go @@ -93,7 +93,7 @@ func main() { return err } } - c.App.Metadata["repoType"] = repo.Markets + c.App.Metadata["repoType"] = repo.MarketsRepoType{} return nil } @@ -150,7 +150,7 @@ func main() { // this command is explicitly called on markets, inform // common commands by overriding the repoType. if c.Bool("call-on-markets") { - c.App.Metadata["repoType"] = repo.Markets + c.App.Metadata["repoType"] = repo.MarketsRepoType{} } return nil }, @@ -164,7 +164,7 @@ func main() { }, } app.Setup() - app.Metadata["repoType"] = repo.StorageMiner + app.Metadata["repoType"] = repo.StorageMinerRepoType{} lcli.RunApp(app) } diff --git a/cmd/lotus-miner/run.go b/cmd/lotus-miner/run.go index f276f319c..f5a138fba 100644 --- a/cmd/lotus-miner/run.go +++ b/cmd/lotus-miner/run.go @@ -119,7 +119,7 @@ var runCmd = &cli.Command{ return xerrors.Errorf("repo at '%s' is not initialized, run 'lotus-miner init' to set it up", minerRepoPath) } - lr, err := r.Lock(repo.StorageMiner) + lr, err := r.Lock(repo.StorageMinerRepoType{}) if err != nil { return err } diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index 9e6843dbf..a1812cc36 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -108,7 +108,7 @@ func main() { Commands: local, } app.Setup() - app.Metadata["repoType"] = repo.Worker + app.Metadata["repoType"] = repo.WorkerRepoType{} if err := app.Run(os.Args); err != nil { log.Warnf("%+v", err) @@ -310,11 +310,11 @@ var runCmd = &cli.Command{ return err } if !ok { - if err := r.Init(repo.Worker); err != nil { + if err := r.Init(repo.WorkerRepoType{}); err != nil { return err } - lr, err := r.Lock(repo.Worker) + lr, err := r.Lock(repo.WorkerRepoType{}) if err != nil { return err } @@ -359,7 +359,7 @@ var runCmd = &cli.Command{ } } - lr, err := r.Lock(repo.Worker) + lr, err := r.Lock(repo.WorkerRepoType{}) if err != nil { return err } @@ -397,7 +397,7 @@ var runCmd = &cli.Command{ } // Setup remote sector store - sminfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) + sminfo, err := lcli.GetAPIInfo(cctx, repo.StorageMinerRepoType{}) if err != nil { return xerrors.Errorf("could not get api info: %w", err) } @@ -484,7 +484,7 @@ var runCmd = &cli.Command{ return xerrors.Errorf("setting api endpoint: %w", err) } - ainfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) + ainfo, err := lcli.GetAPIInfo(cctx, repo.StorageMinerRepoType{}) if err != nil { return xerrors.Errorf("could not get miner API info: %w", err) } diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index 1a22be3c3..2550741e0 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -486,7 +486,7 @@ var chainBalanceStateCmd = &cli.Command{ return err } - lkrepo, err := fsrepo.Lock(repo.FullNode) + lkrepo, err := fsrepo.Lock(repo.FullNodeRepoType{}) if err != nil { return err } @@ -710,7 +710,7 @@ var chainPledgeCmd = &cli.Command{ return err } - lkrepo, err := fsrepo.Lock(repo.FullNode) + lkrepo, err := fsrepo.Lock(repo.FullNodeRepoType{}) if err != nil { return err } diff --git a/cmd/lotus-shed/datastore.go b/cmd/lotus-shed/datastore.go index ff740a772..5c5567654 100644 --- a/cmd/lotus-shed/datastore.go +++ b/cmd/lotus-shed/datastore.go @@ -39,10 +39,10 @@ var datastoreListCmd = &cli.Command{ Name: "list", Description: "list datastore keys", Flags: []cli.Flag{ - &cli.IntFlag{ + &cli.StringFlag{ Name: "repo-type", - Usage: "node type (1 - full, 2 - storage, 3 - worker)", - Value: 1, + Usage: "node type (FullNode, StorageMiner, Worker, Wallet)", + Value: "FullNode", }, &cli.BoolFlag{ Name: "top-level", @@ -70,7 +70,7 @@ var datastoreListCmd = &cli.Command{ return xerrors.Errorf("lotus repo doesn't exist") } - lr, err := r.Lock(repo.RepoType(cctx.Int("repo-type"))) + lr, err := r.Lock(repo.NewRepoTypeFromString(cctx.String("repo-type"))) if err != nil { return err } @@ -108,10 +108,10 @@ var datastoreGetCmd = &cli.Command{ Name: "get", Description: "list datastore keys", Flags: []cli.Flag{ - &cli.IntFlag{ + &cli.StringFlag{ Name: "repo-type", - Usage: "node type (1 - full, 2 - storage, 3 - worker)", - Value: 1, + Usage: "node type (FullNode, StorageMiner, Worker, Wallet)", + Value: "FullNode", }, &cli.StringFlag{ Name: "enc", @@ -136,7 +136,7 @@ var datastoreGetCmd = &cli.Command{ return xerrors.Errorf("lotus repo doesn't exist") } - lr, err := r.Lock(repo.RepoType(cctx.Int("repo-type"))) + lr, err := r.Lock(repo.NewRepoTypeFromString(cctx.String("repo-type"))) if err != nil { return err } diff --git a/cmd/lotus-shed/export-car.go b/cmd/lotus-shed/export-car.go index 97e4fb6c6..0d00e3480 100644 --- a/cmd/lotus-shed/export-car.go +++ b/cmd/lotus-shed/export-car.go @@ -67,7 +67,7 @@ var exportCarCmd = &cli.Command{ return xerrors.Errorf("lotus repo doesn't exist") } - lr, err := r.Lock(repo.FullNode) + lr, err := r.Lock(repo.FullNodeRepoType{}) if err != nil { return err } diff --git a/cmd/lotus-shed/export.go b/cmd/lotus-shed/export.go index 3851e4922..39614a8a7 100644 --- a/cmd/lotus-shed/export.go +++ b/cmd/lotus-shed/export.go @@ -59,7 +59,7 @@ var exportChainCmd = &cli.Command{ return xerrors.Errorf("lotus repo doesn't exist") } - lr, err := r.Lock(repo.FullNode) + lr, err := r.Lock(repo.FullNodeRepoType{}) if err != nil { return err } diff --git a/cmd/lotus-shed/import-car.go b/cmd/lotus-shed/import-car.go index 973e7b31b..b05deb2c5 100644 --- a/cmd/lotus-shed/import-car.go +++ b/cmd/lotus-shed/import-car.go @@ -35,7 +35,7 @@ var importCarCmd = &cli.Command{ return xerrors.Errorf("lotus repo doesn't exist") } - lr, err := r.Lock(repo.FullNode) + lr, err := r.Lock(repo.FullNodeRepoType{}) if err != nil { return err } @@ -112,7 +112,7 @@ var importObjectCmd = &cli.Command{ return xerrors.Errorf("lotus repo doesn't exist") } - lr, err := r.Lock(repo.FullNode) + lr, err := r.Lock(repo.FullNodeRepoType{}) if err != nil { return err } diff --git a/cmd/lotus-shed/keyinfo.go b/cmd/lotus-shed/keyinfo.go index 3c99b5050..aec19d8c7 100644 --- a/cmd/lotus-shed/keyinfo.go +++ b/cmd/lotus-shed/keyinfo.go @@ -183,7 +183,7 @@ var keyinfoImportCmd = &cli.Command{ return err } - lkrepo, err := fsrepo.Lock(repo.FullNode) + lkrepo, err := fsrepo.Lock(repo.FullNodeRepoType{}) if err != nil { return err } diff --git a/cmd/lotus-shed/market.go b/cmd/lotus-shed/market.go index aaef4690e..924b3441c 100644 --- a/cmd/lotus-shed/market.go +++ b/cmd/lotus-shed/market.go @@ -301,7 +301,7 @@ func openLockedRepo(path string) (repo.LockedRepo, error) { } // Lock the repo - lr, err := rpo.Lock(repo.StorageMiner) + lr, err := rpo.Lock(repo.StorageMinerRepoType{}) if err != nil { return nil, xerrors.Errorf("locking repo %s: %w", path, err) } diff --git a/cmd/lotus-shed/migrations.go b/cmd/lotus-shed/migrations.go index 85987c658..c1a8f7b5e 100644 --- a/cmd/lotus-shed/migrations.go +++ b/cmd/lotus-shed/migrations.go @@ -47,7 +47,7 @@ var migrationsCmd = &cli.Command{ return err } - lkrepo, err := fsrepo.Lock(repo.FullNode) + lkrepo, err := fsrepo.Lock(repo.FullNodeRepoType{}) if err != nil { return err } diff --git a/cmd/lotus-shed/miner-peerid.go b/cmd/lotus-shed/miner-peerid.go index 3ccfb429b..c02a226be 100644 --- a/cmd/lotus-shed/miner-peerid.go +++ b/cmd/lotus-shed/miner-peerid.go @@ -54,7 +54,7 @@ var minerPeeridCmd = &cli.Command{ return err } - lkrepo, err := fsrepo.Lock(repo.FullNode) + lkrepo, err := fsrepo.Lock(repo.FullNodeRepoType{}) if err != nil { return err } diff --git a/cmd/lotus-shed/miner-types.go b/cmd/lotus-shed/miner-types.go index 05ef7b0a7..f0655096c 100644 --- a/cmd/lotus-shed/miner-types.go +++ b/cmd/lotus-shed/miner-types.go @@ -50,7 +50,7 @@ var minerTypesCmd = &cli.Command{ return err } - lkrepo, err := fsrepo.Lock(repo.FullNode) + lkrepo, err := fsrepo.Lock(repo.FullNodeRepoType{}) if err != nil { return err } diff --git a/cmd/lotus-shed/pruning.go b/cmd/lotus-shed/pruning.go index 164ff197a..184fea85b 100644 --- a/cmd/lotus-shed/pruning.go +++ b/cmd/lotus-shed/pruning.go @@ -123,7 +123,7 @@ var stateTreePruneCmd = &cli.Command{ return err } - lkrepo, err := fsrepo.Lock(repo.FullNode) + lkrepo, err := fsrepo.Lock(repo.FullNodeRepoType{}) if err != nil { return err } diff --git a/cmd/lotus-shed/rpc.go b/cmd/lotus-shed/rpc.go index 81171916e..0c870f090 100644 --- a/cmd/lotus-shed/rpc.go +++ b/cmd/lotus-shed/rpc.go @@ -34,9 +34,11 @@ var rpcCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { - rt := repo.FullNode + var rt repo.RepoType if cctx.Bool("miner") { - rt = repo.StorageMiner + rt = repo.StorageMinerRepoType{} + } else { + rt = repo.FullNodeRepoType{} } addr, headers, err := lcli.GetRawAPI(cctx, rt, cctx.String("version")) @@ -123,7 +125,7 @@ var rpcCmd = &cli.Command{ return send(cctx.Args().Get(0), params) } - cctx.App.Metadata["repoType"] = repo.FullNode + cctx.App.Metadata["repoType"] = repo.FullNodeRepoType{} if err := lcli.VersionCmd.Action(cctx); err != nil { return err } diff --git a/cmd/lotus-shed/splitstore.go b/cmd/lotus-shed/splitstore.go index 58563955f..a907b3135 100644 --- a/cmd/lotus-shed/splitstore.go +++ b/cmd/lotus-shed/splitstore.go @@ -67,7 +67,7 @@ var splitstoreRollbackCmd = &cli.Command{ return xerrors.Errorf("lotus repo doesn't exist") } - lr, err := r.Lock(repo.FullNode) + lr, err := r.Lock(repo.FullNodeRepoType{}) if err != nil { return xerrors.Errorf("error locking repo: %w", err) } @@ -153,7 +153,7 @@ var splitstoreClearCmd = &cli.Command{ return xerrors.Errorf("lotus repo doesn't exist") } - lr, err := r.Lock(repo.FullNode) + lr, err := r.Lock(repo.FullNodeRepoType{}) if err != nil { return xerrors.Errorf("error locking repo: %w", err) } diff --git a/cmd/lotus-shed/terminations.go b/cmd/lotus-shed/terminations.go index 0691f35da..8edf4e131 100644 --- a/cmd/lotus-shed/terminations.go +++ b/cmd/lotus-shed/terminations.go @@ -53,7 +53,7 @@ var terminationsCmd = &cli.Command{ return err } - lkrepo, err := fsrepo.Lock(repo.FullNode) + lkrepo, err := fsrepo.Lock(repo.FullNodeRepoType{}) if err != nil { return err } diff --git a/cmd/lotus-sim/simulation/node.go b/cmd/lotus-sim/simulation/node.go index da027ff4f..2a9769381 100644 --- a/cmd/lotus-sim/simulation/node.go +++ b/cmd/lotus-sim/simulation/node.go @@ -41,7 +41,7 @@ func OpenNode(ctx context.Context, path string) (*Node, error) { // NewNode constructs a new node from the given repo. func NewNode(ctx context.Context, r repo.Repo) (nd *Node, _err error) { - lr, err := r.Lock(repo.FullNode) + lr, err := r.Lock(repo.FullNodeRepoType{}) if err != nil { return nil, err } diff --git a/cmd/lotus-wallet/main.go b/cmd/lotus-wallet/main.go index 91f23d092..21563a5b4 100644 --- a/cmd/lotus-wallet/main.go +++ b/cmd/lotus-wallet/main.go @@ -278,12 +278,12 @@ func openRepo(cctx *cli.Context) (repo.LockedRepo, types.KeyStore, error) { return nil, nil, err } if !ok { - if err := r.Init(repo.Worker); err != nil { + if err := r.Init(repo.WorkerRepoType{}); err != nil { return nil, nil, err } } - lr, err := r.Lock(repo.Wallet) + lr, err := r.Lock(repo.WalletRepoType{}) if err != nil { return nil, nil, err } diff --git a/cmd/lotus/backup.go b/cmd/lotus/backup.go index 4bdd21322..7403d75c0 100644 --- a/cmd/lotus/backup.go +++ b/cmd/lotus/backup.go @@ -18,7 +18,7 @@ import ( "github.com/filecoin-project/lotus/node/repo" ) -var backupCmd = lcli.BackupCmd("repo", repo.FullNode, func(cctx *cli.Context) (lcli.BackupAPI, jsonrpc.ClientCloser, error) { +var backupCmd = lcli.BackupCmd("repo", repo.FullNodeRepoType{}, func(cctx *cli.Context) (lcli.BackupAPI, jsonrpc.ClientCloser, error) { return lcli.GetFullNodeAPI(cctx) }) @@ -39,7 +39,7 @@ func restore(cctx *cli.Context, r repo.Repo) error { } defer f.Close() // nolint:errcheck - lr, err := r.Lock(repo.FullNode) + lr, err := r.Lock(repo.FullNodeRepoType{}) if err != nil { return err } diff --git a/cmd/lotus/config.go b/cmd/lotus/config.go index fcb7e2b08..3ca0683f7 100644 --- a/cmd/lotus/config.go +++ b/cmd/lotus/config.go @@ -66,7 +66,7 @@ var configUpdateCmd = &cli.Command{ return xerrors.Errorf("repo not initialized") } - lr, err := r.LockRO(repo.FullNode) + lr, err := r.LockRO(repo.FullNodeRepoType{}) if err != nil { return xerrors.Errorf("locking repo: %w", err) } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index f285ba74e..559713d18 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -227,7 +227,7 @@ var DaemonCmd = &cli.Command{ r.SetConfigPath(cctx.String("config")) } - err = r.Init(repo.FullNode) + err = r.Init(repo.FullNodeRepoType{}) if err != nil && err != repo.ErrRepoExists { return xerrors.Errorf("repo init error: %w", err) } @@ -462,7 +462,7 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool) l = st.Size() } - lr, err := r.Lock(repo.FullNode) + lr, err := r.Lock(repo.FullNodeRepoType{}) if err != nil { return err } diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go index c19b9fce4..eeb4192b4 100644 --- a/cmd/lotus/main.go +++ b/cmd/lotus/main.go @@ -106,7 +106,7 @@ func main() { app.Setup() app.Metadata["traceContext"] = ctx - app.Metadata["repoType"] = repo.FullNode + app.Metadata["repoType"] = repo.FullNodeRepoType{} lcli.RunApp(app) } diff --git a/documentation/en/architecture/architecture.md b/documentation/en/architecture/architecture.md index 64914d539..200b4a516 100644 --- a/documentation/en/architecture/architecture.md +++ b/documentation/en/architecture/architecture.md @@ -341,8 +341,8 @@ FIXME: This section needs to be clarified / corrected...I don't fully understand At the end of the `Repo()` function we see two mutually exclusive configuration calls based on the `RepoType` (`node/repo/fsrepo.go`). ```Go - ApplyIf(isType(repo.FullNode), ConfigFullNode(c)), - ApplyIf(isType(repo.StorageMiner), ConfigStorageMiner(c)), + ApplyIf(isType(repo.FullNodeRepoType{}), ConfigFullNode(c)), + ApplyIf(isType(repo.StorageMinerRepoType{}), ConfigStorageMiner(c)), ``` As we said, the repo fully identifies the node so a repo type is also a *node* type, in this case a full node or a miner. (FIXME: What is the difference between the two, does *full* imply miner?) In this case the `daemon` command will create a `FullNode`, this is specified in the command logic itself in `main.DaemonCmd()`, the `FsRepo` created (and passed to `node.Repo()`) will be initiated with that type (see `(*FsRepo).Init(t RepoType)`). @@ -352,7 +352,7 @@ FIXME: Much of this might need to be subsumed into the p2p section The `node.Online()` configuration function (`node/builder.go`) initializes components that involve connecting to, or interacting with, the Filecoin network. These connections are managed through the libp2p stack (FIXME link to this section when it exists). -We discuss some of the components found in the full node type (that is, included in the `ApplyIf(isType(repo.FullNode),` call). +We discuss some of the components found in the full node type (that is, included in the `ApplyIf(isType(repo.FullNodeRepoType{}),` call). #### Chainstore diff --git a/extern/sector-storage/stores/remote_test.go b/extern/sector-storage/stores/remote_test.go index a7a82a728..33e29820d 100644 --- a/extern/sector-storage/stores/remote_test.go +++ b/extern/sector-storage/stores/remote_test.go @@ -73,8 +73,8 @@ func TestMoveShared(t *testing.T) { openRepo := func(dir string) repo.LockedRepo { r, err := repo.NewFS(dir) require.NoError(t, err) - require.NoError(t, r.Init(repo.Worker)) - lr, err := r.Lock(repo.Worker) + require.NoError(t, r.Init(repo.WorkerRepoType{})) + lr, err := r.Lock(repo.WorkerRepoType{}) require.NoError(t, err) t.Cleanup(func() { diff --git a/itests/deals_concurrent_test.go b/itests/deals_concurrent_test.go index 18d8da02a..81b1444d9 100644 --- a/itests/deals_concurrent_test.go +++ b/itests/deals_concurrent_test.go @@ -146,7 +146,7 @@ func TestSimultanenousTransferLimit(t *testing.T) { ) runTest := func(t *testing.T) { client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs(), kit.ConstructorOpts( - node.ApplyIf(node.IsType(repo.StorageMiner), node.Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync(graphsyncThrottle, 0, graphsyncThrottle))), + node.ApplyIf(node.IsType(repo.StorageMinerRepoType{}), node.Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync(graphsyncThrottle, 0, graphsyncThrottle))), node.Override(new(dtypes.Graphsync), modules.Graphsync(graphsyncThrottle, graphsyncThrottle)), )) ens.InterconnectAll().BeginMining(250 * time.Millisecond) diff --git a/itests/kit/ensemble.go b/itests/kit/ensemble.go index 6a0158429..2594babca 100644 --- a/itests/kit/ensemble.go +++ b/itests/kit/ensemble.go @@ -443,7 +443,7 @@ func (n *Ensemble) Start() *Ensemble { r := repo.NewMemory(nil) - lr, err := r.Lock(repo.StorageMiner) + lr, err := r.Lock(repo.StorageMinerRepoType{}) require.NoError(n.t, err) c, err := lr.Config() diff --git a/itests/sector_finalize_early_test.go b/itests/sector_finalize_early_test.go index 233bc8fcb..78eec3fa7 100644 --- a/itests/sector_finalize_early_test.go +++ b/itests/sector_finalize_early_test.go @@ -35,7 +35,7 @@ func TestDealsWithFinalizeEarly(t *testing.T) { var blockTime = 50 * time.Millisecond client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.ConstructorOpts( - node.ApplyIf(node.IsType(repo.StorageMiner), node.Override(new(dtypes.GetSealingConfigFunc), func() (dtypes.GetSealingConfigFunc, error) { + node.ApplyIf(node.IsType(repo.StorageMinerRepoType{}), node.Override(new(dtypes.GetSealingConfigFunc), func() (dtypes.GetSealingConfigFunc, error) { return func() (sealiface.Config, error) { cf := config.DefaultStorageMiner() cf.Sealing.FinalizeEarly = true diff --git a/itests/sector_miner_collateral_test.go b/itests/sector_miner_collateral_test.go index af67b132b..5d05f3d2d 100644 --- a/itests/sector_miner_collateral_test.go +++ b/itests/sector_miner_collateral_test.go @@ -38,7 +38,7 @@ func TestMinerBalanceCollateral(t *testing.T) { defer cancel() opts := kit.ConstructorOpts( - node.ApplyIf(node.IsType(repo.StorageMiner), node.Override(new(dtypes.GetSealingConfigFunc), func() (dtypes.GetSealingConfigFunc, error) { + node.ApplyIf(node.IsType(repo.StorageMinerRepoType{}), node.Override(new(dtypes.GetSealingConfigFunc), func() (dtypes.GetSealingConfigFunc, error) { return func() (sealiface.Config, error) { return sealiface.Config{ MaxWaitDealsSectors: 4, diff --git a/node/builder.go b/node/builder.go index 6770e8dc8..152c32a73 100644 --- a/node/builder.go +++ b/node/builder.go @@ -224,12 +224,16 @@ var LibP2P = Options( ) func IsType(t repo.RepoType) func(s *Settings) bool { - return func(s *Settings) bool { return s.nodeType == t } + return func(s *Settings) bool { return s.nodeType.Type() == t.Type() } } -func isFullOrLiteNode(s *Settings) bool { return s.nodeType == repo.FullNode } -func isFullNode(s *Settings) bool { return s.nodeType == repo.FullNode && !s.Lite } -func isLiteNode(s *Settings) bool { return s.nodeType == repo.FullNode && s.Lite } +func isFullOrLiteNode(s *Settings) bool { return s.nodeType.Type() == repo.FullNodeRepoType{}.Type() } +func isFullNode(s *Settings) bool { + return s.nodeType.Type() == repo.FullNodeRepoType{}.Type() && !s.Lite +} +func isLiteNode(s *Settings) bool { + return s.nodeType.Type() == repo.FullNodeRepoType{}.Type() && s.Lite +} func Base() Option { return Options( @@ -241,7 +245,7 @@ func Base() Option { LibP2P, ), ApplyIf(isFullOrLiteNode, ChainNode), - ApplyIf(IsType(repo.StorageMiner), MinerNode), + ApplyIf(IsType(repo.StorageMinerRepoType{}), MinerNode), ) } @@ -314,8 +318,8 @@ func Repo(r repo.Repo) Option { Override(new(*dtypes.APIAlg), modules.APISecret), - ApplyIf(IsType(repo.FullNode), ConfigFullNode(c)), - ApplyIf(IsType(repo.StorageMiner), ConfigStorageMiner(c)), + ApplyIf(IsType(repo.FullNodeRepoType{}), ConfigFullNode(c)), + ApplyIf(IsType(repo.StorageMinerRepoType{}), ConfigStorageMiner(c)), )(settings) } } diff --git a/node/builder_chain.go b/node/builder_chain.go index 0d10dcb9b..e0d959bd6 100644 --- a/node/builder_chain.go +++ b/node/builder_chain.go @@ -248,7 +248,7 @@ func Lite(enable bool) FullOption { func FullAPI(out *api.FullNode, fopts ...FullOption) Option { return Options( func(s *Settings) error { - s.nodeType = repo.FullNode + s.nodeType = repo.FullNodeRepoType{} s.enableLibp2pNode = true return nil }, diff --git a/node/builder_miner.go b/node/builder_miner.go index e813a2d24..b689ab5e0 100644 --- a/node/builder_miner.go +++ b/node/builder_miner.go @@ -151,8 +151,8 @@ func ConfigStorageMiner(c interface{}) Option { Override(new(dtypes.RetrievalPricingFunc), modules.RetrievalPricingFunc(cfg.Dealmaking)), // DAG Store - Override(new(dagstore.MinerAPI), modules.NewMinerAPI), - Override(DAGStoreKey, modules.DAGStore), + Override(new(dagstore.MinerAPI), modules.NewMinerAPI(cfg.DAGStore)), + Override(DAGStoreKey, modules.DAGStore(cfg.DAGStore)), // Markets (retrieval) Override(new(dagstore.SectorAccessor), sectoraccessor.NewSectorAccessor), @@ -221,7 +221,7 @@ func StorageMiner(out *api.StorageMiner, subsystemsCfg config.MinerSubsystemConf ), func(s *Settings) error { - s.nodeType = repo.StorageMiner + s.nodeType = repo.StorageMinerRepoType{} s.enableLibp2pNode = subsystemsCfg.EnableMarkets return nil }, diff --git a/node/modules/storageminer_dagstore.go b/node/modules/storageminer_dagstore.go index 513acaad1..6905a2a98 100644 --- a/node/modules/storageminer_dagstore.go +++ b/node/modules/storageminer_dagstore.go @@ -23,88 +23,70 @@ const ( ) // NewMinerAPI creates a new MinerAPI adaptor for the dagstore mounts. -func NewMinerAPI(lc fx.Lifecycle, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, sa mdagstore.SectorAccessor) (mdagstore.MinerAPI, error) { - cfg, err := extractDAGStoreConfig(r) - if err != nil { - return nil, err - } - - // caps the amount of concurrent calls to the storage, so that we don't - // spam it during heavy processes like bulk migration. - if v, ok := os.LookupEnv("LOTUS_DAGSTORE_MOUNT_CONCURRENCY"); ok { - concurrency, err := strconv.Atoi(v) - if err == nil { - cfg.MaxConcurrencyStorageCalls = concurrency - } - } - - mountApi := mdagstore.NewMinerAPI(pieceStore, sa, cfg.MaxConcurrencyStorageCalls, cfg.MaxConcurrentUnseals) - ready := make(chan error, 1) - pieceStore.OnReady(func(err error) { - ready <- err - }) - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - if err := <-ready; err != nil { - return fmt.Errorf("aborting dagstore start; piecestore failed to start: %s", err) +func NewMinerAPI(cfg config.DAGStoreConfig) func(fx.Lifecycle, repo.LockedRepo, dtypes.ProviderPieceStore, mdagstore.SectorAccessor) (mdagstore.MinerAPI, error) { + return func(lc fx.Lifecycle, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, sa mdagstore.SectorAccessor) (mdagstore.MinerAPI, error) { + // caps the amount of concurrent calls to the storage, so that we don't + // spam it during heavy processes like bulk migration. + if v, ok := os.LookupEnv("LOTUS_DAGSTORE_MOUNT_CONCURRENCY"); ok { + concurrency, err := strconv.Atoi(v) + if err == nil { + cfg.MaxConcurrencyStorageCalls = concurrency } - return mountApi.Start(ctx) - }, - OnStop: func(context.Context) error { - return nil - }, - }) + } - return mountApi, nil + mountApi := mdagstore.NewMinerAPI(pieceStore, sa, cfg.MaxConcurrencyStorageCalls, cfg.MaxConcurrentUnseals) + ready := make(chan error, 1) + pieceStore.OnReady(func(err error) { + ready <- err + }) + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + if err := <-ready; err != nil { + return fmt.Errorf("aborting dagstore start; piecestore failed to start: %s", err) + } + return mountApi.Start(ctx) + }, + OnStop: func(context.Context) error { + return nil + }, + }) + + return mountApi, nil + } } // DAGStore constructs a DAG store using the supplied minerAPI, and the // user configuration. It returns both the DAGStore and the Wrapper suitable for // passing to markets. -func DAGStore(lc fx.Lifecycle, r repo.LockedRepo, minerAPI mdagstore.MinerAPI) (*dagstore.DAGStore, *mdagstore.Wrapper, error) { - cfg, err := extractDAGStoreConfig(r) - if err != nil { - return nil, nil, err - } - - // fall back to default root directory if not explicitly set in the config. - if cfg.RootDir == "" { - cfg.RootDir = filepath.Join(r.Path(), DefaultDAGStoreDir) - } - - v, ok := os.LookupEnv(EnvDAGStoreCopyConcurrency) - if ok { - concurrency, err := strconv.Atoi(v) - if err == nil { - cfg.MaxConcurrentReadyFetches = concurrency +func DAGStore(cfg config.DAGStoreConfig) func(fx.Lifecycle, repo.LockedRepo, mdagstore.MinerAPI) (*dagstore.DAGStore, *mdagstore.Wrapper, error) { + return func(lc fx.Lifecycle, r repo.LockedRepo, minerAPI mdagstore.MinerAPI) (*dagstore.DAGStore, *mdagstore.Wrapper, error) { + // fall back to default root directory if not explicitly set in the config. + if cfg.RootDir == "" { + cfg.RootDir = filepath.Join(r.Path(), DefaultDAGStoreDir) } + + v, ok := os.LookupEnv(EnvDAGStoreCopyConcurrency) + if ok { + concurrency, err := strconv.Atoi(v) + if err == nil { + cfg.MaxConcurrentReadyFetches = concurrency + } + } + + dagst, w, err := mdagstore.NewDAGStore(cfg, minerAPI) + if err != nil { + return nil, nil, xerrors.Errorf("failed to create DAG store: %w", err) + } + + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + return w.Start(ctx) + }, + OnStop: func(context.Context) error { + return w.Close() + }, + }) + + return dagst, w, nil } - - dagst, w, err := mdagstore.NewDAGStore(cfg, minerAPI) - if err != nil { - return nil, nil, xerrors.Errorf("failed to create DAG store: %w", err) - } - - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - return w.Start(ctx) - }, - OnStop: func(context.Context) error { - return w.Close() - }, - }) - - return dagst, w, nil -} - -func extractDAGStoreConfig(r repo.LockedRepo) (config.DAGStoreConfig, error) { - cfg, err := r.Config() - if err != nil { - return config.DAGStoreConfig{}, xerrors.Errorf("could not load config: %w", err) - } - mcfg, ok := cfg.(*config.StorageMiner) - if !ok { - return config.DAGStoreConfig{}, xerrors.Errorf("config not expected type; expected config.StorageMiner, got: %T", cfg) - } - return mcfg.DAGStore, nil } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 5c1c91bc5..92eba78ae 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -41,47 +41,83 @@ const ( fsKeystore = "keystore" ) -type RepoType int - -const ( - _ = iota // Default is invalid - FullNode RepoType = iota - StorageMiner - Worker - Wallet - Markets -) - -func (t RepoType) String() string { - s := [...]string{ - "__invalid__", - "FullNode", - "StorageMiner", - "Worker", - "Wallet", - "Markets", +func NewRepoTypeFromString(t string) RepoType { + switch t { + case "FullNode": + return FullNodeRepoType{} + case "StorageMiner": + return StorageMinerRepoType{} + case "Worker": + return WorkerRepoType{} + case "Wallet": + return WalletRepoType{} + default: + panic("unknown RepoType") } - if t < 0 || int(t) > len(s) { - return "__invalid__" - } - return s[t] +} + +type RepoType interface { + Type() string + Config() interface{} +} + +type FullNodeRepoType struct { +} + +func (f FullNodeRepoType) Type() string { + return "FullNode" +} + +func (f FullNodeRepoType) Config() interface{} { + return config.DefaultFullNode() +} + +type StorageMinerRepoType struct { +} + +func (f StorageMinerRepoType) Type() string { + return "StorageMiner" +} + +func (f StorageMinerRepoType) Config() interface{} { + return config.DefaultStorageMiner() +} + +type MarketsRepoType struct { +} + +func (f MarketsRepoType) Type() string { + return "Markets" +} + +func (f MarketsRepoType) Config() interface{} { + return config.DefaultStorageMiner() +} + +type WorkerRepoType struct { +} + +func (f WorkerRepoType) Type() string { + return "Worker" +} + +func (f WorkerRepoType) Config() interface{} { + return &struct{}{} +} + +type WalletRepoType struct { +} + +func (f WalletRepoType) Type() string { + return "Wallet" +} + +func (f WalletRepoType) Config() interface{} { + return &struct{}{} } func defConfForType(t RepoType) interface{} { - switch t { - case FullNode: - return config.DefaultFullNode() - case StorageMiner, Markets: - // markets is a specialised miner service - // this taxonomy needs to be cleaned up - return config.DefaultStorageMiner() - case Worker: - return &struct{}{} - case Wallet: - return &struct{}{} - default: - panic(fmt.Sprintf("unknown RepoType(%d)", int(t))) - } + return t.Config() } var log = logging.Logger("repo") diff --git a/node/repo/fsrepo_test.go b/node/repo/fsrepo_test.go index 381ebdcbe..76241b07d 100644 --- a/node/repo/fsrepo_test.go +++ b/node/repo/fsrepo_test.go @@ -18,7 +18,7 @@ func genFsRepo(t *testing.T) (*FsRepo, func()) { t.Fatal(err) } - err = repo.Init(FullNode) + err = repo.Init(FullNodeRepoType{}) if err != ErrRepoExists && err != nil { t.Fatal(err) } diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index 3be4ab567..9feef684a 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -106,7 +106,7 @@ func (lmem *lockedMemRepo) Path() string { panic(err) // only used in tests, probably fine } - if lmem.t == StorageMiner { + if lmem.t.Type() == "StorageMiner" || lmem.t.Type() == "Boost" { // this is required due to the method makeDealStaging from cmd/lotus-storage-miner/init.go // deal-staging is the directory deal files are staged in before being sealed into sectors // for offline deal flow. diff --git a/node/repo/repo_test.go b/node/repo/repo_test.go index cd19f86f6..55c61845c 100644 --- a/node/repo/repo_test.go +++ b/node/repo/repo_test.go @@ -21,12 +21,12 @@ func basicTest(t *testing.T, repo Repo) { } assert.Nil(t, apima, "with no api endpoint, return should be nil") - lrepo, err := repo.Lock(FullNode) + lrepo, err := repo.Lock(FullNodeRepoType{}) assert.NoError(t, err, "should be able to lock once") assert.NotNil(t, lrepo, "locked repo shouldn't be nil") { - lrepo2, err := repo.Lock(FullNode) + lrepo2, err := repo.Lock(FullNodeRepoType{}) if assert.Error(t, err) { assert.Equal(t, ErrRepoAlreadyLocked, err) } @@ -36,7 +36,7 @@ func basicTest(t *testing.T, repo Repo) { err = lrepo.Close() assert.NoError(t, err, "should be able to unlock") - lrepo, err = repo.Lock(FullNode) + lrepo, err = repo.Lock(FullNodeRepoType{}) assert.NoError(t, err, "should be able to relock") assert.NotNil(t, lrepo, "locked repo shouldn't be nil") @@ -80,7 +80,7 @@ func basicTest(t *testing.T, repo Repo) { k1 := types.KeyInfo{Type: "foo"} k2 := types.KeyInfo{Type: "bar"} - lrepo, err = repo.Lock(FullNode) + lrepo, err = repo.Lock(FullNodeRepoType{}) assert.NoError(t, err, "should be able to relock") assert.NotNil(t, lrepo, "locked repo shouldn't be nil") diff --git a/testplans/lotus-soup/testkit/role_miner.go b/testplans/lotus-soup/testkit/role_miner.go index 7204c71fe..e3d7c7d62 100644 --- a/testplans/lotus-soup/testkit/role_miner.go +++ b/testplans/lotus-soup/testkit/role_miner.go @@ -138,7 +138,7 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) { return nil, err } - err = minerRepo.Init(repo.StorageMiner) + err = minerRepo.Init(repo.StorageMinerRepoType{}) if err != nil { return nil, err } @@ -146,7 +146,7 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) { var subsystems config.MinerSubsystemConfig { - lr, err := minerRepo.Lock(repo.StorageMiner) + lr, err := minerRepo.Lock(repo.StorageMinerRepoType{}) if err != nil { return nil, err } @@ -244,7 +244,7 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) { return nil, err } - err = nodeRepo.Init(repo.FullNode) + err = nodeRepo.Init(repo.FullNodeRepoType{}) if err != nil { return nil, err } From f1f878a0003f68e0cfade0e93150a71070389d9a Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 16 Feb 2022 12:26:41 +0100 Subject: [PATCH 02/13] unexport repo types; use a global var for every repo type --- chain/gen/gen.go | 2 +- chain/store/store_test.go | 2 +- cli/auth.go | 2 +- cli/client_retr.go | 4 +- cli/pprof.go | 2 +- cli/util/api.go | 16 ++++---- cmd/lotus-miner/actor_test.go | 2 +- cmd/lotus-miner/allinfo_test.go | 2 +- cmd/lotus-miner/backup.go | 2 +- cmd/lotus-miner/config.go | 2 +- cmd/lotus-miner/init.go | 6 +-- cmd/lotus-miner/init_restore.go | 4 +- cmd/lotus-miner/main.go | 6 +-- cmd/lotus-miner/run.go | 2 +- cmd/lotus-seal-worker/main.go | 12 +++--- cmd/lotus-shed/balances.go | 4 +- cmd/lotus-shed/export-car.go | 2 +- cmd/lotus-shed/export.go | 2 +- cmd/lotus-shed/import-car.go | 4 +- cmd/lotus-shed/keyinfo.go | 2 +- cmd/lotus-shed/market.go | 2 +- cmd/lotus-shed/migrations.go | 2 +- cmd/lotus-shed/miner-peerid.go | 2 +- cmd/lotus-shed/miner-types.go | 2 +- cmd/lotus-shed/pruning.go | 2 +- cmd/lotus-shed/rpc.go | 6 +-- cmd/lotus-shed/splitstore.go | 4 +- cmd/lotus-shed/terminations.go | 2 +- cmd/lotus-sim/simulation/node.go | 2 +- cmd/lotus-wallet/main.go | 2 +- cmd/lotus/backup.go | 4 +- cmd/lotus/config.go | 2 +- cmd/lotus/daemon.go | 4 +- cmd/lotus/main.go | 2 +- documentation/en/architecture/architecture.md | 6 +-- extern/sector-storage/stores/remote_test.go | 4 +- itests/deals_concurrent_test.go | 2 +- itests/kit/ensemble.go | 2 +- itests/sector_finalize_early_test.go | 2 +- itests/sector_miner_collateral_test.go | 2 +- node/builder.go | 12 +++--- node/builder_chain.go | 2 +- node/builder_miner.go | 2 +- node/repo/fsrepo.go | 38 +++++++++++-------- node/repo/fsrepo_test.go | 2 +- node/repo/repo_test.go | 8 ++-- testplans/lotus-soup/testkit/role_miner.go | 6 +-- 47 files changed, 107 insertions(+), 99 deletions(-) diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 51289cdf8..4153830dc 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -121,7 +121,7 @@ func NewGeneratorWithSectorsAndUpgradeSchedule(numSectors int, us stmgr.UpgradeS policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) mr := repo.NewMemory(nil) - lr, err := mr.Lock(repo.StorageMinerRepoType{}) + lr, err := mr.Lock(repo.StorageMiner) if err != nil { return nil, xerrors.Errorf("taking mem-repo lock failed: %w", err) } diff --git a/chain/store/store_test.go b/chain/store/store_test.go index d75917017..a759a48a8 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -48,7 +48,7 @@ func BenchmarkGetRandomness(b *testing.B) { b.Fatal(err) } - lr, err := r.Lock(repo.FullNodeRepoType{}) + lr, err := r.Lock(repo.FullNode) if err != nil { b.Fatal(err) } diff --git a/cli/auth.go b/cli/auth.go index f5a29a41e..e6eeace89 100644 --- a/cli/auth.go +++ b/cli/auth.go @@ -114,7 +114,7 @@ var AuthApiInfoToken = &cli.Command{ ti, ok := cctx.App.Metadata["repoType"] if !ok { log.Errorf("unknown repo type, are you sure you want to use GetCommonAPI?") - ti = repo.FullNodeRepoType{} + ti = repo.FullNode } t, ok := ti.(repo.RepoType) if !ok { diff --git a/cli/client_retr.go b/cli/client_retr.go index 1b7061ad9..9b195a5d8 100644 --- a/cli/client_retr.go +++ b/cli/client_retr.go @@ -403,7 +403,7 @@ var clientRetrieveCatCmd = &cli.Command{ return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments")) } - ainfo, err := GetAPIInfo(cctx, repo.FullNodeRepoType{}) + ainfo, err := GetAPIInfo(cctx, repo.FullNode) if err != nil { return xerrors.Errorf("could not get API info: %w", err) } @@ -482,7 +482,7 @@ var clientRetrieveLsCmd = &cli.Command{ return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments")) } - ainfo, err := GetAPIInfo(cctx, repo.FullNodeRepoType{}) + ainfo, err := GetAPIInfo(cctx, repo.FullNode) if err != nil { return xerrors.Errorf("could not get API info: %w", err) } diff --git a/cli/pprof.go b/cli/pprof.go index 81eb03448..ae4016e11 100644 --- a/cli/pprof.go +++ b/cli/pprof.go @@ -26,7 +26,7 @@ var PprofGoroutines = &cli.Command{ ti, ok := cctx.App.Metadata["repoType"] if !ok { log.Errorf("unknown repo type, are you sure you want to use GetAPI?") - ti = repo.FullNodeRepoType{} + ti = repo.FullNode } t, ok := ti.(repo.RepoType) if !ok { diff --git a/cli/util/api.go b/cli/util/api.go index e8bf4af2d..70051cbe0 100644 --- a/cli/util/api.go +++ b/cli/util/api.go @@ -200,7 +200,7 @@ func GetCommonAPI(ctx *cli.Context) (api.CommonNet, jsonrpc.ClientCloser, error) ti, ok := ctx.App.Metadata["repoType"] if !ok { log.Errorf("unknown repo type, are you sure you want to use GetCommonAPI?") - ti = repo.FullNodeRepoType{} + ti = repo.FullNode } t, ok := ti.(repo.RepoType) if !ok { @@ -232,7 +232,7 @@ func GetFullNodeAPI(ctx *cli.Context) (v0api.FullNode, jsonrpc.ClientCloser, err return &v0api.WrapperV1Full{FullNode: tn.(v1api.FullNode)}, func() {}, nil } - addr, headers, err := GetRawAPI(ctx, repo.FullNodeRepoType{}, "v0") + addr, headers, err := GetRawAPI(ctx, repo.FullNode, "v0") if err != nil { return nil, nil, err } @@ -249,7 +249,7 @@ func GetFullNodeAPIV1(ctx *cli.Context) (v1api.FullNode, jsonrpc.ClientCloser, e return tn.(v1api.FullNode), func() {}, nil } - addr, headers, err := GetRawAPI(ctx, repo.FullNodeRepoType{}, "v1") + addr, headers, err := GetRawAPI(ctx, repo.FullNode, "v1") if err != nil { return nil, nil, err } @@ -293,7 +293,7 @@ func GetStorageMinerAPI(ctx *cli.Context, opts ...GetStorageMinerOption) (api.St return tn.(api.StorageMiner), func() {}, nil } - addr, headers, err := GetRawAPI(ctx, repo.StorageMinerRepoType{}, "v0") + addr, headers, err := GetRawAPI(ctx, repo.StorageMiner, "v0") if err != nil { return nil, nil, err } @@ -322,7 +322,7 @@ func GetStorageMinerAPI(ctx *cli.Context, opts ...GetStorageMinerOption) (api.St } func GetWorkerAPI(ctx *cli.Context) (api.Worker, jsonrpc.ClientCloser, error) { - addr, headers, err := GetRawAPI(ctx, repo.WorkerRepoType{}, "v0") + addr, headers, err := GetRawAPI(ctx, repo.Worker, "v0") if err != nil { return nil, nil, err } @@ -340,7 +340,7 @@ func GetMarketsAPI(ctx *cli.Context) (api.StorageMiner, jsonrpc.ClientCloser, er return tn.(api.StorageMiner), func() {}, nil } - addr, headers, err := GetRawAPI(ctx, repo.MarketsRepoType{}, "v0") + addr, headers, err := GetRawAPI(ctx, repo.Markets, "v0") if err != nil { return nil, nil, err } @@ -356,7 +356,7 @@ func GetMarketsAPI(ctx *cli.Context) (api.StorageMiner, jsonrpc.ClientCloser, er } func GetGatewayAPI(ctx *cli.Context) (api.Gateway, jsonrpc.ClientCloser, error) { - addr, headers, err := GetRawAPI(ctx, repo.FullNodeRepoType{}, "v1") + addr, headers, err := GetRawAPI(ctx, repo.FullNode, "v1") if err != nil { return nil, nil, err } @@ -369,7 +369,7 @@ func GetGatewayAPI(ctx *cli.Context) (api.Gateway, jsonrpc.ClientCloser, error) } func GetGatewayAPIV0(ctx *cli.Context) (v0api.Gateway, jsonrpc.ClientCloser, error) { - addr, headers, err := GetRawAPI(ctx, repo.FullNodeRepoType{}, "v0") + addr, headers, err := GetRawAPI(ctx, repo.FullNode, "v0") if err != nil { return nil, nil, err } diff --git a/cmd/lotus-miner/actor_test.go b/cmd/lotus-miner/actor_test.go index d9f7297b9..5650a9ac5 100644 --- a/cmd/lotus-miner/actor_test.go +++ b/cmd/lotus-miner/actor_test.go @@ -40,7 +40,7 @@ func TestWorkerKeyChange(t *testing.T) { run := func(cmd *cli.Command, args ...string) error { app := cli.NewApp() app.Metadata = map[string]interface{}{ - "repoType": repo.StorageMinerRepoType{}, + "repoType": repo.StorageMiner, "testnode-full": client1, "testnode-storage": miner, } diff --git a/cmd/lotus-miner/allinfo_test.go b/cmd/lotus-miner/allinfo_test.go index c85e5282c..5f30b4fec 100644 --- a/cmd/lotus-miner/allinfo_test.go +++ b/cmd/lotus-miner/allinfo_test.go @@ -36,7 +36,7 @@ func TestMinerAllInfo(t *testing.T) { run := func(t *testing.T) { app := cli.NewApp() app.Metadata = map[string]interface{}{ - "repoType": repo.StorageMinerRepoType{}, + "repoType": repo.StorageMiner, "testnode-full": client, "testnode-storage": miner, } diff --git a/cmd/lotus-miner/backup.go b/cmd/lotus-miner/backup.go index 8fe087561..cf8c9f912 100644 --- a/cmd/lotus-miner/backup.go +++ b/cmd/lotus-miner/backup.go @@ -9,6 +9,6 @@ import ( "github.com/filecoin-project/lotus/node/repo" ) -var backupCmd = lcli.BackupCmd(FlagMinerRepo, repo.StorageMinerRepoType{}, func(cctx *cli.Context) (lcli.BackupAPI, jsonrpc.ClientCloser, error) { +var backupCmd = lcli.BackupCmd(FlagMinerRepo, repo.StorageMiner, func(cctx *cli.Context) (lcli.BackupAPI, jsonrpc.ClientCloser, error) { return lcli.GetStorageMinerAPI(cctx) }) diff --git a/cmd/lotus-miner/config.go b/cmd/lotus-miner/config.go index 42bb4fdef..652426583 100644 --- a/cmd/lotus-miner/config.go +++ b/cmd/lotus-miner/config.go @@ -66,7 +66,7 @@ var configUpdateCmd = &cli.Command{ return xerrors.Errorf("repo not initialized") } - lr, err := r.LockRO(repo.StorageMinerRepoType{}) + lr, err := r.LockRO(repo.StorageMiner) if err != nil { return xerrors.Errorf("locking repo: %w", err) } diff --git a/cmd/lotus-miner/init.go b/cmd/lotus-miner/init.go index 30a2d6844..59ea75b10 100644 --- a/cmd/lotus-miner/init.go +++ b/cmd/lotus-miner/init.go @@ -203,12 +203,12 @@ var initCmd = &cli.Command{ log.Info("Initializing repo") - if err := r.Init(repo.StorageMinerRepoType{}); err != nil { + if err := r.Init(repo.StorageMiner); err != nil { return err } { - lr, err := r.Lock(repo.StorageMinerRepoType{}) + lr, err := r.Lock(repo.StorageMiner) if err != nil { return err } @@ -410,7 +410,7 @@ func findMarketDealID(ctx context.Context, api v1api.FullNode, deal market2.Deal } func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode, r repo.Repo, ssize abi.SectorSize, gasPrice types.BigInt) error { - lr, err := r.Lock(repo.StorageMinerRepoType{}) + lr, err := r.Lock(repo.StorageMiner) if err != nil { return err } diff --git a/cmd/lotus-miner/init_restore.go b/cmd/lotus-miner/init_restore.go index 077ac8011..1aaa7909a 100644 --- a/cmd/lotus-miner/init_restore.go +++ b/cmd/lotus-miner/init_restore.go @@ -159,11 +159,11 @@ func restore(ctx context.Context, cctx *cli.Context, targetPath string, strConfi log.Info("Initializing repo") - if err := r.Init(repo.StorageMinerRepoType{}); err != nil { + if err := r.Init(repo.StorageMiner); err != nil { return err } - lr, err := r.Lock(repo.StorageMinerRepoType{}) + lr, err := r.Lock(repo.StorageMiner) if err != nil { return err } diff --git a/cmd/lotus-miner/main.go b/cmd/lotus-miner/main.go index 01a931a5f..57b5d8a3e 100644 --- a/cmd/lotus-miner/main.go +++ b/cmd/lotus-miner/main.go @@ -93,7 +93,7 @@ func main() { return err } } - c.App.Metadata["repoType"] = repo.MarketsRepoType{} + c.App.Metadata["repoType"] = repo.Markets return nil } @@ -150,7 +150,7 @@ func main() { // this command is explicitly called on markets, inform // common commands by overriding the repoType. if c.Bool("call-on-markets") { - c.App.Metadata["repoType"] = repo.MarketsRepoType{} + c.App.Metadata["repoType"] = repo.Markets } return nil }, @@ -164,7 +164,7 @@ func main() { }, } app.Setup() - app.Metadata["repoType"] = repo.StorageMinerRepoType{} + app.Metadata["repoType"] = repo.StorageMiner lcli.RunApp(app) } diff --git a/cmd/lotus-miner/run.go b/cmd/lotus-miner/run.go index f5a138fba..f276f319c 100644 --- a/cmd/lotus-miner/run.go +++ b/cmd/lotus-miner/run.go @@ -119,7 +119,7 @@ var runCmd = &cli.Command{ return xerrors.Errorf("repo at '%s' is not initialized, run 'lotus-miner init' to set it up", minerRepoPath) } - lr, err := r.Lock(repo.StorageMinerRepoType{}) + lr, err := r.Lock(repo.StorageMiner) if err != nil { return err } diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index a1812cc36..9e6843dbf 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -108,7 +108,7 @@ func main() { Commands: local, } app.Setup() - app.Metadata["repoType"] = repo.WorkerRepoType{} + app.Metadata["repoType"] = repo.Worker if err := app.Run(os.Args); err != nil { log.Warnf("%+v", err) @@ -310,11 +310,11 @@ var runCmd = &cli.Command{ return err } if !ok { - if err := r.Init(repo.WorkerRepoType{}); err != nil { + if err := r.Init(repo.Worker); err != nil { return err } - lr, err := r.Lock(repo.WorkerRepoType{}) + lr, err := r.Lock(repo.Worker) if err != nil { return err } @@ -359,7 +359,7 @@ var runCmd = &cli.Command{ } } - lr, err := r.Lock(repo.WorkerRepoType{}) + lr, err := r.Lock(repo.Worker) if err != nil { return err } @@ -397,7 +397,7 @@ var runCmd = &cli.Command{ } // Setup remote sector store - sminfo, err := lcli.GetAPIInfo(cctx, repo.StorageMinerRepoType{}) + sminfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) if err != nil { return xerrors.Errorf("could not get api info: %w", err) } @@ -484,7 +484,7 @@ var runCmd = &cli.Command{ return xerrors.Errorf("setting api endpoint: %w", err) } - ainfo, err := lcli.GetAPIInfo(cctx, repo.StorageMinerRepoType{}) + ainfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) if err != nil { return xerrors.Errorf("could not get miner API info: %w", err) } diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index 2550741e0..1a22be3c3 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -486,7 +486,7 @@ var chainBalanceStateCmd = &cli.Command{ return err } - lkrepo, err := fsrepo.Lock(repo.FullNodeRepoType{}) + lkrepo, err := fsrepo.Lock(repo.FullNode) if err != nil { return err } @@ -710,7 +710,7 @@ var chainPledgeCmd = &cli.Command{ return err } - lkrepo, err := fsrepo.Lock(repo.FullNodeRepoType{}) + lkrepo, err := fsrepo.Lock(repo.FullNode) if err != nil { return err } diff --git a/cmd/lotus-shed/export-car.go b/cmd/lotus-shed/export-car.go index 0d00e3480..97e4fb6c6 100644 --- a/cmd/lotus-shed/export-car.go +++ b/cmd/lotus-shed/export-car.go @@ -67,7 +67,7 @@ var exportCarCmd = &cli.Command{ return xerrors.Errorf("lotus repo doesn't exist") } - lr, err := r.Lock(repo.FullNodeRepoType{}) + lr, err := r.Lock(repo.FullNode) if err != nil { return err } diff --git a/cmd/lotus-shed/export.go b/cmd/lotus-shed/export.go index 39614a8a7..3851e4922 100644 --- a/cmd/lotus-shed/export.go +++ b/cmd/lotus-shed/export.go @@ -59,7 +59,7 @@ var exportChainCmd = &cli.Command{ return xerrors.Errorf("lotus repo doesn't exist") } - lr, err := r.Lock(repo.FullNodeRepoType{}) + lr, err := r.Lock(repo.FullNode) if err != nil { return err } diff --git a/cmd/lotus-shed/import-car.go b/cmd/lotus-shed/import-car.go index b05deb2c5..973e7b31b 100644 --- a/cmd/lotus-shed/import-car.go +++ b/cmd/lotus-shed/import-car.go @@ -35,7 +35,7 @@ var importCarCmd = &cli.Command{ return xerrors.Errorf("lotus repo doesn't exist") } - lr, err := r.Lock(repo.FullNodeRepoType{}) + lr, err := r.Lock(repo.FullNode) if err != nil { return err } @@ -112,7 +112,7 @@ var importObjectCmd = &cli.Command{ return xerrors.Errorf("lotus repo doesn't exist") } - lr, err := r.Lock(repo.FullNodeRepoType{}) + lr, err := r.Lock(repo.FullNode) if err != nil { return err } diff --git a/cmd/lotus-shed/keyinfo.go b/cmd/lotus-shed/keyinfo.go index aec19d8c7..3c99b5050 100644 --- a/cmd/lotus-shed/keyinfo.go +++ b/cmd/lotus-shed/keyinfo.go @@ -183,7 +183,7 @@ var keyinfoImportCmd = &cli.Command{ return err } - lkrepo, err := fsrepo.Lock(repo.FullNodeRepoType{}) + lkrepo, err := fsrepo.Lock(repo.FullNode) if err != nil { return err } diff --git a/cmd/lotus-shed/market.go b/cmd/lotus-shed/market.go index 924b3441c..aaef4690e 100644 --- a/cmd/lotus-shed/market.go +++ b/cmd/lotus-shed/market.go @@ -301,7 +301,7 @@ func openLockedRepo(path string) (repo.LockedRepo, error) { } // Lock the repo - lr, err := rpo.Lock(repo.StorageMinerRepoType{}) + lr, err := rpo.Lock(repo.StorageMiner) if err != nil { return nil, xerrors.Errorf("locking repo %s: %w", path, err) } diff --git a/cmd/lotus-shed/migrations.go b/cmd/lotus-shed/migrations.go index c1a8f7b5e..85987c658 100644 --- a/cmd/lotus-shed/migrations.go +++ b/cmd/lotus-shed/migrations.go @@ -47,7 +47,7 @@ var migrationsCmd = &cli.Command{ return err } - lkrepo, err := fsrepo.Lock(repo.FullNodeRepoType{}) + lkrepo, err := fsrepo.Lock(repo.FullNode) if err != nil { return err } diff --git a/cmd/lotus-shed/miner-peerid.go b/cmd/lotus-shed/miner-peerid.go index c02a226be..3ccfb429b 100644 --- a/cmd/lotus-shed/miner-peerid.go +++ b/cmd/lotus-shed/miner-peerid.go @@ -54,7 +54,7 @@ var minerPeeridCmd = &cli.Command{ return err } - lkrepo, err := fsrepo.Lock(repo.FullNodeRepoType{}) + lkrepo, err := fsrepo.Lock(repo.FullNode) if err != nil { return err } diff --git a/cmd/lotus-shed/miner-types.go b/cmd/lotus-shed/miner-types.go index f0655096c..05ef7b0a7 100644 --- a/cmd/lotus-shed/miner-types.go +++ b/cmd/lotus-shed/miner-types.go @@ -50,7 +50,7 @@ var minerTypesCmd = &cli.Command{ return err } - lkrepo, err := fsrepo.Lock(repo.FullNodeRepoType{}) + lkrepo, err := fsrepo.Lock(repo.FullNode) if err != nil { return err } diff --git a/cmd/lotus-shed/pruning.go b/cmd/lotus-shed/pruning.go index 184fea85b..164ff197a 100644 --- a/cmd/lotus-shed/pruning.go +++ b/cmd/lotus-shed/pruning.go @@ -123,7 +123,7 @@ var stateTreePruneCmd = &cli.Command{ return err } - lkrepo, err := fsrepo.Lock(repo.FullNodeRepoType{}) + lkrepo, err := fsrepo.Lock(repo.FullNode) if err != nil { return err } diff --git a/cmd/lotus-shed/rpc.go b/cmd/lotus-shed/rpc.go index 0c870f090..b253304a1 100644 --- a/cmd/lotus-shed/rpc.go +++ b/cmd/lotus-shed/rpc.go @@ -36,9 +36,9 @@ var rpcCmd = &cli.Command{ Action: func(cctx *cli.Context) error { var rt repo.RepoType if cctx.Bool("miner") { - rt = repo.StorageMinerRepoType{} + rt = repo.StorageMiner } else { - rt = repo.FullNodeRepoType{} + rt = repo.FullNode } addr, headers, err := lcli.GetRawAPI(cctx, rt, cctx.String("version")) @@ -125,7 +125,7 @@ var rpcCmd = &cli.Command{ return send(cctx.Args().Get(0), params) } - cctx.App.Metadata["repoType"] = repo.FullNodeRepoType{} + cctx.App.Metadata["repoType"] = repo.FullNode if err := lcli.VersionCmd.Action(cctx); err != nil { return err } diff --git a/cmd/lotus-shed/splitstore.go b/cmd/lotus-shed/splitstore.go index a907b3135..58563955f 100644 --- a/cmd/lotus-shed/splitstore.go +++ b/cmd/lotus-shed/splitstore.go @@ -67,7 +67,7 @@ var splitstoreRollbackCmd = &cli.Command{ return xerrors.Errorf("lotus repo doesn't exist") } - lr, err := r.Lock(repo.FullNodeRepoType{}) + lr, err := r.Lock(repo.FullNode) if err != nil { return xerrors.Errorf("error locking repo: %w", err) } @@ -153,7 +153,7 @@ var splitstoreClearCmd = &cli.Command{ return xerrors.Errorf("lotus repo doesn't exist") } - lr, err := r.Lock(repo.FullNodeRepoType{}) + lr, err := r.Lock(repo.FullNode) if err != nil { return xerrors.Errorf("error locking repo: %w", err) } diff --git a/cmd/lotus-shed/terminations.go b/cmd/lotus-shed/terminations.go index 8edf4e131..0691f35da 100644 --- a/cmd/lotus-shed/terminations.go +++ b/cmd/lotus-shed/terminations.go @@ -53,7 +53,7 @@ var terminationsCmd = &cli.Command{ return err } - lkrepo, err := fsrepo.Lock(repo.FullNodeRepoType{}) + lkrepo, err := fsrepo.Lock(repo.FullNode) if err != nil { return err } diff --git a/cmd/lotus-sim/simulation/node.go b/cmd/lotus-sim/simulation/node.go index 2a9769381..da027ff4f 100644 --- a/cmd/lotus-sim/simulation/node.go +++ b/cmd/lotus-sim/simulation/node.go @@ -41,7 +41,7 @@ func OpenNode(ctx context.Context, path string) (*Node, error) { // NewNode constructs a new node from the given repo. func NewNode(ctx context.Context, r repo.Repo) (nd *Node, _err error) { - lr, err := r.Lock(repo.FullNodeRepoType{}) + lr, err := r.Lock(repo.FullNode) if err != nil { return nil, err } diff --git a/cmd/lotus-wallet/main.go b/cmd/lotus-wallet/main.go index 21563a5b4..d51175c0d 100644 --- a/cmd/lotus-wallet/main.go +++ b/cmd/lotus-wallet/main.go @@ -278,7 +278,7 @@ func openRepo(cctx *cli.Context) (repo.LockedRepo, types.KeyStore, error) { return nil, nil, err } if !ok { - if err := r.Init(repo.WorkerRepoType{}); err != nil { + if err := r.Init(repo.Worker); err != nil { return nil, nil, err } } diff --git a/cmd/lotus/backup.go b/cmd/lotus/backup.go index 7403d75c0..4bdd21322 100644 --- a/cmd/lotus/backup.go +++ b/cmd/lotus/backup.go @@ -18,7 +18,7 @@ import ( "github.com/filecoin-project/lotus/node/repo" ) -var backupCmd = lcli.BackupCmd("repo", repo.FullNodeRepoType{}, func(cctx *cli.Context) (lcli.BackupAPI, jsonrpc.ClientCloser, error) { +var backupCmd = lcli.BackupCmd("repo", repo.FullNode, func(cctx *cli.Context) (lcli.BackupAPI, jsonrpc.ClientCloser, error) { return lcli.GetFullNodeAPI(cctx) }) @@ -39,7 +39,7 @@ func restore(cctx *cli.Context, r repo.Repo) error { } defer f.Close() // nolint:errcheck - lr, err := r.Lock(repo.FullNodeRepoType{}) + lr, err := r.Lock(repo.FullNode) if err != nil { return err } diff --git a/cmd/lotus/config.go b/cmd/lotus/config.go index 3ca0683f7..fcb7e2b08 100644 --- a/cmd/lotus/config.go +++ b/cmd/lotus/config.go @@ -66,7 +66,7 @@ var configUpdateCmd = &cli.Command{ return xerrors.Errorf("repo not initialized") } - lr, err := r.LockRO(repo.FullNodeRepoType{}) + lr, err := r.LockRO(repo.FullNode) if err != nil { return xerrors.Errorf("locking repo: %w", err) } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 559713d18..f285ba74e 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -227,7 +227,7 @@ var DaemonCmd = &cli.Command{ r.SetConfigPath(cctx.String("config")) } - err = r.Init(repo.FullNodeRepoType{}) + err = r.Init(repo.FullNode) if err != nil && err != repo.ErrRepoExists { return xerrors.Errorf("repo init error: %w", err) } @@ -462,7 +462,7 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool) l = st.Size() } - lr, err := r.Lock(repo.FullNodeRepoType{}) + lr, err := r.Lock(repo.FullNode) if err != nil { return err } diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go index eeb4192b4..c19b9fce4 100644 --- a/cmd/lotus/main.go +++ b/cmd/lotus/main.go @@ -106,7 +106,7 @@ func main() { app.Setup() app.Metadata["traceContext"] = ctx - app.Metadata["repoType"] = repo.FullNodeRepoType{} + app.Metadata["repoType"] = repo.FullNode lcli.RunApp(app) } diff --git a/documentation/en/architecture/architecture.md b/documentation/en/architecture/architecture.md index 200b4a516..64914d539 100644 --- a/documentation/en/architecture/architecture.md +++ b/documentation/en/architecture/architecture.md @@ -341,8 +341,8 @@ FIXME: This section needs to be clarified / corrected...I don't fully understand At the end of the `Repo()` function we see two mutually exclusive configuration calls based on the `RepoType` (`node/repo/fsrepo.go`). ```Go - ApplyIf(isType(repo.FullNodeRepoType{}), ConfigFullNode(c)), - ApplyIf(isType(repo.StorageMinerRepoType{}), ConfigStorageMiner(c)), + ApplyIf(isType(repo.FullNode), ConfigFullNode(c)), + ApplyIf(isType(repo.StorageMiner), ConfigStorageMiner(c)), ``` As we said, the repo fully identifies the node so a repo type is also a *node* type, in this case a full node or a miner. (FIXME: What is the difference between the two, does *full* imply miner?) In this case the `daemon` command will create a `FullNode`, this is specified in the command logic itself in `main.DaemonCmd()`, the `FsRepo` created (and passed to `node.Repo()`) will be initiated with that type (see `(*FsRepo).Init(t RepoType)`). @@ -352,7 +352,7 @@ FIXME: Much of this might need to be subsumed into the p2p section The `node.Online()` configuration function (`node/builder.go`) initializes components that involve connecting to, or interacting with, the Filecoin network. These connections are managed through the libp2p stack (FIXME link to this section when it exists). -We discuss some of the components found in the full node type (that is, included in the `ApplyIf(isType(repo.FullNodeRepoType{}),` call). +We discuss some of the components found in the full node type (that is, included in the `ApplyIf(isType(repo.FullNode),` call). #### Chainstore diff --git a/extern/sector-storage/stores/remote_test.go b/extern/sector-storage/stores/remote_test.go index 33e29820d..a7a82a728 100644 --- a/extern/sector-storage/stores/remote_test.go +++ b/extern/sector-storage/stores/remote_test.go @@ -73,8 +73,8 @@ func TestMoveShared(t *testing.T) { openRepo := func(dir string) repo.LockedRepo { r, err := repo.NewFS(dir) require.NoError(t, err) - require.NoError(t, r.Init(repo.WorkerRepoType{})) - lr, err := r.Lock(repo.WorkerRepoType{}) + require.NoError(t, r.Init(repo.Worker)) + lr, err := r.Lock(repo.Worker) require.NoError(t, err) t.Cleanup(func() { diff --git a/itests/deals_concurrent_test.go b/itests/deals_concurrent_test.go index 81b1444d9..18d8da02a 100644 --- a/itests/deals_concurrent_test.go +++ b/itests/deals_concurrent_test.go @@ -146,7 +146,7 @@ func TestSimultanenousTransferLimit(t *testing.T) { ) runTest := func(t *testing.T) { client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs(), kit.ConstructorOpts( - node.ApplyIf(node.IsType(repo.StorageMinerRepoType{}), node.Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync(graphsyncThrottle, 0, graphsyncThrottle))), + node.ApplyIf(node.IsType(repo.StorageMiner), node.Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync(graphsyncThrottle, 0, graphsyncThrottle))), node.Override(new(dtypes.Graphsync), modules.Graphsync(graphsyncThrottle, graphsyncThrottle)), )) ens.InterconnectAll().BeginMining(250 * time.Millisecond) diff --git a/itests/kit/ensemble.go b/itests/kit/ensemble.go index 2594babca..6a0158429 100644 --- a/itests/kit/ensemble.go +++ b/itests/kit/ensemble.go @@ -443,7 +443,7 @@ func (n *Ensemble) Start() *Ensemble { r := repo.NewMemory(nil) - lr, err := r.Lock(repo.StorageMinerRepoType{}) + lr, err := r.Lock(repo.StorageMiner) require.NoError(n.t, err) c, err := lr.Config() diff --git a/itests/sector_finalize_early_test.go b/itests/sector_finalize_early_test.go index 78eec3fa7..233bc8fcb 100644 --- a/itests/sector_finalize_early_test.go +++ b/itests/sector_finalize_early_test.go @@ -35,7 +35,7 @@ func TestDealsWithFinalizeEarly(t *testing.T) { var blockTime = 50 * time.Millisecond client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.ConstructorOpts( - node.ApplyIf(node.IsType(repo.StorageMinerRepoType{}), node.Override(new(dtypes.GetSealingConfigFunc), func() (dtypes.GetSealingConfigFunc, error) { + node.ApplyIf(node.IsType(repo.StorageMiner), node.Override(new(dtypes.GetSealingConfigFunc), func() (dtypes.GetSealingConfigFunc, error) { return func() (sealiface.Config, error) { cf := config.DefaultStorageMiner() cf.Sealing.FinalizeEarly = true diff --git a/itests/sector_miner_collateral_test.go b/itests/sector_miner_collateral_test.go index 5d05f3d2d..af67b132b 100644 --- a/itests/sector_miner_collateral_test.go +++ b/itests/sector_miner_collateral_test.go @@ -38,7 +38,7 @@ func TestMinerBalanceCollateral(t *testing.T) { defer cancel() opts := kit.ConstructorOpts( - node.ApplyIf(node.IsType(repo.StorageMinerRepoType{}), node.Override(new(dtypes.GetSealingConfigFunc), func() (dtypes.GetSealingConfigFunc, error) { + node.ApplyIf(node.IsType(repo.StorageMiner), node.Override(new(dtypes.GetSealingConfigFunc), func() (dtypes.GetSealingConfigFunc, error) { return func() (sealiface.Config, error) { return sealiface.Config{ MaxWaitDealsSectors: 4, diff --git a/node/builder.go b/node/builder.go index 152c32a73..f26bddfa9 100644 --- a/node/builder.go +++ b/node/builder.go @@ -227,12 +227,12 @@ func IsType(t repo.RepoType) func(s *Settings) bool { return func(s *Settings) bool { return s.nodeType.Type() == t.Type() } } -func isFullOrLiteNode(s *Settings) bool { return s.nodeType.Type() == repo.FullNodeRepoType{}.Type() } +func isFullOrLiteNode(s *Settings) bool { return s.nodeType.Type() == repo.FullNode.Type() } func isFullNode(s *Settings) bool { - return s.nodeType.Type() == repo.FullNodeRepoType{}.Type() && !s.Lite + return s.nodeType.Type() == repo.FullNode.Type() && !s.Lite } func isLiteNode(s *Settings) bool { - return s.nodeType.Type() == repo.FullNodeRepoType{}.Type() && s.Lite + return s.nodeType.Type() == repo.FullNode.Type() && s.Lite } func Base() Option { @@ -245,7 +245,7 @@ func Base() Option { LibP2P, ), ApplyIf(isFullOrLiteNode, ChainNode), - ApplyIf(IsType(repo.StorageMinerRepoType{}), MinerNode), + ApplyIf(IsType(repo.StorageMiner), MinerNode), ) } @@ -318,8 +318,8 @@ func Repo(r repo.Repo) Option { Override(new(*dtypes.APIAlg), modules.APISecret), - ApplyIf(IsType(repo.FullNodeRepoType{}), ConfigFullNode(c)), - ApplyIf(IsType(repo.StorageMinerRepoType{}), ConfigStorageMiner(c)), + ApplyIf(IsType(repo.FullNode), ConfigFullNode(c)), + ApplyIf(IsType(repo.StorageMiner), ConfigStorageMiner(c)), )(settings) } } diff --git a/node/builder_chain.go b/node/builder_chain.go index e0d959bd6..0d10dcb9b 100644 --- a/node/builder_chain.go +++ b/node/builder_chain.go @@ -248,7 +248,7 @@ func Lite(enable bool) FullOption { func FullAPI(out *api.FullNode, fopts ...FullOption) Option { return Options( func(s *Settings) error { - s.nodeType = repo.FullNodeRepoType{} + s.nodeType = repo.FullNode s.enableLibp2pNode = true return nil }, diff --git a/node/builder_miner.go b/node/builder_miner.go index b689ab5e0..71204b3c6 100644 --- a/node/builder_miner.go +++ b/node/builder_miner.go @@ -221,7 +221,7 @@ func StorageMiner(out *api.StorageMiner, subsystemsCfg config.MinerSubsystemConf ), func(s *Settings) error { - s.nodeType = repo.StorageMinerRepoType{} + s.nodeType = repo.StorageMiner s.enableLibp2pNode = subsystemsCfg.EnableMarkets return nil }, diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 92eba78ae..e5f8918ab 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -44,11 +44,11 @@ const ( func NewRepoTypeFromString(t string) RepoType { switch t { case "FullNode": - return FullNodeRepoType{} + return FullNode case "StorageMiner": - return StorageMinerRepoType{} + return StorageMiner case "Worker": - return WorkerRepoType{} + return Worker case "Wallet": return WalletRepoType{} default: @@ -61,47 +61,55 @@ type RepoType interface { Config() interface{} } -type FullNodeRepoType struct { +var FullNode fullNode + +type fullNode struct { } -func (f FullNodeRepoType) Type() string { +func (f fullNode) Type() string { return "FullNode" } -func (f FullNodeRepoType) Config() interface{} { +func (f fullNode) Config() interface{} { return config.DefaultFullNode() } -type StorageMinerRepoType struct { +var StorageMiner storageMiner + +type storageMiner struct { } -func (f StorageMinerRepoType) Type() string { +func (f storageMiner) Type() string { return "StorageMiner" } -func (f StorageMinerRepoType) Config() interface{} { +func (f storageMiner) Config() interface{} { return config.DefaultStorageMiner() } -type MarketsRepoType struct { +var Markets markets + +type markets struct { } -func (f MarketsRepoType) Type() string { +func (f markets) Type() string { return "Markets" } -func (f MarketsRepoType) Config() interface{} { +func (f markets) Config() interface{} { return config.DefaultStorageMiner() } -type WorkerRepoType struct { +type worker struct { } -func (f WorkerRepoType) Type() string { +var Worker worker + +func (f worker) Type() string { return "Worker" } -func (f WorkerRepoType) Config() interface{} { +func (f worker) Config() interface{} { return &struct{}{} } diff --git a/node/repo/fsrepo_test.go b/node/repo/fsrepo_test.go index 76241b07d..381ebdcbe 100644 --- a/node/repo/fsrepo_test.go +++ b/node/repo/fsrepo_test.go @@ -18,7 +18,7 @@ func genFsRepo(t *testing.T) (*FsRepo, func()) { t.Fatal(err) } - err = repo.Init(FullNodeRepoType{}) + err = repo.Init(FullNode) if err != ErrRepoExists && err != nil { t.Fatal(err) } diff --git a/node/repo/repo_test.go b/node/repo/repo_test.go index 55c61845c..cd19f86f6 100644 --- a/node/repo/repo_test.go +++ b/node/repo/repo_test.go @@ -21,12 +21,12 @@ func basicTest(t *testing.T, repo Repo) { } assert.Nil(t, apima, "with no api endpoint, return should be nil") - lrepo, err := repo.Lock(FullNodeRepoType{}) + lrepo, err := repo.Lock(FullNode) assert.NoError(t, err, "should be able to lock once") assert.NotNil(t, lrepo, "locked repo shouldn't be nil") { - lrepo2, err := repo.Lock(FullNodeRepoType{}) + lrepo2, err := repo.Lock(FullNode) if assert.Error(t, err) { assert.Equal(t, ErrRepoAlreadyLocked, err) } @@ -36,7 +36,7 @@ func basicTest(t *testing.T, repo Repo) { err = lrepo.Close() assert.NoError(t, err, "should be able to unlock") - lrepo, err = repo.Lock(FullNodeRepoType{}) + lrepo, err = repo.Lock(FullNode) assert.NoError(t, err, "should be able to relock") assert.NotNil(t, lrepo, "locked repo shouldn't be nil") @@ -80,7 +80,7 @@ func basicTest(t *testing.T, repo Repo) { k1 := types.KeyInfo{Type: "foo"} k2 := types.KeyInfo{Type: "bar"} - lrepo, err = repo.Lock(FullNodeRepoType{}) + lrepo, err = repo.Lock(FullNode) assert.NoError(t, err, "should be able to relock") assert.NotNil(t, lrepo, "locked repo shouldn't be nil") diff --git a/testplans/lotus-soup/testkit/role_miner.go b/testplans/lotus-soup/testkit/role_miner.go index e3d7c7d62..7204c71fe 100644 --- a/testplans/lotus-soup/testkit/role_miner.go +++ b/testplans/lotus-soup/testkit/role_miner.go @@ -138,7 +138,7 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) { return nil, err } - err = minerRepo.Init(repo.StorageMinerRepoType{}) + err = minerRepo.Init(repo.StorageMiner) if err != nil { return nil, err } @@ -146,7 +146,7 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) { var subsystems config.MinerSubsystemConfig { - lr, err := minerRepo.Lock(repo.StorageMinerRepoType{}) + lr, err := minerRepo.Lock(repo.StorageMiner) if err != nil { return nil, err } @@ -244,7 +244,7 @@ func PrepareMiner(t *TestEnvironment) (*LotusMiner, error) { return nil, err } - err = nodeRepo.Init(repo.FullNodeRepoType{}) + err = nodeRepo.Init(repo.FullNode) if err != nil { return nil, err } From 949ec6511813d47adc535980540e419c035e5853 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 16 Feb 2022 12:29:16 +0100 Subject: [PATCH 03/13] drop defConfForType --- cmd/lotus-wallet/main.go | 2 +- node/repo/fsrepo.go | 18 ++++++++---------- node/repo/memrepo.go | 12 ++---------- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/cmd/lotus-wallet/main.go b/cmd/lotus-wallet/main.go index d51175c0d..91f23d092 100644 --- a/cmd/lotus-wallet/main.go +++ b/cmd/lotus-wallet/main.go @@ -283,7 +283,7 @@ func openRepo(cctx *cli.Context) (repo.LockedRepo, types.KeyStore, error) { } } - lr, err := r.Lock(repo.WalletRepoType{}) + lr, err := r.Lock(repo.Wallet) if err != nil { return nil, nil, err } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index e5f8918ab..ef91c650b 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -50,7 +50,7 @@ func NewRepoTypeFromString(t string) RepoType { case "Worker": return Worker case "Wallet": - return WalletRepoType{} + return Wallet default: panic("unknown RepoType") } @@ -113,21 +113,19 @@ func (f worker) Config() interface{} { return &struct{}{} } -type WalletRepoType struct { +var Wallet wallet + +type wallet struct { } -func (f WalletRepoType) Type() string { +func (f wallet) Type() string { return "Wallet" } -func (f WalletRepoType) Config() interface{} { +func (f wallet) Config() interface{} { return &struct{}{} } -func defConfForType(t RepoType) interface{} { - return t.Config() -} - var log = logging.Logger("repo") var ErrRepoExists = xerrors.New("repo exists") @@ -209,7 +207,7 @@ func (fsr *FsRepo) initConfig(t RepoType) error { return err } - comm, err := config.ConfigComment(defConfForType(t)) + comm, err := config.ConfigComment(t.Config()) if err != nil { return xerrors.Errorf("comment: %w", err) } @@ -450,7 +448,7 @@ func (fsr *fsLockedRepo) Config() (interface{}, error) { } func (fsr *fsLockedRepo) loadConfigFromDisk() (interface{}, error) { - return config.FromFile(fsr.configPath, defConfForType(fsr.repoType)) + return config.FromFile(fsr.configPath, fsr.repoType.Config()) } func (fsr *fsLockedRepo) SetConfig(c func(interface{})) error { diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index 9feef684a..a8e2d020f 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -36,9 +36,6 @@ type MemRepo struct { keystore map[string]types.KeyInfo blockstore blockstore.Blockstore - // given a repo type, produce the default config - configF func(t RepoType) interface{} - // holds the current config value config struct { sync.Mutex @@ -144,7 +141,6 @@ var _ Repo = &MemRepo{} // MemRepoOptions contains options for memory repo type MemRepoOptions struct { Ds datastore.Datastore - ConfigF func(RepoType) interface{} KeyStore map[string]types.KeyInfo } @@ -155,9 +151,6 @@ func NewMemory(opts *MemRepoOptions) *MemRepo { if opts == nil { opts = &MemRepoOptions{} } - if opts.ConfigF == nil { - opts.ConfigF = defConfForType - } if opts.Ds == nil { opts.Ds = dssync.MutexWrap(datastore.NewMapDatastore()) } @@ -169,7 +162,6 @@ func NewMemory(opts *MemRepoOptions) *MemRepo { repoLock: make(chan struct{}, 1), blockstore: blockstore.WrapIDStore(blockstore.NewMemorySync()), datastore: opts.Ds, - configF: opts.ConfigF, keystore: opts.KeyStore, } } @@ -284,7 +276,7 @@ func (lmem *lockedMemRepo) Config() (interface{}, error) { defer lmem.mem.config.Unlock() if lmem.mem.config.val == nil { - lmem.mem.config.val = lmem.mem.configF(lmem.t) + lmem.mem.config.val = lmem.t.Config() } return lmem.mem.config.val, nil @@ -299,7 +291,7 @@ func (lmem *lockedMemRepo) SetConfig(c func(interface{})) error { defer lmem.mem.config.Unlock() if lmem.mem.config.val == nil { - lmem.mem.config.val = lmem.mem.configF(lmem.t) + lmem.mem.config.val = lmem.t.Config() } c(lmem.mem.config.val) From 066d0edf51babdd59f2be32cfef60b15478e6419 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 16 Feb 2022 13:28:00 +0100 Subject: [PATCH 04/13] add SupportsStagingDeals trait --- node/repo/fsrepo.go | 33 +++++++++++++++++++-------------- node/repo/memrepo.go | 2 +- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index ef91c650b..d8074e532 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -61,42 +61,47 @@ type RepoType interface { Config() interface{} } +// SupportsStagingDeals is a trait for services that support staging deals +type SupportsStagingDeals interface { + SupportsStagingDeals() +} + var FullNode fullNode type fullNode struct { } -func (f fullNode) Type() string { +func (fullNode) Type() string { return "FullNode" } -func (f fullNode) Config() interface{} { +func (fullNode) Config() interface{} { return config.DefaultFullNode() } var StorageMiner storageMiner -type storageMiner struct { -} +type storageMiner struct{} -func (f storageMiner) Type() string { +func (storageMiner) SupportsStagingDeals() {} + +func (storageMiner) Type() string { return "StorageMiner" } -func (f storageMiner) Config() interface{} { +func (storageMiner) Config() interface{} { return config.DefaultStorageMiner() } var Markets markets -type markets struct { -} +type markets struct{} -func (f markets) Type() string { +func (markets) Type() string { return "Markets" } -func (f markets) Config() interface{} { +func (markets) Config() interface{} { return config.DefaultStorageMiner() } @@ -105,11 +110,11 @@ type worker struct { var Worker worker -func (f worker) Type() string { +func (worker) Type() string { return "Worker" } -func (f worker) Config() interface{} { +func (worker) Config() interface{} { return &struct{}{} } @@ -118,11 +123,11 @@ var Wallet wallet type wallet struct { } -func (f wallet) Type() string { +func (wallet) Type() string { return "Wallet" } -func (f wallet) Config() interface{} { +func (wallet) Config() interface{} { return &struct{}{} } diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index a8e2d020f..977eec8b6 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -103,7 +103,7 @@ func (lmem *lockedMemRepo) Path() string { panic(err) // only used in tests, probably fine } - if lmem.t.Type() == "StorageMiner" || lmem.t.Type() == "Boost" { + if _, ok := lmem.t.(SupportsStagingDeals); ok { // this is required due to the method makeDealStaging from cmd/lotus-storage-miner/init.go // deal-staging is the directory deal files are staged in before being sealed into sectors // for offline deal flow. From acbfc879aaad3506e3b0de2a2e8111a9831a912e Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 23 Feb 2022 16:21:35 +0100 Subject: [PATCH 05/13] config: extract DealmakingConfig and Sealing config from Lotus for Boost (#8172) * refactor sealingCfg and dealmakingCfg to be able to abstract Config * fix error messages --- cli/util/api.go | 4 +- itests/sector_finalize_early_test.go | 2 +- node/config/doc_gen.go | 16 ++ node/config/types.go | 26 ++++ node/modules/storageminer.go | 213 ++++++++++++++++++--------- 5 files changed, 185 insertions(+), 76 deletions(-) diff --git a/cli/util/api.go b/cli/util/api.go index 70051cbe0..5283b8431 100644 --- a/cli/util/api.go +++ b/cli/util/api.go @@ -175,13 +175,13 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { } } - return APIInfo{}, fmt.Errorf("could not determine API endpoint for node type: %v", t) + return APIInfo{}, fmt.Errorf("could not determine API endpoint for node type: %v", t.Type()) } func GetRawAPI(ctx *cli.Context, t repo.RepoType, version string) (string, http.Header, error) { ainfo, err := GetAPIInfo(ctx, t) if err != nil { - return "", nil, xerrors.Errorf("could not get API info for %s: %w", t, err) + return "", nil, xerrors.Errorf("could not get API info for %s: %w", t.Type(), err) } addr, err := ainfo.DialArgs(version) diff --git a/itests/sector_finalize_early_test.go b/itests/sector_finalize_early_test.go index 233bc8fcb..0f0fcdec6 100644 --- a/itests/sector_finalize_early_test.go +++ b/itests/sector_finalize_early_test.go @@ -39,7 +39,7 @@ func TestDealsWithFinalizeEarly(t *testing.T) { return func() (sealiface.Config, error) { cf := config.DefaultStorageMiner() cf.Sealing.FinalizeEarly = true - return modules.ToSealingConfig(cf), nil + return modules.ToSealingConfig(cf.Dealmaking, cf.Sealing), nil }, nil })))) // no mock proofs. ens.InterconnectAll().BeginMining(blockTime) diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 2a7f63180..8a5146033 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -340,6 +340,14 @@ see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-f Comment: ``, }, }, + "DealmakingConfiger interface": []DocField{ + { + Name: "GetDealmakingConfig()", + Type: "DealmakingConfig", + + Comment: ``, + }, + }, "FeeConfig": []DocField{ { Name: "DefaultMaxFee", @@ -798,6 +806,14 @@ submitting proofs to the chain individually`, Comment: ``, }, }, + "SealingConfiger interface": []DocField{ + { + Name: "GetSealingConfig()", + Type: "SealingConfig", + + Comment: ``, + }, + }, "Splitstore": []DocField{ { Name: "ColdStoreType", diff --git a/node/config/types.go b/node/config/types.go index b073735d3..5a9563302 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -52,6 +52,32 @@ type StorageMiner struct { DAGStore DAGStoreConfig } +type DealmakingConfiger interface { + GetDealmakingConfig() DealmakingConfig + SetDealmakingConfig(DealmakingConfig) +} + +func (c *StorageMiner) GetDealmakingConfig() DealmakingConfig { + return c.Dealmaking +} + +func (c *StorageMiner) SetDealmakingConfig(other DealmakingConfig) { + c.Dealmaking = other +} + +type SealingConfiger interface { + GetSealingConfig() SealingConfig + SetSealingConfig(SealingConfig) +} + +func (c *StorageMiner) GetSealingConfig() SealingConfig { + return c.Sealing +} + +func (c *StorageMiner) SetSealingConfig(other SealingConfig) { + c.Sealing = other +} + type DAGStoreConfig struct { // Path to the dagstore root directory. This directory contains three // subdirectories, which can be symlinked to alternative locations if diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index f4a2364fd..7cafaf83d 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -758,8 +758,9 @@ func StorageAuthWithURL(apiInfo string) func(ctx helpers.MetricsCtx, ca v0api.Co func NewConsiderOnlineStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOnlineStorageDealsConfigFunc, error) { return func() (out bool, err error) { - err = readCfg(r, func(cfg *config.StorageMiner) { - out = cfg.Dealmaking.ConsiderOnlineStorageDeals + err = readDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + out = cfg.ConsiderOnlineStorageDeals }) return }, nil @@ -767,8 +768,10 @@ func NewConsiderOnlineStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.Consider func NewSetConsideringOnlineStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsiderOnlineStorageDealsConfigFunc, error) { return func(b bool) (err error) { - err = mutateCfg(r, func(cfg *config.StorageMiner) { - cfg.Dealmaking.ConsiderOnlineStorageDeals = b + err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + cfg.ConsiderOnlineStorageDeals = b + c.SetDealmakingConfig(cfg) }) return }, nil @@ -776,8 +779,9 @@ func NewSetConsideringOnlineStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsi func NewConsiderOnlineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOnlineRetrievalDealsConfigFunc, error) { return func() (out bool, err error) { - err = readCfg(r, func(cfg *config.StorageMiner) { - out = cfg.Dealmaking.ConsiderOnlineRetrievalDeals + err = readDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + out = cfg.ConsiderOnlineRetrievalDeals }) return }, nil @@ -785,8 +789,10 @@ func NewConsiderOnlineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.Consid func NewSetConsiderOnlineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.SetConsiderOnlineRetrievalDealsConfigFunc, error) { return func(b bool) (err error) { - err = mutateCfg(r, func(cfg *config.StorageMiner) { - cfg.Dealmaking.ConsiderOnlineRetrievalDeals = b + err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + cfg.ConsiderOnlineRetrievalDeals = b + c.SetDealmakingConfig(cfg) }) return }, nil @@ -794,8 +800,9 @@ func NewSetConsiderOnlineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.Set func NewStorageDealPieceCidBlocklistConfigFunc(r repo.LockedRepo) (dtypes.StorageDealPieceCidBlocklistConfigFunc, error) { return func() (out []cid.Cid, err error) { - err = readCfg(r, func(cfg *config.StorageMiner) { - out = cfg.Dealmaking.PieceCidBlocklist + err = readDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + out = cfg.PieceCidBlocklist }) return }, nil @@ -803,8 +810,10 @@ func NewStorageDealPieceCidBlocklistConfigFunc(r repo.LockedRepo) (dtypes.Storag func NewSetStorageDealPieceCidBlocklistConfigFunc(r repo.LockedRepo) (dtypes.SetStorageDealPieceCidBlocklistConfigFunc, error) { return func(blocklist []cid.Cid) (err error) { - err = mutateCfg(r, func(cfg *config.StorageMiner) { - cfg.Dealmaking.PieceCidBlocklist = blocklist + err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + cfg.PieceCidBlocklist = blocklist + c.SetDealmakingConfig(cfg) }) return }, nil @@ -812,8 +821,9 @@ func NewSetStorageDealPieceCidBlocklistConfigFunc(r repo.LockedRepo) (dtypes.Set func NewConsiderOfflineStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOfflineStorageDealsConfigFunc, error) { return func() (out bool, err error) { - err = readCfg(r, func(cfg *config.StorageMiner) { - out = cfg.Dealmaking.ConsiderOfflineStorageDeals + err = readDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + out = cfg.ConsiderOfflineStorageDeals }) return }, nil @@ -821,8 +831,10 @@ func NewConsiderOfflineStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.Conside func NewSetConsideringOfflineStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsiderOfflineStorageDealsConfigFunc, error) { return func(b bool) (err error) { - err = mutateCfg(r, func(cfg *config.StorageMiner) { - cfg.Dealmaking.ConsiderOfflineStorageDeals = b + err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + cfg.ConsiderOfflineStorageDeals = b + c.SetDealmakingConfig(cfg) }) return }, nil @@ -830,8 +842,9 @@ func NewSetConsideringOfflineStorageDealsFunc(r repo.LockedRepo) (dtypes.SetCons func NewConsiderOfflineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderOfflineRetrievalDealsConfigFunc, error) { return func() (out bool, err error) { - err = readCfg(r, func(cfg *config.StorageMiner) { - out = cfg.Dealmaking.ConsiderOfflineRetrievalDeals + err = readDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + out = cfg.ConsiderOfflineRetrievalDeals }) return }, nil @@ -839,8 +852,10 @@ func NewConsiderOfflineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.Consi func NewSetConsiderOfflineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.SetConsiderOfflineRetrievalDealsConfigFunc, error) { return func(b bool) (err error) { - err = mutateCfg(r, func(cfg *config.StorageMiner) { - cfg.Dealmaking.ConsiderOfflineRetrievalDeals = b + err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + cfg.ConsiderOfflineRetrievalDeals = b + c.SetDealmakingConfig(cfg) }) return }, nil @@ -848,8 +863,9 @@ func NewSetConsiderOfflineRetrievalDealsConfigFunc(r repo.LockedRepo) (dtypes.Se func NewConsiderVerifiedStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderVerifiedStorageDealsConfigFunc, error) { return func() (out bool, err error) { - err = readCfg(r, func(cfg *config.StorageMiner) { - out = cfg.Dealmaking.ConsiderVerifiedStorageDeals + err = readDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + out = cfg.ConsiderVerifiedStorageDeals }) return }, nil @@ -857,8 +873,10 @@ func NewConsiderVerifiedStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.Consid func NewSetConsideringVerifiedStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsiderVerifiedStorageDealsConfigFunc, error) { return func(b bool) (err error) { - err = mutateCfg(r, func(cfg *config.StorageMiner) { - cfg.Dealmaking.ConsiderVerifiedStorageDeals = b + err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + cfg.ConsiderVerifiedStorageDeals = b + c.SetDealmakingConfig(cfg) }) return }, nil @@ -866,8 +884,9 @@ func NewSetConsideringVerifiedStorageDealsFunc(r repo.LockedRepo) (dtypes.SetCon func NewConsiderUnverifiedStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.ConsiderUnverifiedStorageDealsConfigFunc, error) { return func() (out bool, err error) { - err = readCfg(r, func(cfg *config.StorageMiner) { - out = cfg.Dealmaking.ConsiderUnverifiedStorageDeals + err = readDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + out = cfg.ConsiderUnverifiedStorageDeals }) return }, nil @@ -875,8 +894,10 @@ func NewConsiderUnverifiedStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.Cons func NewSetConsideringUnverifiedStorageDealsFunc(r repo.LockedRepo) (dtypes.SetConsiderUnverifiedStorageDealsConfigFunc, error) { return func(b bool) (err error) { - err = mutateCfg(r, func(cfg *config.StorageMiner) { - cfg.Dealmaking.ConsiderUnverifiedStorageDeals = b + err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + cfg.ConsiderUnverifiedStorageDeals = b + c.SetDealmakingConfig(cfg) }) return }, nil @@ -884,8 +905,8 @@ func NewSetConsideringUnverifiedStorageDealsFunc(r repo.LockedRepo) (dtypes.SetC func NewSetSealConfigFunc(r repo.LockedRepo) (dtypes.SetSealingConfigFunc, error) { return func(cfg sealiface.Config) (err error) { - err = mutateCfg(r, func(c *config.StorageMiner) { - c.Sealing = config.SealingConfig{ + err = mutateSealingCfg(r, func(c config.SealingConfiger) { + newCfg := config.SealingConfig{ MaxWaitDealsSectors: cfg.MaxWaitDealsSectors, MaxSealingSectors: cfg.MaxSealingSectors, MaxSealingSectorsForDeals: cfg.MaxSealingSectorsForDeals, @@ -915,50 +936,53 @@ func NewSetSealConfigFunc(r repo.LockedRepo) (dtypes.SetSealingConfigFunc, error TerminateBatchMin: cfg.TerminateBatchMin, TerminateBatchWait: config.Duration(cfg.TerminateBatchWait), } + c.SetSealingConfig(newCfg) }) return }, nil } -func ToSealingConfig(cfg *config.StorageMiner) sealiface.Config { +func ToSealingConfig(dealmakingCfg config.DealmakingConfig, sealingCfg config.SealingConfig) sealiface.Config { return sealiface.Config{ - MaxWaitDealsSectors: cfg.Sealing.MaxWaitDealsSectors, - MaxSealingSectors: cfg.Sealing.MaxSealingSectors, - MaxSealingSectorsForDeals: cfg.Sealing.MaxSealingSectorsForDeals, - StartEpochSealingBuffer: abi.ChainEpoch(cfg.Dealmaking.StartEpochSealingBuffer), - MakeNewSectorForDeals: cfg.Dealmaking.MakeNewSectorForDeals, - CommittedCapacitySectorLifetime: time.Duration(cfg.Sealing.CommittedCapacitySectorLifetime), - WaitDealsDelay: time.Duration(cfg.Sealing.WaitDealsDelay), - AlwaysKeepUnsealedCopy: cfg.Sealing.AlwaysKeepUnsealedCopy, - FinalizeEarly: cfg.Sealing.FinalizeEarly, + MaxWaitDealsSectors: sealingCfg.MaxWaitDealsSectors, + MaxSealingSectors: sealingCfg.MaxSealingSectors, + MaxSealingSectorsForDeals: sealingCfg.MaxSealingSectorsForDeals, + StartEpochSealingBuffer: abi.ChainEpoch(dealmakingCfg.StartEpochSealingBuffer), + MakeNewSectorForDeals: dealmakingCfg.MakeNewSectorForDeals, + CommittedCapacitySectorLifetime: time.Duration(sealingCfg.CommittedCapacitySectorLifetime), + WaitDealsDelay: time.Duration(sealingCfg.WaitDealsDelay), + AlwaysKeepUnsealedCopy: sealingCfg.AlwaysKeepUnsealedCopy, + FinalizeEarly: sealingCfg.FinalizeEarly, - CollateralFromMinerBalance: cfg.Sealing.CollateralFromMinerBalance, - AvailableBalanceBuffer: types.BigInt(cfg.Sealing.AvailableBalanceBuffer), - DisableCollateralFallback: cfg.Sealing.DisableCollateralFallback, + CollateralFromMinerBalance: sealingCfg.CollateralFromMinerBalance, + AvailableBalanceBuffer: types.BigInt(sealingCfg.AvailableBalanceBuffer), + DisableCollateralFallback: sealingCfg.DisableCollateralFallback, - BatchPreCommits: cfg.Sealing.BatchPreCommits, - MaxPreCommitBatch: cfg.Sealing.MaxPreCommitBatch, - PreCommitBatchWait: time.Duration(cfg.Sealing.PreCommitBatchWait), - PreCommitBatchSlack: time.Duration(cfg.Sealing.PreCommitBatchSlack), + BatchPreCommits: sealingCfg.BatchPreCommits, + MaxPreCommitBatch: sealingCfg.MaxPreCommitBatch, + PreCommitBatchWait: time.Duration(sealingCfg.PreCommitBatchWait), + PreCommitBatchSlack: time.Duration(sealingCfg.PreCommitBatchSlack), - AggregateCommits: cfg.Sealing.AggregateCommits, - MinCommitBatch: cfg.Sealing.MinCommitBatch, - MaxCommitBatch: cfg.Sealing.MaxCommitBatch, - CommitBatchWait: time.Duration(cfg.Sealing.CommitBatchWait), - CommitBatchSlack: time.Duration(cfg.Sealing.CommitBatchSlack), - AggregateAboveBaseFee: types.BigInt(cfg.Sealing.AggregateAboveBaseFee), - BatchPreCommitAboveBaseFee: types.BigInt(cfg.Sealing.BatchPreCommitAboveBaseFee), + AggregateCommits: sealingCfg.AggregateCommits, + MinCommitBatch: sealingCfg.MinCommitBatch, + MaxCommitBatch: sealingCfg.MaxCommitBatch, + CommitBatchWait: time.Duration(sealingCfg.CommitBatchWait), + CommitBatchSlack: time.Duration(sealingCfg.CommitBatchSlack), + AggregateAboveBaseFee: types.BigInt(sealingCfg.AggregateAboveBaseFee), + BatchPreCommitAboveBaseFee: types.BigInt(sealingCfg.BatchPreCommitAboveBaseFee), - TerminateBatchMax: cfg.Sealing.TerminateBatchMax, - TerminateBatchMin: cfg.Sealing.TerminateBatchMin, - TerminateBatchWait: time.Duration(cfg.Sealing.TerminateBatchWait), + TerminateBatchMax: sealingCfg.TerminateBatchMax, + TerminateBatchMin: sealingCfg.TerminateBatchMin, + TerminateBatchWait: time.Duration(sealingCfg.TerminateBatchWait), } } func NewGetSealConfigFunc(r repo.LockedRepo) (dtypes.GetSealingConfigFunc, error) { return func() (out sealiface.Config, err error) { - err = readCfg(r, func(cfg *config.StorageMiner) { - out = ToSealingConfig(cfg) + err = readSealingCfg(r, func(dc config.DealmakingConfiger, sc config.SealingConfiger) { + scfg := sc.GetSealingConfig() + dcfg := dc.GetDealmakingConfig() + out = ToSealingConfig(dcfg, scfg) }) return }, nil @@ -966,8 +990,10 @@ func NewGetSealConfigFunc(r repo.LockedRepo) (dtypes.GetSealingConfigFunc, error func NewSetExpectedSealDurationFunc(r repo.LockedRepo) (dtypes.SetExpectedSealDurationFunc, error) { return func(delay time.Duration) (err error) { - err = mutateCfg(r, func(cfg *config.StorageMiner) { - cfg.Dealmaking.ExpectedSealDuration = config.Duration(delay) + err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + cfg.ExpectedSealDuration = config.Duration(delay) + c.SetDealmakingConfig(cfg) }) return }, nil @@ -975,8 +1001,9 @@ func NewSetExpectedSealDurationFunc(r repo.LockedRepo) (dtypes.SetExpectedSealDu func NewGetExpectedSealDurationFunc(r repo.LockedRepo) (dtypes.GetExpectedSealDurationFunc, error) { return func() (out time.Duration, err error) { - err = readCfg(r, func(cfg *config.StorageMiner) { - out = time.Duration(cfg.Dealmaking.ExpectedSealDuration) + err = readDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + out = time.Duration(cfg.ExpectedSealDuration) }) return }, nil @@ -984,8 +1011,10 @@ func NewGetExpectedSealDurationFunc(r repo.LockedRepo) (dtypes.GetExpectedSealDu func NewSetMaxDealStartDelayFunc(r repo.LockedRepo) (dtypes.SetMaxDealStartDelayFunc, error) { return func(delay time.Duration) (err error) { - err = mutateCfg(r, func(cfg *config.StorageMiner) { - cfg.Dealmaking.MaxDealStartDelay = config.Duration(delay) + err = mutateDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + cfg.MaxDealStartDelay = config.Duration(delay) + c.SetDealmakingConfig(cfg) }) return }, nil @@ -993,22 +1022,60 @@ func NewSetMaxDealStartDelayFunc(r repo.LockedRepo) (dtypes.SetMaxDealStartDelay func NewGetMaxDealStartDelayFunc(r repo.LockedRepo) (dtypes.GetMaxDealStartDelayFunc, error) { return func() (out time.Duration, err error) { - err = readCfg(r, func(cfg *config.StorageMiner) { - out = time.Duration(cfg.Dealmaking.MaxDealStartDelay) + err = readDealmakingCfg(r, func(c config.DealmakingConfiger) { + cfg := c.GetDealmakingConfig() + out = time.Duration(cfg.MaxDealStartDelay) }) return }, nil } -func readCfg(r repo.LockedRepo, accessor func(*config.StorageMiner)) error { +func readSealingCfg(r repo.LockedRepo, accessor func(config.DealmakingConfiger, config.SealingConfiger)) error { raw, err := r.Config() if err != nil { return err } - cfg, ok := raw.(*config.StorageMiner) + scfg, ok := raw.(config.SealingConfiger) if !ok { - return xerrors.New("expected address of config.StorageMiner") + return xerrors.New("expected config with sealing config trait") + } + + dcfg, ok := raw.(config.DealmakingConfiger) + if !ok { + return xerrors.New("expected config with dealmaking config trait") + } + + accessor(dcfg, scfg) + + return nil +} + +func mutateSealingCfg(r repo.LockedRepo, mutator func(config.SealingConfiger)) error { + var typeErr error + + setConfigErr := r.SetConfig(func(raw interface{}) { + cfg, ok := raw.(config.SealingConfiger) + if !ok { + typeErr = errors.New("expected config with sealing config trait") + return + } + + mutator(cfg) + }) + + return multierr.Combine(typeErr, setConfigErr) +} + +func readDealmakingCfg(r repo.LockedRepo, accessor func(config.DealmakingConfiger)) error { + raw, err := r.Config() + if err != nil { + return err + } + + cfg, ok := raw.(config.DealmakingConfiger) + if !ok { + return xerrors.New("expected config with dealmaking config trait") } accessor(cfg) @@ -1016,13 +1083,13 @@ func readCfg(r repo.LockedRepo, accessor func(*config.StorageMiner)) error { return nil } -func mutateCfg(r repo.LockedRepo, mutator func(*config.StorageMiner)) error { +func mutateDealmakingCfg(r repo.LockedRepo, mutator func(config.DealmakingConfiger)) error { var typeErr error setConfigErr := r.SetConfig(func(raw interface{}) { - cfg, ok := raw.(*config.StorageMiner) + cfg, ok := raw.(config.DealmakingConfiger) if !ok { - typeErr = errors.New("expected miner config") + typeErr = errors.New("expected config with dealmaking config trait") return } From e1c2cf446ec0d455906afece755deee065a6a151 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 14 Jan 2022 15:22:10 +0100 Subject: [PATCH 06/13] add `lotus-test` image as CI build step --- .circleci/config.yml | 13 +++++++++++++ Dockerfile.lotus | 45 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 26345edf9..41b99a858 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -628,6 +628,11 @@ jobs: default: "latest" description: A comma-separated string containing docker image tags to build and push (default = latest) + target: + type: string + default: "lotus-all-in-one" + description: Docker target to build + steps: - run: name: Confirm that environment variables are set @@ -667,6 +672,7 @@ jobs: docker build \ <<#parameters.extra-build-args>><><> \ + --target <> \ -f <>/<> \ $docker_tag_args \ <> @@ -1031,7 +1037,14 @@ workflows: dockerfile: Dockerfile.lotus path: . repo: lotus-dev + target: lotus-all-in-one tag: '${CIRCLE_SHA1:0:8}' + - build-and-push-image: + dockerfile: Dockerfile.lotus + path: . + repo: lotus-test + tag: '${CIRCLE_SHA1:0:8}' + target: lotus-test - publish-packer-mainnet: requires: - build-all diff --git a/Dockerfile.lotus b/Dockerfile.lotus index 812ad9f61..dd37c5273 100644 --- a/Dockerfile.lotus +++ b/Dockerfile.lotus @@ -28,6 +28,14 @@ WORKDIR /opt/filecoin RUN make clean deps +FROM builder-local AS builder-test +MAINTAINER Lotus Development Team + +WORKDIR /opt/filecoin + +RUN make debug + + FROM builder-local AS builder MAINTAINER Lotus Development Team @@ -197,6 +205,43 @@ RUN chown fc: /var/lib/lotus-worker RUN chown fc: /var/lib/lotus-wallet +VOLUME /var/tmp/filecoin-proof-parameters +VOLUME /var/lib/lotus +VOLUME /var/lib/lotus-miner +VOLUME /var/lib/lotus-worker +VOLUME /var/lib/lotus-wallet + +EXPOSE 1234 +EXPOSE 2345 +EXPOSE 3456 +EXPOSE 1777 + +### +from base as lotus-test + +ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters +ENV LOTUS_MINER_PATH /var/lib/lotus-miner +ENV LOTUS_PATH /var/lib/lotus +ENV LOTUS_WORKER_PATH /var/lib/lotus-worker +ENV WALLET_PATH /var/lib/lotus-wallet + +COPY --from=builder-test /opt/filecoin/lotus /usr/local/bin/ +COPY --from=builder-test /opt/filecoin/lotus-miner /usr/local/bin/ +COPY --from=builder-test /opt/filecoin/lotus-worker /usr/local/bin/ +COPY --from=builder-test /opt/filecoin/lotus-seed /usr/local/bin/ + +RUN mkdir /var/tmp/filecoin-proof-parameters +RUN mkdir /var/lib/lotus +RUN mkdir /var/lib/lotus-miner +RUN mkdir /var/lib/lotus-worker +RUN mkdir /var/lib/lotus-wallet +RUN chown fc: /var/tmp/filecoin-proof-parameters +RUN chown fc: /var/lib/lotus +RUN chown fc: /var/lib/lotus-miner +RUN chown fc: /var/lib/lotus-worker +RUN chown fc: /var/lib/lotus-wallet + + VOLUME /var/tmp/filecoin-proof-parameters VOLUME /var/lib/lotus VOLUME /var/lib/lotus-miner From 62e352ef887cfdfa8fbb9db339021a499bfd0e56 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 3 Mar 2022 16:45:11 +0100 Subject: [PATCH 07/13] move APIInfoEnvVars, RepoFlags, APIFlags under RepoType --- cli/auth.go | 3 +- cli/util/api.go | 65 ++---------------------------------- node/repo/fsrepo.go | 80 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 64 deletions(-) diff --git a/cli/auth.go b/cli/auth.go index e6eeace89..caea4cb42 100644 --- a/cli/auth.go +++ b/cli/auth.go @@ -9,7 +9,6 @@ import ( "github.com/filecoin-project/go-jsonrpc/auth" "github.com/filecoin-project/lotus/api" - cliutil "github.com/filecoin-project/lotus/cli/util" "github.com/filecoin-project/lotus/node/repo" ) @@ -128,7 +127,7 @@ var AuthApiInfoToken = &cli.Command{ // TODO: Log in audit log when it is implemented - currentEnv, _, _ := cliutil.EnvsForAPIInfos(t) + currentEnv, _, _ := t.APIInfoEnvVars() fmt.Printf("%s=%s:%s\n", currentEnv, string(token), ainfo.Addr) return nil }, diff --git a/cli/util/api.go b/cli/util/api.go index 5283b8431..6c673d91f 100644 --- a/cli/util/api.go +++ b/cli/util/api.go @@ -28,63 +28,6 @@ const ( metadataTraceContext = "traceContext" ) -// flagsForAPI returns flags passed on the command line with the listen address -// of the API server (only used by the tests), in the order of precedence they -// should be applied for the requested kind of node. -func flagsForAPI(t repo.RepoType) []string { - switch t.Type() { - case "FullNode": - return []string{"api-url"} - case "StorageMiner": - return []string{"miner-api-url"} - case "Worker": - return []string{"worker-api-url"} - case "Markets": - // support split markets-miner and monolith deployments. - return []string{"markets-api-url", "miner-api-url"} - default: - panic(fmt.Sprintf("Unknown repo type: %v", t)) - } -} - -func flagsForRepo(t repo.RepoType) []string { - switch t.Type() { - case "FullNode": - return []string{"repo"} - case "StorageMiner": - return []string{"miner-repo"} - case "Worker": - return []string{"worker-repo"} - case "Markets": - // support split markets-miner and monolith deployments. - return []string{"markets-repo", "miner-repo"} - default: - panic(fmt.Sprintf("Unknown repo type: %v", t)) - } -} - -// EnvsForAPIInfos returns the environment variables to use in order of precedence -// to determine the API endpoint of the specified node type. -// -// It returns the current variables and deprecated ones separately, so that -// the user can log a warning when deprecated ones are found to be in use. -func EnvsForAPIInfos(t repo.RepoType) (primary string, fallbacks []string, deprecated []string) { - switch t.Type() { - case "FullNode": - return "FULLNODE_API_INFO", nil, nil - case "StorageMiner": - // TODO remove deprecated deprecation period - return "MINER_API_INFO", nil, []string{"STORAGE_API_INFO"} - case "Worker": - return "WORKER_API_INFO", nil, nil - case "Markets": - // support split markets-miner and monolith deployments. - return "MARKETS_API_INFO", []string{"MINER_API_INFO"}, nil - default: - panic(fmt.Sprintf("Unknown repo type: %v", t)) - } -} - // GetAPIInfo returns the API endpoint to use for the specified kind of repo. // // The order of precedence is as follows: @@ -96,8 +39,7 @@ func EnvsForAPIInfos(t repo.RepoType) (primary string, fallbacks []string, depre func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { // Check if there was a flag passed with the listen address of the API // server (only used by the tests) - apiFlags := flagsForAPI(t) - for _, f := range apiFlags { + for _, f := range t.APIFlags() { if !ctx.IsSet(f) { continue } @@ -111,7 +53,7 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { // Note: it is not correct/intuitive to prefer environment variables over // CLI flags (repo flags below). // - primaryEnv, fallbacksEnvs, deprecatedEnvs := EnvsForAPIInfos(t) + primaryEnv, fallbacksEnvs, deprecatedEnvs := t.APIInfoEnvVars() env, ok := os.LookupEnv(primaryEnv) if ok { return ParseApiInfo(env), nil @@ -125,8 +67,7 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { } } - repoFlags := flagsForRepo(t) - for _, f := range repoFlags { + for _, f := range t.RepoFlags() { // cannot use ctx.IsSet because it ignores default values path := ctx.String(f) if path == "" { diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index d8074e532..9688a518b 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -59,6 +59,20 @@ func NewRepoTypeFromString(t string) RepoType { type RepoType interface { Type() string Config() interface{} + + // APIFlags returns flags passed on the command line with the listen address + // of the API server (only used by the tests), in the order of precedence they + // should be applied for the requested kind of node. + APIFlags() []string + + RepoFlags() []string + + // APIInfoEnvVars returns the environment variables to use in order of precedence + // to determine the API endpoint of the specified node type. + // + // It returns the current variables and deprecated ones separately, so that + // the user can log a warning when deprecated ones are found to be in use. + APIInfoEnvVars() (string, []string, []string) } // SupportsStagingDeals is a trait for services that support staging deals @@ -79,6 +93,18 @@ func (fullNode) Config() interface{} { return config.DefaultFullNode() } +func (fullNode) APIFlags() []string { + return []string{"api-url"} +} + +func (fullNode) RepoFlags() []string { + return []string{"repo"} +} + +func (fullNode) APIInfoEnvVars() (primary string, fallbacks []string, deprecated []string) { + return "FULLNODE_API_INFO", nil, nil +} + var StorageMiner storageMiner type storageMiner struct{} @@ -93,10 +119,25 @@ func (storageMiner) Config() interface{} { return config.DefaultStorageMiner() } +func (storageMiner) APIFlags() []string { + return []string{"miner-api-url"} +} + +func (storageMiner) RepoFlags() []string { + return []string{"miner-repo"} +} + +func (storageMiner) APIInfoEnvVars() (primary string, fallbacks []string, deprecated []string) { + // TODO remove deprecated deprecation period + return "MINER_API_INFO", nil, []string{"STORAGE_API_INFO"} +} + var Markets markets type markets struct{} +func (markets) SupportsStagingDeals() {} + func (markets) Type() string { return "Markets" } @@ -105,6 +146,21 @@ func (markets) Config() interface{} { return config.DefaultStorageMiner() } +func (markets) APIFlags() []string { + // support split markets-miner and monolith deployments. + return []string{"markets-api-url", "miner-api-url"} +} + +func (markets) RepoFlags() []string { + // support split markets-miner and monolith deployments. + return []string{"markets-repo", "miner-repo"} +} + +func (markets) APIInfoEnvVars() (primary string, fallbacks []string, deprecated []string) { + // support split markets-miner and monolith deployments. + return "MARKETS_API_INFO", []string{"MINER_API_INFO"}, nil +} + type worker struct { } @@ -118,6 +174,18 @@ func (worker) Config() interface{} { return &struct{}{} } +func (worker) APIFlags() []string { + return []string{"worker-api-url"} +} + +func (worker) RepoFlags() []string { + return []string{"worker-repo"} +} + +func (worker) APIInfoEnvVars() (primary string, fallbacks []string, deprecated []string) { + return "WORKER_API_INFO", nil, nil +} + var Wallet wallet type wallet struct { @@ -131,6 +199,18 @@ func (wallet) Config() interface{} { return &struct{}{} } +func (wallet) APIFlags() []string { + panic("not supported") +} + +func (wallet) RepoFlags() []string { + panic("not supported") +} + +func (wallet) APIInfoEnvVars() (primary string, fallbacks []string, deprecated []string) { + panic("not supported") +} + var log = logging.Logger("repo") var ErrRepoExists = xerrors.New("repo exists") From 5734a39e8e681e82aa883a3ab2d223b39229bacf Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 3 Mar 2022 16:51:45 +0100 Subject: [PATCH 08/13] move DealmakingConfiger and SealingConfiger away from config/types.go --- node/config/doc_gen.go | 16 ---------------- node/config/dynamic_config.go | 27 +++++++++++++++++++++++++++ node/config/types.go | 26 -------------------------- 3 files changed, 27 insertions(+), 42 deletions(-) create mode 100644 node/config/dynamic_config.go diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 8a5146033..2a7f63180 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -340,14 +340,6 @@ see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-f Comment: ``, }, }, - "DealmakingConfiger interface": []DocField{ - { - Name: "GetDealmakingConfig()", - Type: "DealmakingConfig", - - Comment: ``, - }, - }, "FeeConfig": []DocField{ { Name: "DefaultMaxFee", @@ -806,14 +798,6 @@ submitting proofs to the chain individually`, Comment: ``, }, }, - "SealingConfiger interface": []DocField{ - { - Name: "GetSealingConfig()", - Type: "SealingConfig", - - Comment: ``, - }, - }, "Splitstore": []DocField{ { Name: "ColdStoreType", diff --git a/node/config/dynamic_config.go b/node/config/dynamic_config.go new file mode 100644 index 000000000..777ca4712 --- /dev/null +++ b/node/config/dynamic_config.go @@ -0,0 +1,27 @@ +package config + +type DealmakingConfiger interface { + GetDealmakingConfig() DealmakingConfig + SetDealmakingConfig(DealmakingConfig) +} + +func (c *StorageMiner) GetDealmakingConfig() DealmakingConfig { + return c.Dealmaking +} + +func (c *StorageMiner) SetDealmakingConfig(other DealmakingConfig) { + c.Dealmaking = other +} + +type SealingConfiger interface { + GetSealingConfig() SealingConfig + SetSealingConfig(SealingConfig) +} + +func (c *StorageMiner) GetSealingConfig() SealingConfig { + return c.Sealing +} + +func (c *StorageMiner) SetSealingConfig(other SealingConfig) { + c.Sealing = other +} diff --git a/node/config/types.go b/node/config/types.go index 5a9563302..b073735d3 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -52,32 +52,6 @@ type StorageMiner struct { DAGStore DAGStoreConfig } -type DealmakingConfiger interface { - GetDealmakingConfig() DealmakingConfig - SetDealmakingConfig(DealmakingConfig) -} - -func (c *StorageMiner) GetDealmakingConfig() DealmakingConfig { - return c.Dealmaking -} - -func (c *StorageMiner) SetDealmakingConfig(other DealmakingConfig) { - c.Dealmaking = other -} - -type SealingConfiger interface { - GetSealingConfig() SealingConfig - SetSealingConfig(SealingConfig) -} - -func (c *StorageMiner) GetSealingConfig() SealingConfig { - return c.Sealing -} - -func (c *StorageMiner) SetSealingConfig(other SealingConfig) { - c.Sealing = other -} - type DAGStoreConfig struct { // Path to the dagstore root directory. This directory contains three // subdirectories, which can be symlinked to alternative locations if From 7cdf6dc680c53a7bed28eb9716af79c57eedee62 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 7 Mar 2022 13:00:26 +0100 Subject: [PATCH 09/13] remove .Type() now that we have singletons for repos --- node/builder.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/node/builder.go b/node/builder.go index e9a928f73..c4f858ed1 100644 --- a/node/builder.go +++ b/node/builder.go @@ -226,15 +226,15 @@ var LibP2P = Options( ) func IsType(t repo.RepoType) func(s *Settings) bool { - return func(s *Settings) bool { return s.nodeType.Type() == t.Type() } + return func(s *Settings) bool { return s.nodeType == t } } -func isFullOrLiteNode(s *Settings) bool { return s.nodeType.Type() == repo.FullNode.Type() } +func isFullOrLiteNode(s *Settings) bool { return s.nodeType == repo.FullNode } func isFullNode(s *Settings) bool { - return s.nodeType.Type() == repo.FullNode.Type() && !s.Lite + return s.nodeType == repo.FullNode && !s.Lite } func isLiteNode(s *Settings) bool { - return s.nodeType.Type() == repo.FullNode.Type() && s.Lite + return s.nodeType == repo.FullNode && s.Lite } func Base() Option { From 42b2adddee1f5d58ddb814d4fc63a22b5e1aa602 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 18 Mar 2022 10:42:21 +0100 Subject: [PATCH 10/13] fix: itests: Don't hang on exit in MineBlocksMustPost --- itests/kit/blockminer.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/itests/kit/blockminer.go b/itests/kit/blockminer.go index a232d82e0..03f5d6e34 100644 --- a/itests/kit/blockminer.go +++ b/itests/kit/blockminer.go @@ -192,7 +192,11 @@ func (bm *BlockMiner) MineBlocksMustPost(ctx context.Context, blocktime time.Dur InjectNulls: abi.ChainEpoch(nulls + i), Done: reportSuccessFn, }) - success = <-wait + select { + case success = <-wait: + case <-ctx.Done(): + return + } if !success { // if we are mining a new null block and it brings us past deadline boundary we need to wait for miner to post if ts.Height()+1+abi.ChainEpoch(nulls+i) >= dlinfo.Last() { From fee69a889b7aaac5067198307eb18bd40b833d42 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 17 Jan 2022 11:57:05 +0100 Subject: [PATCH 11/13] modify .circleci/template.yml --- .circleci/config.yml | 2 +- .circleci/template.yml | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 41b99a858..f5fe9bcc4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1037,8 +1037,8 @@ workflows: dockerfile: Dockerfile.lotus path: . repo: lotus-dev - target: lotus-all-in-one tag: '${CIRCLE_SHA1:0:8}' + target: lotus-all-in-one - build-and-push-image: dockerfile: Dockerfile.lotus path: . diff --git a/.circleci/template.yml b/.circleci/template.yml index c8643e3ac..dc3b37f2d 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -628,6 +628,11 @@ jobs: default: "latest" description: A comma-separated string containing docker image tags to build and push (default = latest) + target: + type: string + default: "lotus-all-in-one" + description: Docker target to build + steps: - run: name: Confirm that environment variables are set @@ -667,6 +672,7 @@ jobs: docker build \ <<#parameters.extra-build-args>><><> \ + --target <> \ -f <>/<> \ $docker_tag_args \ <> @@ -862,6 +868,13 @@ workflows: path: . repo: lotus-dev tag: '${CIRCLE_SHA1:0:8}' + target: lotus-all-in-one + - build-and-push-image: + dockerfile: Dockerfile.lotus + path: . + repo: lotus-test + tag: '${CIRCLE_SHA1:0:8}' + target: lotus-test - publish-packer-mainnet: requires: - build-all From 0ed8647b6f71490087361a9d4fd6b8b6ca4f2d41 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 21 Mar 2022 10:37:35 +0100 Subject: [PATCH 12/13] extract unixfs filestore into lib --- .../import.go => lib/unixfs/filestore.go | 21 +++++++++++-------- .../unixfs/filestore_test.go | 11 +++------- node/impl/client/client.go | 12 +++++------ 3 files changed, 21 insertions(+), 23 deletions(-) rename node/impl/client/import.go => lib/unixfs/filestore.go (86%) rename node/impl/client/import_test.go => lib/unixfs/filestore_test.go (93%) diff --git a/node/impl/client/import.go b/lib/unixfs/filestore.go similarity index 86% rename from node/impl/client/import.go rename to lib/unixfs/filestore.go index 367cc73c7..de9063f61 100644 --- a/node/impl/client/import.go +++ b/lib/unixfs/filestore.go @@ -1,4 +1,4 @@ -package client +package unixfs import ( "context" @@ -19,12 +19,15 @@ import ( "github.com/ipfs/go-merkledag" "github.com/ipfs/go-unixfs/importer/balanced" ihelper "github.com/ipfs/go-unixfs/importer/helpers" + mh "github.com/multiformats/go-multihash" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/build" ) -func unixFSCidBuilder() (cid.Builder, error) { +var DefaultHashFunction = uint64(mh.BLAKE2B_MIN + 31) + +func CidBuilder() (cid.Builder, error) { prefix, err := merkledag.PrefixForCidVersion(1) if err != nil { return nil, fmt.Errorf("failed to initialize UnixFS CID Builder: %w", err) @@ -37,9 +40,9 @@ func unixFSCidBuilder() (cid.Builder, error) { return b, nil } -// createUnixFSFilestore takes a standard file whose path is src, forms a UnixFS DAG, and +// CreateFilestore takes a standard file whose path is src, forms a UnixFS DAG, and // writes a CARv2 file with positional mapping (backed by the go-filestore library). -func (a *API) createUnixFSFilestore(ctx context.Context, srcPath string, dstPath string) (cid.Cid, error) { +func CreateFilestore(ctx context.Context, srcPath string, dstPath string) (cid.Cid, error) { // This method uses a two-phase approach with a staging CAR blockstore and // a final CAR blockstore. // @@ -80,7 +83,7 @@ func (a *API) createUnixFSFilestore(ctx context.Context, srcPath string, dstPath return cid.Undef, xerrors.Errorf("failed to create temporary filestore: %w", err) } - finalRoot1, err := buildUnixFS(ctx, file, fstore, true) + finalRoot1, err := Build(ctx, file, fstore, true) if err != nil { _ = fstore.Close() return cid.Undef, xerrors.Errorf("failed to import file to store to compute root: %w", err) @@ -102,7 +105,7 @@ func (a *API) createUnixFSFilestore(ctx context.Context, srcPath string, dstPath return cid.Undef, xerrors.Errorf("failed to rewind file: %w", err) } - finalRoot2, err := buildUnixFS(ctx, file, bs, true) + finalRoot2, err := Build(ctx, file, bs, true) if err != nil { _ = bs.Close() return cid.Undef, xerrors.Errorf("failed to create UnixFS DAG with carv2 blockstore: %w", err) @@ -119,10 +122,10 @@ func (a *API) createUnixFSFilestore(ctx context.Context, srcPath string, dstPath return finalRoot1, nil } -// buildUnixFS builds a UnixFS DAG out of the supplied reader, +// Build builds a UnixFS DAG out of the supplied reader, // and imports the DAG into the supplied service. -func buildUnixFS(ctx context.Context, reader io.Reader, into bstore.Blockstore, filestore bool) (cid.Cid, error) { - b, err := unixFSCidBuilder() +func Build(ctx context.Context, reader io.Reader, into bstore.Blockstore, filestore bool) (cid.Cid, error) { + b, err := CidBuilder() if err != nil { return cid.Undef, err } diff --git a/node/impl/client/import_test.go b/lib/unixfs/filestore_test.go similarity index 93% rename from node/impl/client/import_test.go rename to lib/unixfs/filestore_test.go index 06295efc7..3368b5b22 100644 --- a/node/impl/client/import_test.go +++ b/lib/unixfs/filestore_test.go @@ -1,5 +1,5 @@ //stm: #unit -package client +package unixfs import ( "bytes" @@ -21,8 +21,6 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/go-fil-markets/stores" - - "github.com/filecoin-project/lotus/node/repo/imports" ) // This test uses a full "dense" CARv2, and not a filestore (positional mapping). @@ -42,7 +40,7 @@ func TestRoundtripUnixFS_Dense(t *testing.T) { blockstore.UseWholeCIDs(true)) require.NoError(t, err) - root, err := buildUnixFS(ctx, bytes.NewBuffer(inputContents), bs, false) + root, err := Build(ctx, bytes.NewBuffer(inputContents), bs, false) require.NoError(t, err) require.NotEqual(t, cid.Undef, root) require.NoError(t, bs.Finalize()) @@ -78,9 +76,6 @@ func TestRoundtripUnixFS_Dense(t *testing.T) { func TestRoundtripUnixFS_Filestore(t *testing.T) { //stm: @CLIENT_DATA_IMPORT_001 ctx := context.Background() - a := &API{ - Imports: &imports.Manager{}, - } inputPath, inputContents := genInputFile(t) defer os.Remove(inputPath) //nolint:errcheck @@ -88,7 +83,7 @@ func TestRoundtripUnixFS_Filestore(t *testing.T) { dst := newTmpFile(t) defer os.Remove(dst) //nolint:errcheck - root, err := a.createUnixFSFilestore(ctx, inputPath, dst) + root, err := CreateFilestore(ctx, inputPath, dst) require.NoError(t, err) require.NotEqual(t, cid.Undef, root) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index f105d3152..47c069913 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -42,7 +42,6 @@ import ( "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multibase" - mh "github.com/multiformats/go-multihash" "go.uber.org/fx" "github.com/filecoin-project/go-address" @@ -56,6 +55,7 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket/network" "github.com/filecoin-project/go-fil-markets/stores" + "github.com/filecoin-project/lotus/lib/unixfs" "github.com/filecoin-project/lotus/markets/retrievaladapter" "github.com/filecoin-project/lotus/markets/storageadapter" @@ -79,7 +79,7 @@ import ( var log = logging.Logger("client") -var DefaultHashFunction = uint64(mh.BLAKE2B_MIN + 31) +var DefaultHashFunction = unixfs.DefaultHashFunction // 8 days ~= SealDuration + PreCommit + MaxProveCommitDuration + 8 hour buffer const dealStartBufferHours uint64 = 8 * 24 @@ -548,7 +548,7 @@ func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (res *api.Impor }() // perform the unixfs chunking. - root, err = a.createUnixFSFilestore(ctx, ref.Path, carPath) + root, err = unixfs.CreateFilestore(ctx, ref.Path, carPath) if err != nil { return nil, xerrors.Errorf("failed to import file using unixfs: %w", err) } @@ -618,7 +618,7 @@ func (a *API) ClientImportLocal(ctx context.Context, r io.Reader) (cid.Cid, erro // once the DAG is formed and the root is calculated, we overwrite the // inner carv1 header with the final root. - b, err := unixFSCidBuilder() + b, err := unixfs.CidBuilder() if err != nil { return cid.Undef, err } @@ -635,7 +635,7 @@ func (a *API) ClientImportLocal(ctx context.Context, r io.Reader) (cid.Cid, erro return cid.Undef, xerrors.Errorf("failed to create carv2 read/write blockstore: %w", err) } - root, err := buildUnixFS(ctx, file, bs, false) + root, err := unixfs.Build(ctx, file, bs, false) if err != nil { return cid.Undef, xerrors.Errorf("failed to build unixfs dag: %w", err) } @@ -1364,7 +1364,7 @@ func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath stri defer os.Remove(tmp) //nolint:errcheck // generate and import the UnixFS DAG into a filestore (positional reference) CAR. - root, err := a.createUnixFSFilestore(ctx, ref.Path, tmp) + root, err := unixfs.CreateFilestore(ctx, ref.Path, tmp) if err != nil { return xerrors.Errorf("failed to import file using unixfs: %w", err) } From 7fea2e4d6c2435658f2f942bcddd2f3cd8ebda1b Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 21 Mar 2022 12:33:07 +0200 Subject: [PATCH 13/13] update go-libp2p to v0.18 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 03450afdb..3f066006f 100644 --- a/go.mod +++ b/go.mod @@ -112,7 +112,7 @@ require ( github.com/kelseyhightower/envconfig v1.4.0 github.com/libp2p/go-buffer-pool v0.0.2 github.com/libp2p/go-eventbus v0.2.1 - github.com/libp2p/go-libp2p v0.18.0-rc6 + github.com/libp2p/go-libp2p v0.18.0 github.com/libp2p/go-libp2p-connmgr v0.3.1 github.com/libp2p/go-libp2p-core v0.14.0 github.com/libp2p/go-libp2p-discovery v0.6.0 diff --git a/go.sum b/go.sum index 79629f1b5..f0f1e5868 100644 --- a/go.sum +++ b/go.sum @@ -1065,8 +1065,8 @@ github.com/libp2p/go-libp2p v0.16.0/go.mod h1:ump42BsirwAWxKzsCiFnTtN1Yc+DuPu76f github.com/libp2p/go-libp2p v0.17.0/go.mod h1:Fkin50rsGdv5mm5BshBUtPRZknt9esfmYXBOYcwOTgw= github.com/libp2p/go-libp2p v0.18.0-rc1/go.mod h1:RgYlH7IIWHXREimC92bw5Lg1V2R5XmSzuLHb5fTnr+8= github.com/libp2p/go-libp2p v0.18.0-rc3/go.mod h1:WYL+Xw1iuwi6rdfzw5VIEpD+HqzYucHZ6fcUuumbI3M= -github.com/libp2p/go-libp2p v0.18.0-rc6 h1:IR6TVPYGo1wDY0tY61gyPQVxK1koOkXh49ejVfAnH7A= -github.com/libp2p/go-libp2p v0.18.0-rc6/go.mod h1:oOUOAlBrm1L0+jxT10h2TMUMTDz6pV3EvmkJ3beDYGQ= +github.com/libp2p/go-libp2p v0.18.0 h1:moKKKG875KNGsCjZxTIFB75ihHiVjFeWg5I4aR1pDLk= +github.com/libp2p/go-libp2p v0.18.0/go.mod h1:+veaZ9z1SZQhmc5PW78jvnnxZ89Mgvmh4cggO11ETmw= github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo= github.com/libp2p/go-libp2p-asn-util v0.1.0 h1:rABPCO77SjdbJ/eJ/ynIo8vWICy1VEnL5JAxJbQLo1E= github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I=