From 9c6d531ae73891ce29b9355010bb2b80f9dfcee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Sep 2022 11:06:30 +0200 Subject: [PATCH] sector import: Plumb sector download task canfig --- cmd/lotus-worker/main.go | 9 +++++++++ documentation/en/cli-lotus-worker.md | 1 + documentation/en/default-lotus-miner-config.toml | 6 ++++-- itests/kit/ensemble.go | 1 + itests/sector_import_test.go | 3 +-- node/config/def.go | 1 + node/config/doc_gen.go | 8 +++++++- node/config/storage.go | 1 + node/config/types.go | 2 +- storage/sealer/manager.go | 4 ++++ 10 files changed, 30 insertions(+), 6 deletions(-) diff --git a/cmd/lotus-worker/main.go b/cmd/lotus-worker/main.go index 286cce883..8b8db5aa7 100644 --- a/cmd/lotus-worker/main.go +++ b/cmd/lotus-worker/main.go @@ -224,6 +224,12 @@ var runCmd = &cli.Command{ Value: true, EnvVars: []string{"LOTUS_WORKER_REGEN_SECTOR_KEY"}, }, + &cli.BoolFlag{ + Name: "sector-download", + Usage: "enable external sector data download", + Value: false, + EnvVars: []string{"LOTUS_WORKER_SECTOR_DOWNLOAD"}, + }, &cli.BoolFlag{ Name: "windowpost", Usage: "enable window post", @@ -373,6 +379,9 @@ var runCmd = &cli.Command{ if (workerType == sealtasks.WorkerSealing || cctx.IsSet("addpiece")) && cctx.Bool("addpiece") { taskTypes = append(taskTypes, sealtasks.TTAddPiece, sealtasks.TTDataCid) } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("sector-download")) && cctx.Bool("sector-download") { + taskTypes = append(taskTypes, sealtasks.TTDownloadSector) + } if (workerType == sealtasks.WorkerSealing || cctx.IsSet("precommit1")) && cctx.Bool("precommit1") { taskTypes = append(taskTypes, sealtasks.TTPreCommit1) } diff --git a/documentation/en/cli-lotus-worker.md b/documentation/en/cli-lotus-worker.md index ec81d28d7..920fb3001 100644 --- a/documentation/en/cli-lotus-worker.md +++ b/documentation/en/cli-lotus-worker.md @@ -53,6 +53,7 @@ OPTIONS: --prove-replica-update2 enable prove replica update 2 (default: true) [$LOTUS_WORKER_PROVE_REPLICA_UPDATE2] --regen-sector-key enable regen sector key (default: true) [$LOTUS_WORKER_REGEN_SECTOR_KEY] --replica-update enable replica update (default: true) [$LOTUS_WORKER_REPLICA_UPDATE] + --sector-download enable external sector data download (default: false) [$LOTUS_WORKER_SECTOR_DOWNLOAD] --timeout value used when 'listen' is unspecified. must be a valid duration recognized by golang's time.ParseDuration function (default: "30m") [$LOTUS_WORKER_TIMEOUT] --unseal enable unsealing (32G sectors: 1 core, 128GiB Memory) (default: true) [$LOTUS_WORKER_UNSEAL] --windowpost enable window post (default: false) [$LOTUS_WORKER_WINDOWPOST] diff --git a/documentation/en/default-lotus-miner-config.toml b/documentation/en/default-lotus-miner-config.toml index ae235e01e..0147f979e 100644 --- a/documentation/en/default-lotus-miner-config.toml +++ b/documentation/en/default-lotus-miner-config.toml @@ -612,8 +612,10 @@ # env var: LOTUS_STORAGE_PARALLELFETCHLIMIT #ParallelFetchLimit = 10 - # Local worker config - # + # type: bool + # env var: LOTUS_STORAGE_ALLOWSECTORDOWNLOAD + #AllowSectorDownload = true + # type: bool # env var: LOTUS_STORAGE_ALLOWADDPIECE #AllowAddPiece = true diff --git a/itests/kit/ensemble.go b/itests/kit/ensemble.go index bcc2d93c2..8fa781e65 100644 --- a/itests/kit/ensemble.go +++ b/itests/kit/ensemble.go @@ -636,6 +636,7 @@ func (n *Ensemble) Start() *Ensemble { scfg := config.DefaultStorageMiner() if noLocal { + scfg.Storage.AllowSectorDownload = false scfg.Storage.AllowAddPiece = false scfg.Storage.AllowPreCommit1 = false scfg.Storage.AllowPreCommit2 = false diff --git a/itests/sector_import_test.go b/itests/sector_import_test.go index 848d8f8e6..e820d9d80 100644 --- a/itests/sector_import_test.go +++ b/itests/sector_import_test.go @@ -161,8 +161,7 @@ func TestSectorImportAfterPC2(t *testing.T) { require.Len(t, ng, 1) require.Equal(t, snum, ng[0]) - // todo wait sealed - + miner.WaitSectorsProving(ctx, map[abi.SectorNumber]struct{}{snum: {}}) } func remoteGetSector(sectorRoot string) func(w http.ResponseWriter, r *http.Request) { diff --git a/node/config/def.go b/node/config/def.go index ad2150af3..0566c7d99 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -146,6 +146,7 @@ func DefaultStorageMiner() *StorageMiner { }, Storage: SealerConfig{ + AllowSectorDownload: true, AllowAddPiece: true, AllowPreCommit1: true, AllowPreCommit2: true, diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 6b5927448..3d90afd0b 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -796,11 +796,17 @@ This parameter is ONLY applicable if the retrieval pricing policy strategy has b Comment: ``, }, + { + Num: "AllowSectorDownload", + Type: "bool", + + Comment: ``, + }, { Name: "AllowAddPiece", Type: "bool", - Comment: `Local worker config`, + Comment: ``, }, { Name: "AllowPreCommit1", diff --git a/node/config/storage.go b/node/config/storage.go index 20f1791e6..0c72eabd1 100644 --- a/node/config/storage.go +++ b/node/config/storage.go @@ -54,6 +54,7 @@ func WriteStorageFile(path string, config paths.StorageConfig) error { func (c *StorageMiner) StorageManager() sealer.Config { return sealer.Config{ ParallelFetchLimit: c.Storage.ParallelFetchLimit, + AllowSectorDownload: c.Storage.AllowSectorDownload, AllowAddPiece: c.Storage.AllowAddPiece, AllowPreCommit1: c.Storage.AllowPreCommit1, AllowPreCommit2: c.Storage.AllowPreCommit2, diff --git a/node/config/types.go b/node/config/types.go index 3c85587c3..1f55da1c6 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -409,7 +409,7 @@ type SealingConfig struct { type SealerConfig struct { ParallelFetchLimit int - // Local worker config + AllowSectorDownload bool AllowAddPiece bool AllowPreCommit1 bool AllowPreCommit2 bool diff --git a/storage/sealer/manager.go b/storage/sealer/manager.go index 78c0622f4..a4d31e21b 100644 --- a/storage/sealer/manager.go +++ b/storage/sealer/manager.go @@ -108,6 +108,7 @@ type Config struct { ParallelFetchLimit int // Local worker config + AllowSectorDownload bool AllowAddPiece bool AllowPreCommit1 bool AllowPreCommit2 bool @@ -182,6 +183,9 @@ func New(ctx context.Context, lstor *paths.Local, stor paths.Store, ls paths.Loc localTasks := []sealtasks.TaskType{ sealtasks.TTCommit1, sealtasks.TTProveReplicaUpdate1, sealtasks.TTFinalize, sealtasks.TTFetch, sealtasks.TTFinalizeReplicaUpdate, } + if sc.AllowSectorDownload { + localTasks = append(localTasks, sealtasks.TTDownloadSector) + } if sc.AllowAddPiece { localTasks = append(localTasks, sealtasks.TTAddPiece, sealtasks.TTDataCid) }