From 3daf9103a8b1303ae11f38c1b82477978355c6f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 4 Dec 2019 20:44:15 +0100 Subject: [PATCH] paramfetch: Only pull necessary params --- build/paramfetch.go | 11 ++++------- cli/params.go | 12 ++++-------- cmd/lotus-bench/main.go | 2 +- cmd/lotus-seed/seed/seed.go | 2 +- cmd/lotus-storage-miner/init.go | 8 +++++--- cmd/lotus-storage-miner/run.go | 4 ---- cmd/lotus/daemon.go | 2 +- lib/sectorbuilder/sectorbuilder_test.go | 6 +++--- node/builder.go | 2 ++ node/modules/storageminer.go | 8 ++++++++ 10 files changed, 29 insertions(+), 28 deletions(-) diff --git a/build/paramfetch.go b/build/paramfetch.go index b167f1ca5..1bb0f4b25 100644 --- a/build/paramfetch.go +++ b/build/paramfetch.go @@ -39,7 +39,7 @@ type fetch struct { errs []error } -func GetParams(storage bool, tests bool) error { +func GetParams(storageSize uint64) error { if err := os.Mkdir(paramdir, 0755); err != nil && !os.IsExist(err) { return err } @@ -54,10 +54,7 @@ func GetParams(storage bool, tests bool) error { ft := &fetch{} for name, info := range params { - if !(SupportedSectorSize(info.SectorSize) || (tests && info.SectorSize == 1<<10)) { - continue - } - if !storage && strings.HasSuffix(name, ".params") { + if storageSize != info.SectorSize && strings.HasSuffix(name, ".params") { continue } @@ -83,8 +80,8 @@ func (ft *fetch) maybeFetchAsync(name string, info paramFile) { return } - ft.fetchLk.Lock() - defer ft.fetchLk.Unlock() + /*ft.fetchLk.Lock() + defer ft.fetchLk.Unlock()*/ if err := doFetch(path, info); err != nil { ft.errs = append(ft.errs, xerrors.Errorf("fetching file %s failed: %w", path, err)) diff --git a/cli/params.go b/cli/params.go index eabee8d5e..df7e240ec 100644 --- a/cli/params.go +++ b/cli/params.go @@ -10,17 +10,13 @@ var fetchParamCmd = &cli.Command{ Name: "fetch-params", Usage: "Fetch proving parameters", Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "only-verify-keys", - Usage: "only download the verify keys", - }, - &cli.BoolFlag{ - Name: "include-test-params", - Usage: "download params used for tests", + &cli.Uint64Flag{ + Name: "proving-params", + Usage: "download params used creating proofs for given size", }, }, Action: func(cctx *cli.Context) error { - err := build.GetParams(!cctx.Bool("only-verify-keys"), cctx.Bool("include-test-params")) + err := build.GetParams(cctx.Uint64("proving-params")) if err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 33fdfcbf2..7964186f2 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -113,7 +113,7 @@ func main() { } } - if err := build.GetParams(true, false); err != nil { + if err := build.GetParams(sectorSize); err != nil { return xerrors.Errorf("getting params: %w", err) } sb, err := sectorbuilder.New(cfg, mds) diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index d52c1b32c..88b79ec3d 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -48,7 +48,7 @@ func PreSeal(maddr address.Address, ssize uint64, sectors int, sbroot string, pr return nil, err } - if err := build.GetParams(true, false); err != nil { + if err := build.GetParams(ssize); err != nil { return nil, xerrors.Errorf("getting params: %w", err) } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 05889b012..7ae748c5f 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -75,8 +75,10 @@ var initCmd = &cli.Command{ Action: func(cctx *cli.Context) error { log.Info("Initializing lotus storage miner") + ssize := cctx.Uint64("sector-size") + log.Info("Checking proof parameters") - if err := build.GetParams(true, false); err != nil { + if err := build.GetParams(ssize); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } @@ -152,7 +154,7 @@ var initCmd = &cli.Command{ } oldsb, err := sectorbuilder.New(§orbuilder.Config{ - SectorSize: 1024, + SectorSize: ssize, WorkerThreads: 2, SealedDir: filepath.Join(pssb, "sealed"), CacheDir: filepath.Join(pssb, "cache"), @@ -164,7 +166,7 @@ var initCmd = &cli.Command{ } nsb, err := sectorbuilder.New(§orbuilder.Config{ - SectorSize: 1024, + SectorSize: ssize, WorkerThreads: 2, SealedDir: filepath.Join(lr.Path(), "sealed"), CacheDir: filepath.Join(lr.Path(), "cache"), diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 2498fe419..3d833290d 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -41,10 +41,6 @@ var runCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { - if err := build.GetParams(true, false); err != nil { - return xerrors.Errorf("fetching proof parameters: %w", err) - } - if !cctx.Bool("enable-gpu-proving") { os.Setenv("BELLMAN_NO_GPU", "true") } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index bbf036275..ce2cc64b9 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -63,7 +63,7 @@ var DaemonCmd = &cli.Command{ return xerrors.Errorf("repo init error: %w", err) } - if err := build.GetParams(false, false); err != nil { + if err := build.GetParams(0); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index 6f97f0a34..cc8cb9563 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -126,7 +126,7 @@ func TestSealAndVerify(t *testing.T) { build.SectorSizes = []uint64{sectorSize} - if err := build.GetParams(true, true); err != nil { + if err := build.GetParams(sectorSize); err != nil { t.Fatalf("%+v", err) } @@ -195,7 +195,7 @@ func TestSealPoStNoCommit(t *testing.T) { build.SectorSizes = []uint64{sectorSize} - if err := build.GetParams(true, true); err != nil { + if err := build.GetParams(sectorSize); err != nil { t.Fatalf("%+v", err) } @@ -257,7 +257,7 @@ func TestSealAndVerify2(t *testing.T) { build.SectorSizes = []uint64{sectorSize} - if err := build.GetParams(true, true); err != nil { + if err := build.GetParams(sectorSize); err != nil { t.Fatalf("%+v", err) } diff --git a/node/builder.go b/node/builder.go index bc8dbea59..eedacadaf 100644 --- a/node/builder.go +++ b/node/builder.go @@ -91,6 +91,7 @@ const ( RegisterClientValidatorKey // storage miner + GetParamsKey HandleDealsKey HandleRetrievalKey RunSectorServiceKey @@ -250,6 +251,7 @@ func Online() Option { Override(new(*deals.Provider), deals.NewProvider), Override(RegisterProviderValidatorKey, modules.RegisterProviderValidator), Override(HandleRetrievalKey, modules.HandleRetrieval), + Override(GetParamsKey, modules.GetParams), Override(HandleDealsKey, modules.HandleDeals), Override(new(gen.ElectionPoStProver), storage.NewElectionPoStProver), Override(new(*miner.Miner), modules.SetupBlockProducer), diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index eaa2a837a..20f06ace0 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -44,6 +44,14 @@ func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { return address.NewFromBytes(maddrb) } +func GetParams(sbc *sectorbuilder.Config) error { + if err := build.GetParams(sbc.SectorSize); err != nil { + return xerrors.Errorf("fetching proof parameters: %w", err) + } + + return nil +} + func SectorBuilderConfig(storagePath string, threads uint) func(dtypes.MetadataDS, api.FullNode) (*sectorbuilder.Config, error) { return func(ds dtypes.MetadataDS, api api.FullNode) (*sectorbuilder.Config, error) { minerAddr, err := minerAddrFromDS(ds)