From 08cc689293d5bc931ca353249cd05588a0ffff9d Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 10 Dec 2019 18:11:59 +0100 Subject: [PATCH 1/3] add option to symlink to presealed sectors --- cmd/lotus-storage-miner/init.go | 11 +++++- lib/sectorbuilder/sectorbuilder.go | 59 +++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 50c338f63..204103522 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -76,12 +76,21 @@ var initCmd = &cli.Command{ Name: "nosync", Usage: "don't check full-node sync status", }, + &cli.BoolFlag{ + Name: "symlink-imported-sectors", + Usage: "attempt to symlink to presealed sectors instead of copying them into place", + }, }, Action: func(cctx *cli.Context) error { log.Info("Initializing lotus storage miner") ssize := cctx.Uint64("sector-size") + symlink := cctx.Bool("symlink-imported-sectors") + if symlink { + log.Info("will attempt to symlink to imported sectors") + } + log.Info("Checking proof parameters") if err := build.GetParams(ssize); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) @@ -184,7 +193,7 @@ var initCmd = &cli.Command{ return xerrors.Errorf("failed to open up sectorbuilder: %w", err) } - if err := nsb.ImportFrom(oldsb); err != nil { + if err := nsb.ImportFrom(oldsb, symlink); err != nil { return err } if err := lr.Close(); err != nil { diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 5dcb2e551..ec14362a0 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -3,7 +3,9 @@ package sectorbuilder import ( "fmt" "io" + "io/ioutil" "os" + "path/filepath" "strconv" "sync" "sync/atomic" @@ -681,7 +683,7 @@ func fallbackPostChallengeCount(sectors uint64) uint64 { return challengeCount } -func (sb *SectorBuilder) ImportFrom(osb *SectorBuilder) error { +func (sb *SectorBuilder) ImportFrom(osb *SectorBuilder, symlink bool) error { if err := dcopy.Copy(osb.cacheDir, sb.cacheDir); err != nil { return err } @@ -716,3 +718,58 @@ func (sb *SectorBuilder) SetLastSectorID(id uint64) error { sb.lastID = id return nil } + +func migrate(from, to string, symlink bool) error { + st, err := os.Stat(from) + if err != nil { + return err + } + + if st.IsDir() { + return migrateDir(from, to, symlink) + } + return migrateFile(from, to, symlink) +} + +func migrateDir(from, to string, symlink bool) error { + tost, err := os.Stat(to) + if err != nil { + if !os.IsNotExist(err) { + return err + } + + if err := os.MkdirAll(to, 0755); err != nil { + return err + } + } else if !tost.IsDir() { + return xerrors.Errorf("target %q already exists and is a file (expected directory)") + } + + dirents, err := ioutil.ReadDir(from) + if err != nil { + return err + } + + for _, inf := range dirents { + n := inf.Name() + if inf.IsDir() { + if err := migrate(filepath.Join(from, n), filepath.Join(to, n), symlink); err != nil { + return err + } + } else { + if err := migrate(filepath.Join(from, n), filepath.Join(to, n), symlink); err != nil { + return err + } + } + } + + return nil +} + +func migrateFile(from, to string, symlink bool) error { + if symlink { + return os.Symlink(from, to) + } + + return dcopy.Copy(from, to) +} From 4d86a4bd4a241e5d3ac287d106c26af2549221a8 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 10 Dec 2019 09:32:50 -0800 Subject: [PATCH 2/3] fixup --- cmd/lotus-seed/main.go | 2 +- lib/sectorbuilder/sectorbuilder.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/lotus-seed/main.go b/cmd/lotus-seed/main.go index 8bb4ace73..754030ade 100644 --- a/cmd/lotus-seed/main.go +++ b/cmd/lotus-seed/main.go @@ -268,7 +268,7 @@ var aggregateSectorDirsCmd = &cli.Command{ return err } - if err := agsb.ImportFrom(sb); err != nil { + if err := agsb.ImportFrom(sb, false); err != nil { return xerrors.Errorf("importing sectors from %q failed: %w", dir, err) } } diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index ec14362a0..9a401cc97 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -684,15 +684,15 @@ func fallbackPostChallengeCount(sectors uint64) uint64 { } func (sb *SectorBuilder) ImportFrom(osb *SectorBuilder, symlink bool) error { - if err := dcopy.Copy(osb.cacheDir, sb.cacheDir); err != nil { + if err := migrate(osb.cacheDir, sb.cacheDir, true); err != nil { return err } - if err := dcopy.Copy(osb.sealedDir, sb.sealedDir); err != nil { + if err := migrate(osb.sealedDir, sb.sealedDir, true); err != nil { return err } - if err := dcopy.Copy(osb.stagedDir, sb.stagedDir); err != nil { + if err := migrate(osb.stagedDir, sb.stagedDir, true); err != nil { return err } From 88f46d45399a99559fb7d114390408ba7c2c00e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 10 Dec 2019 19:13:45 +0100 Subject: [PATCH 3/3] Fix node_test --- node/node_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/node_test.go b/node/node_test.go index 2cfc2b95c..c8ea659bf 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -238,7 +238,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te t.Fatal(err) } - if err := sma.SectorBuilder.ImportFrom(osb); err != nil { + if err := sma.SectorBuilder.ImportFrom(osb, false); err != nil { t.Fatal(err) }