Merge pull request #837 from filecoin-project/feat/symlink-import-preseal

Feat/symlink import preseal

cmd/lotus-seed/main.go

@@ -268,7 +268,7 @@ var aggregateSectorDirsCmd = &cli.Command{
 				return err
 			}
 
-			if err := agsb.ImportFrom(sb); err != nil {
+			if err := agsb.ImportFrom(sb, false); err != nil {
 				return xerrors.Errorf("importing sectors from %q failed: %w", dir, err)
 			}
 		}

cmd/lotus-storage-miner/init.go

@@ -76,12 +76,21 @@ var initCmd = &cli.Command{
 			Name:  "nosync",
 			Usage: "don't check full-node sync status",
 		},
+		&cli.BoolFlag{
+			Name:  "symlink-imported-sectors",
+			Usage: "attempt to symlink to presealed sectors instead of copying them into place",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		log.Info("Initializing lotus storage miner")
 
 		ssize := cctx.Uint64("sector-size")
 
+		symlink := cctx.Bool("symlink-imported-sectors")
+		if symlink {
+			log.Info("will attempt to symlink to imported sectors")
+		}
+
 		log.Info("Checking proof parameters")
 		if err := build.GetParams(ssize); err != nil {
 			return xerrors.Errorf("fetching proof parameters: %w", err)
@@ -184,7 +193,7 @@ var initCmd = &cli.Command{
 				return xerrors.Errorf("failed to open up sectorbuilder: %w", err)
 			}
 
-			if err := nsb.ImportFrom(oldsb); err != nil {
+			if err := nsb.ImportFrom(oldsb, symlink); err != nil {
 				return err
 			}
 			if err := lr.Close(); err != nil {

lib/sectorbuilder/sectorbuilder.go

@@ -3,7 +3,9 @@ package sectorbuilder
 import (
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -681,16 +683,16 @@ func fallbackPostChallengeCount(sectors uint64) uint64 {
 	return challengeCount
 }
 
-func (sb *SectorBuilder) ImportFrom(osb *SectorBuilder) error {
-	if err := dcopy.Copy(osb.cacheDir, sb.cacheDir); err != nil {
+func (sb *SectorBuilder) ImportFrom(osb *SectorBuilder, symlink bool) error {
+	if err := migrate(osb.cacheDir, sb.cacheDir, true); err != nil {
 		return err
 	}
 
-	if err := dcopy.Copy(osb.sealedDir, sb.sealedDir); err != nil {
+	if err := migrate(osb.sealedDir, sb.sealedDir, true); err != nil {
 		return err
 	}
 
-	if err := dcopy.Copy(osb.stagedDir, sb.stagedDir); err != nil {
+	if err := migrate(osb.stagedDir, sb.stagedDir, true); err != nil {
 		return err
 	}
 
@@ -716,3 +718,58 @@ func (sb *SectorBuilder) SetLastSectorID(id uint64) error {
 	sb.lastID = id
 	return nil
 }
+
+func migrate(from, to string, symlink bool) error {
+	st, err := os.Stat(from)
+	if err != nil {
+		return err
+	}
+
+	if st.IsDir() {
+		return migrateDir(from, to, symlink)
+	}
+	return migrateFile(from, to, symlink)
+}
+
+func migrateDir(from, to string, symlink bool) error {
+	tost, err := os.Stat(to)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+
+		if err := os.MkdirAll(to, 0755); err != nil {
+			return err
+		}
+	} else if !tost.IsDir() {
+		return xerrors.Errorf("target %q already exists and is a file (expected directory)")
+	}
+
+	dirents, err := ioutil.ReadDir(from)
+	if err != nil {
+		return err
+	}
+
+	for _, inf := range dirents {
+		n := inf.Name()
+		if inf.IsDir() {
+			if err := migrate(filepath.Join(from, n), filepath.Join(to, n), symlink); err != nil {
+				return err
+			}
+		} else {
+			if err := migrate(filepath.Join(from, n), filepath.Join(to, n), symlink); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func migrateFile(from, to string, symlink bool) error {
+	if symlink {
+		return os.Symlink(from, to)
+	}
+
+	return dcopy.Copy(from, to)
+}

node/node_test.go

@@ -238,7 +238,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
 			t.Fatal(err)
 		}
 
-		if err := sma.SectorBuilder.ImportFrom(osb); err != nil {
+		if err := sma.SectorBuilder.ImportFrom(osb, false); err != nil {
 			t.Fatal(err)
 		}