Merge pull request #766 from filecoin-project/feat/holes

Pre-sealing holes

.circleci/config.yml

@@ -68,7 +68,7 @@ jobs:
       - run: sudo apt-get install npm
       - restore_cache:
           name: restore go mod cache
-          key: v1-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.mod" }}
+          key: v1-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/lotus/go.mod" }}
       - run:
           command: make buildall
       - store_artifacts:
@@ -116,7 +116,7 @@ jobs:
       - go/mod-download
       - restore_cache:
           name: restore go mod cache
-          key: v1-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.mod" }}
+          key: v1-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }}
       - run:
           command: make deps lotus
           no_output_timeout: 30m
@@ -148,7 +148,7 @@ jobs:
                   bash <(curl -s https://codecov.io/bash)
       - save_cache:
           name: save go mod cache
-          key: v1-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.mod" }}
+          key: v1-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }}
           paths:
             - "~/go/pkg"
             - "~/go/src/github.com"

chain/gen/gen.go

@@ -132,7 +132,7 @@ func NewGenerator() (*ChainGen, error) {
 		return nil, err
 	}
 
-	genm1, err := seed.PreSeal(maddr1, 1024, 1, m1temp, []byte("some randomness"))
+	genm1, err := seed.PreSeal(maddr1, 1024, 0, 1, m1temp, []byte("some randomness"))
 	if err != nil {
 		return nil, err
 	}
@@ -147,7 +147,7 @@ func NewGenerator() (*ChainGen, error) {
 		return nil, err
 	}
 
-	genm2, err := seed.PreSeal(maddr2, 1024, 1, m2temp, []byte("some randomness"))
+	genm2, err := seed.PreSeal(maddr2, 1024, 0, 1, m2temp, []byte("some randomness"))
 	if err != nil {
 		return nil, err
 	}

cmd/lotus-seed/main.go

@@ -66,6 +66,11 @@ var preSealCmd = &cli.Command{
 			Value: 1,
 			Usage: "select number of sectors to pre-seal",
 		},
+		&cli.Uint64Flag{
+			Name:  "sector-offset",
+			Value: 0,
+			Usage: "how many sector ids to skip when starting to seal",
+		},
 	},
 	Action: func(c *cli.Context) error {
 		sdir := c.String("sectorbuilder-dir")
@@ -79,7 +84,7 @@ var preSealCmd = &cli.Command{
 			return err
 		}
 
-		gm, err := seed.PreSeal(maddr, c.Uint64("sector-size"), c.Int("num-sectors"), sbroot, []byte(c.String("ticket-preimage")))
+		gm, err := seed.PreSeal(maddr, c.Uint64("sector-size"), c.Uint64("sector-offset"), c.Int("num-sectors"), sbroot, []byte(c.String("ticket-preimage")))
 		if err != nil {
 			return err
 		}

cmd/lotus-seed/seed/seed.go

@@ -26,15 +26,16 @@ import (
 
 var log = logging.Logger("preseal")
 
-func PreSeal(maddr address.Address, ssize uint64, sectors int, sbroot string, preimage []byte) (*genesis.GenesisMiner, error) {
+func PreSeal(maddr address.Address, ssize uint64, offset uint64, sectors int, sbroot string, preimage []byte) (*genesis.GenesisMiner, error) {
 	cfg := &sectorbuilder.Config{
-		Miner:         maddr,
-		SectorSize:    ssize,
-		CacheDir:      filepath.Join(sbroot, "cache"),
-		SealedDir:     filepath.Join(sbroot, "sealed"),
-		StagedDir:     filepath.Join(sbroot, "staging"),
-		UnsealedDir:   filepath.Join(sbroot, "unsealed"),
-		WorkerThreads: 2,
+		Miner:          maddr,
+		SectorSize:     ssize,
+		FallbackLastID: offset,
+		CacheDir:       filepath.Join(sbroot, "cache"),
+		SealedDir:      filepath.Join(sbroot, "sealed"),
+		StagedDir:      filepath.Join(sbroot, "staging"),
+		UnsealedDir:    filepath.Join(sbroot, "unsealed"),
+		WorkerThreads:  2,
 	}
 
 	for _, d := range []string{cfg.CacheDir, cfg.SealedDir, cfg.StagedDir, cfg.UnsealedDir} {
@@ -77,7 +78,7 @@ func PreSeal(maddr address.Address, ssize uint64, sectors int, sbroot string, pr
 			TicketBytes: trand,
 		}
 
-		fmt.Println("Piece info: ", pi)
+		fmt.Printf("sector-id: %d, piece info: %v", sid, pi)
 
 		pco, err := sb.SealPreCommit(sid, ticket, []sectorbuilder.PublicPieceInfo{pi})
 		if err != nil {

lib/sectorbuilder/sectorbuilder.go

@@ -119,12 +119,14 @@ type Config struct {
 	SectorSize uint64
 	Miner      address.Address
 
-	WorkerThreads uint8
+	WorkerThreads  uint8
+	FallbackLastID uint64
 
 	CacheDir    string
 	SealedDir   string
 	StagedDir   string
 	UnsealedDir string
+	_           struct{} // guard against nameless init
 }
 
 func New(cfg *Config, ds dtypes.MetadataDS) (*SectorBuilder, error) {
@@ -151,6 +153,7 @@ func New(cfg *Config, ds dtypes.MetadataDS) (*SectorBuilder, error) {
 		}
 		lastUsedID = uint64(i)
 	case datastore.ErrNotFound:
+		lastUsedID = cfg.FallbackLastID
 	default:
 		return nil, err
 	}

lotuspond/spawn.go

@@ -36,7 +36,7 @@ func (api *api) Spawn() (nodeInfo, error) {
 		}
 
 		sbroot := filepath.Join(dir, "preseal")
-		genm, err := seed.PreSeal(genMiner, build.SectorSizes[0], 1, sbroot, []byte("8"))
+		genm, err := seed.PreSeal(genMiner, build.SectorSizes[0], 0, 1, sbroot, []byte("8"))
 		if err != nil {
 			return nodeInfo{}, xerrors.Errorf("preseal failed: %w", err)
 		}

node/node_test.go

@@ -4,12 +4,13 @@ import (
 	"bytes"
 	"context"
 	"crypto/rand"
-	"github.com/filecoin-project/lotus/build"
 	"io/ioutil"
 	"net/http/httptest"
 	"path/filepath"
 	"testing"
 
+	"github.com/filecoin-project/lotus/build"
+
 	"github.com/libp2p/go-libp2p-core/crypto"
 
 	"github.com/ipfs/go-datastore"
@@ -162,7 +163,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
 		if err != nil {
 			t.Fatal(err)
 		}
-		genm, err := seed.PreSeal(maddr, 1024, 1, tdir, []byte("make genesis mem random"))
+		genm, err := seed.PreSeal(maddr, 1024, 0, 1, tdir, []byte("make genesis mem random"))
 		if err != nil {
 			t.Fatal(err)
 		}