sealmgr: Read only multi-path file manager

cmd/lotus-seed/seed/seed.go

@@ -12,7 +12,6 @@ import (
 	"path/filepath"
 
 	"github.com/google/uuid"
-	badger "github.com/ipfs/go-ds-badger2"
 	logging "github.com/ipfs/go-log/v2"
 	"golang.org/x/xerrors"
 
@@ -53,11 +52,6 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum
 		return nil, nil, err
 	}
 
-	mds, err := badger.NewDatastore(filepath.Join(sbroot, "badger"), nil)
-	if err != nil {
-		return nil, nil, err
-	}
-
 	sbfs := &fs.Basic{
 		Miner:  maddr,
 		NextID: offset,
@@ -140,17 +134,13 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum
 		return nil, nil, xerrors.Errorf("creating deals: %w", err)
 	}
 
-	if err := mds.Close(); err != nil {
-		return nil, nil, xerrors.Errorf("closing datastore: %w", err)
-	}
-
 	{
-		b, err := json.Marshal(&config.StorageMeta{
+		b, err := json.MarshalIndent(&config.StorageMeta{
 			ID:        uuid.New().String(),
 			Weight:    0, // read-only
 			CanCommit: false,
 			CanStore:  false,
-		})
+		}, "", "  ")
 		if err != nil {
 			return nil, nil, xerrors.Errorf("marshaling storage config: %w", err)
 		}

cmd/lotus-storage-miner/init.go

@@ -3,12 +3,14 @@ package main
 import (
 	"context"
 	"crypto/rand"
+	"encoding/binary"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"strconv"
 
+	"github.com/filecoin-project/go-sectorbuilder"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/market"
 	miner2 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
@@ -162,10 +164,7 @@ var initCmd = &cli.Command{
 			if err != nil {
 				return err
 			}
-			sc, err := lr.GetStorage()
+			var sc config.StorageConfig
-			if err != nil {
-				return xerrors.Errorf("get storage config: %w", err)
-			}
 
 			for _, psp := range pssb {
 				psp, err := homedir.Expand(psp)
@@ -181,10 +180,12 @@ var initCmd = &cli.Command{
 				return xerrors.Errorf("set storage config: %w", err)
 			}
 
-			panic("persist last sector id somehow")
+			if err := lr.Close(); err != nil {
+				return err
+			}
 		}
 
-		if err := storageMinerInit(ctx, cctx, api, r); err != nil {
+		if err := storageMinerInit(ctx, cctx, api, r, ssize); err != nil {
 			log.Errorf("Failed to initialize lotus-storage-miner: %+v", err)
 			path, err := homedir.Expand(repoPath)
 			if err != nil {
@@ -220,6 +221,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, metadata string,
 		return xerrors.Errorf("unmarshaling preseal metadata: %w", err)
 	}
 
+	maxSectorID := abi.SectorNumber(0)
 	for _, sector := range meta.Sectors {
 		sectorKey := datastore.NewKey(sealing.SectorStorePrefix).ChildString(fmt.Sprint(sector.SectorID))
 
@@ -258,6 +260,10 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, metadata string,
 			return err
 		}
 
+		if sector.SectorID > maxSectorID {
+			maxSectorID = sector.SectorID
+		}
+
 		/* // TODO: Import deals into market
 		pnd, err := cborutil.AsIpld(sector.Deal)
 		if err != nil {
@@ -285,7 +291,9 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, metadata string,
 		}*/
 	}
 
-	return nil
+	buf := make([]byte, binary.MaxVarintLen64)
+	size := binary.PutUvarint(buf, uint64(maxSectorID+1))
+	return mds.Put(datastore.NewKey("/storage/nextid"), buf[:size])
 }
 
 func findMarketDealID(ctx context.Context, api lapi.FullNode, deal market.DealProposal) (abi.DealID, error) {
@@ -307,7 +315,7 @@ func findMarketDealID(ctx context.Context, api lapi.FullNode, deal market.DealPr
 	return 0, xerrors.New("deal not found")
 }
 
-func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, r repo.Repo) error {
+func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, r repo.Repo, ssize abi.SectorSize) error {
 	lr, err := r.Lock(repo.StorageMiner)
 	if err != nil {
 		return err
@@ -343,7 +351,19 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode,
 				return err
 			}
 
-			smgr := advmgr.New(lr)
+			ppt, spt, err := lapi.ProofTypeFromSectorSize(ssize)
+			if err != nil {
+				return err
+			}
+
+			smgr, err := advmgr.New(lr, &sectorbuilder.Config{
+				SealProofType: spt,
+				PoStProofType: ppt,
+				Miner:         a,
+			})
+			if err != nil {
+				return err
+			}
 			epp := storage.NewElectionPoStProver(smgr)
 
 			m := miner.NewMiner(api, epp)

documentation/en/local-dev-net.md

@@ -34,7 +34,7 @@ Then, in another console, import the genesis miner key:
 Set up the genesis miner:
 
 ```sh
-./lotus-storage-miner init --genesis-miner --actor=t01000 --sector-size=2048 --pre-sealed-sectors=~/.genesis-sectors --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t0101.json --nosync
+./lotus-storage-miner init --genesis-miner --actor=t01000 --sector-size=2048 --pre-sealed-sectors=~/.genesis-sectors --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t01000.json --nosync
 ```
 
 Now, finally, start up the miner:

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/coreos/go-systemd/v22 v22.0.0
 	github.com/docker/go-units v0.4.0
 	github.com/filecoin-project/chain-validation v0.0.3
-	github.com/filecoin-project/filecoin-ffi v0.0.0-20200226205820-4da0bccccefb
+	github.com/filecoin-project/filecoin-ffi v0.0.0-20200226231125-fc253ccb5294
 	github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be
 	github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e
 	github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
@@ -23,7 +23,8 @@ require (
 	github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663
 	github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200229022239-442fe78a3168
 	github.com/filecoin-project/go-statestore v0.1.0
-	github.com/filecoin-project/specs-actors v0.0.0-20200229011003-1d726e3afd04
+	github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf
+	github.com/filecoin-project/specs-storage v0.0.0-20200303230804-16c9a38030eb
 	github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/google/uuid v1.1.1

go.sum

@@ -130,6 +130,10 @@ github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.m
 github.com/filecoin-project/specs-actors v0.0.0-20200226200336-94c9b92b2775/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU=
 github.com/filecoin-project/specs-actors v0.0.0-20200229011003-1d726e3afd04 h1:O343OeQLkLWLj5ZqQ5nhevAGBTeB5LioiA53ddScqdY=
 github.com/filecoin-project/specs-actors v0.0.0-20200229011003-1d726e3afd04/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU=
+github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf h1:3IojVqJAD5IXMxvZ+WYx+LRbfSB/rOXpYBuHh6o3XkY=
+github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU=
+github.com/filecoin-project/specs-storage v0.0.0-20200303230804-16c9a38030eb h1:oHB9hKaD7g75NFulnfh+SCYS5bSl8hB6Eanf8A6l5tw=
+github.com/filecoin-project/specs-storage v0.0.0-20200303230804-16c9a38030eb/go.mod h1:sC2Ck2l1G8hXI5Do/3sp0yxbMRMnukbFwP9KF1CRFLw=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0=

node/config/storage.go

@@ -40,11 +40,11 @@ func StorageFromFile(path string, def *StorageConfig) (*StorageConfig, error) {
 	}
 
 	defer file.Close() //nolint:errcheck // The file is RO
-	return StorageFromReader(file, *def)
+	return StorageFromReader(file)
 }
 
-func StorageFromReader(reader io.Reader, def StorageConfig) (*StorageConfig, error) {
+func StorageFromReader(reader io.Reader) (*StorageConfig, error) {
-	cfg := def
+	var cfg StorageConfig
 	err := json.NewDecoder(reader).Decode(&cfg)
 	if err != nil {
 		return nil, err
@@ -54,7 +54,7 @@ func StorageFromReader(reader io.Reader, def StorageConfig) (*StorageConfig, err
 }
 
 func WriteStorageFile(path string, config StorageConfig) error {
-	b, err := json.Marshal(config)
+	b, err := json.MarshalIndent(config, "", "  ")
 	if err != nil {
 		return xerrors.Errorf("marshaling storage config: %w", err)
 	}

storage/sealmgr/advmgr/manager.go

@@ -3,14 +3,14 @@ package advmgr
 import (
 	"context"
 	"io"
-	"sync"
 
+	"github.com/filecoin-project/go-address"
 	"github.com/ipfs/go-cid"
+	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-sectorbuilder"
 	"github.com/filecoin-project/specs-actors/actors/abi"
 
-	ffi "github.com/filecoin-project/filecoin-ffi"
 	"github.com/filecoin-project/lotus/node/config"
 	"github.com/filecoin-project/lotus/storage/sealmgr"
 )
@@ -39,8 +39,34 @@ type Worker interface {
 type Manager struct {
 	workers []sealmgr.Worker
 
-	localLk sync.RWMutex
+	sectorbuilder.Prover
-	localStorage LocalStorage
+}
+
+func New(ls LocalStorage, cfg *sectorbuilder.Config) (*Manager, error) {
+	stor := &storage{
+		localStorage: ls,
+	}
+	if err := stor.open(); err != nil {
+		return nil, err
+	}
+
+	mid, err := address.IDFromAddress(cfg.Miner)
+	if err != nil {
+		return nil, xerrors.Errorf("getting miner id: %w", err)
+	}
+
+	prover, err := sectorbuilder.New(&readonlyProvider{stor: stor, miner: abi.ActorID(mid)}, cfg)
+	if err != nil {
+		return nil, xerrors.Errorf("creating prover instance: %w", err)
+	}
+
+	m := &Manager{
+		workers:      nil,
+
+		Prover: prover,
+	}
+
+	return m, nil
 }
 
 func (m Manager) SectorSize() abi.SectorSize {
@@ -79,24 +105,4 @@ func (m Manager) FinalizeSector(context.Context, abi.SectorNumber) error {
 	panic("implement me")
 }
 
-func (m Manager) GenerateEPostCandidates(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]ffi.PoStCandidateWithTicket, error) {
-	panic("implement me")
-}
-
-func (m Manager) GenerateFallbackPoSt(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]ffi.PoStCandidateWithTicket, []abi.PoStProof, error) {
-	panic("implement me")
-}
-
-func (m Manager) ComputeElectionPoSt(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, winners []abi.PoStCandidate) ([]abi.PoStProof, error) {
-	panic("implement me")
-}
-
-func New(ls LocalStorage) *Manager {
-	return &Manager{
-		workers:      nil,
-
-		localStorage: ls,
-	}
-}
-
 var _ sealmgr.Manager = &Manager{}

storage/sealmgr/advmgr/roprov.go

@@ -0,0 +1,28 @@
+package advmgr
+
+import (
+	"github.com/filecoin-project/go-sectorbuilder"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"golang.org/x/xerrors"
+)
+
+type readonlyProvider struct {
+	miner abi.ActorID
+	stor *storage
+}
+
+func (l *readonlyProvider) AcquireSectorNumber() (abi.SectorNumber, error) {
+	return 0, xerrors.New("read-only provider")
+}
+
+func (l *readonlyProvider) FinalizeSector(abi.SectorNumber) error {
+	return xerrors.New("read-only provider")
+}
+
+func (l *readonlyProvider) AcquireSector(id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) {
+	if allocate != 0 {
+		return sectorbuilder.SectorPaths{}, nil, xerrors.New("read-only storage")
+	}
+
+	return l.stor.acquireSector(l.miner, id, existing, allocate, sealing)
+}

storage/sealmgr/advmgr/storage.go

@@ -0,0 +1,157 @@
+package advmgr
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"sync"
+
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-sectorbuilder"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+
+	"github.com/filecoin-project/lotus/node/config"
+)
+
+const metaFile = "storage.json"
+var pathTypes = []sectorbuilder.SectorFileType{sectorbuilder.FTUnsealed, sectorbuilder.FTSealed, sectorbuilder.FTCache}
+
+type storage struct {
+	localLk sync.RWMutex
+	localStorage LocalStorage
+
+	paths []path
+}
+
+type path struct {
+	meta config.StorageMeta
+	local string
+
+	sectors map[abi.SectorID]sectorbuilder.SectorFileType
+}
+
+func openPath(p string, meta config.StorageMeta) (path, error) {
+	out := path{
+		meta:    meta,
+		local:   p,
+		sectors: map[abi.SectorID]sectorbuilder.SectorFileType{},
+	}
+
+	for _, t := range pathTypes {
+		ents, err := ioutil.ReadDir(filepath.Join(p, t.String()))
+		if err != nil {
+			return path{}, xerrors.Errorf("listing %s: %w", filepath.Join(p, t.String()), err)
+		}
+
+		for _, ent := range ents {
+			sid, err := parseSectorID(ent.Name())
+			if err != nil {
+				return path{}, xerrors.Errorf("parse sector id %s: %w", ent.Name(), err)
+			}
+
+			out.sectors[sid] |= t
+		}
+	}
+
+	return out, nil
+}
+
+func (st *storage) open() error {
+	st.localLk.Lock()
+	defer st.localLk.Unlock()
+
+	cfg, err := st.localStorage.GetStorage()
+	if err != nil {
+		return xerrors.Errorf("getting local storage config: %w", err)
+	}
+
+	if len(cfg.StoragePaths) == 0 {
+		return xerrors.New("no local storage paths configured")
+	}
+
+	for _, path := range cfg.StoragePaths {
+		mb, err := ioutil.ReadFile(filepath.Join(path.Path, metaFile))
+		if err != nil {
+			return xerrors.Errorf("reading storage metadata for %s: %w", path.Path, err)
+		}
+
+		var meta config.StorageMeta
+		if err := json.Unmarshal(mb, &meta); err != nil {
+			return xerrors.Errorf("unmarshalling storage metadata for %s: %w", path.Path, err)
+		}
+
+		pi, err := openPath(path.Path, meta)
+		if err != nil {
+			return xerrors.Errorf("opening path %s: %w", path.Path, err)
+		}
+
+		st.paths = append(st.paths, pi)
+	}
+
+	return nil
+}
+
+func (st *storage) acquireSector(mid abi.ActorID, id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) {
+	st.localLk.RLock()
+
+	if allocate != 0 {
+		st.localLk.RUnlock()
+		return sectorbuilder.SectorPaths{}, nil, xerrors.New("acquire alloc todo")
+	}
+
+	var out sectorbuilder.SectorPaths
+
+	for _, fileType := range pathTypes {
+		if fileType & existing == 0 {
+			continue
+		}
+
+		for _, p := range st.paths {
+			s, ok := p.sectors[abi.SectorID{
+				Miner:  mid,
+				Number: id,
+			}]
+			if !ok {
+				continue
+			}
+			if s & fileType == 0 {
+				continue
+			}
+
+			spath := filepath.Join(p.local, fileType.String(), fmt.Sprintf("s-t0%d-%d", mid, id))
+
+			switch fileType {
+			case sectorbuilder.FTUnsealed:
+				out.Unsealed = spath
+			case sectorbuilder.FTSealed:
+				out.Sealed = spath
+			case sectorbuilder.FTCache:
+				out.Cache = spath
+			}
+
+			existing ^= fileType
+		}
+	}
+
+	return out, st.localLk.RUnlock, nil
+}
+
+func parseSectorID(baseName string) (abi.SectorID, error) {
+	var n abi.SectorNumber
+	var mid abi.ActorID
+	read, err := fmt.Sscanf(baseName, "s-t0%d-%d", &mid, &n)
+	if err != nil {
+		return abi.SectorID{}, xerrors.Errorf(": %w", err)
+	}
+
+	if read != 2 {
+		return abi.SectorID{}, xerrors.Errorf("parseSectorID expected to scan 2 values, got %d", read)
+	}
+
+	return abi.SectorID{
+		Miner:  mid,
+		Number: n,
+	}, nil
+}