Merge pull request #111 from filecoin-project/fix/blocksync-mining

Fix mining with no peers

chain/blocksync.go

@@ -15,6 +15,7 @@ import (
 	"github.com/filecoin-project/go-lotus/chain/store"
 	"github.com/filecoin-project/go-lotus/chain/types"
 	"github.com/filecoin-project/go-lotus/lib/cborrpc"
+	"github.com/filecoin-project/go-lotus/node/modules/dtypes"
 
 	"github.com/ipfs/go-cid"
 	cbor "github.com/ipfs/go-ipld-cbor"
@@ -189,7 +190,7 @@ type BlockSync struct {
 	syncPeers   map[peer.ID]struct{}
 }
 
-func NewBlockSyncClient(bserv bserv.BlockService, h host.Host) *BlockSync {
+func NewBlockSyncClient(bserv dtypes.ChainBlockService, h host.Host) *BlockSync {
 	return &BlockSync{
 		bserv:     bserv,
 		newStream: h.NewStream,
@@ -207,20 +208,18 @@ func (bs *BlockSync) getPeers() []peer.ID {
 	return out
 }
 
-func (bs *BlockSync) processStatus(req *BlockSyncRequest, res *BlockSyncResponse) ([]*types.TipSet, error) {
+func (bs *BlockSync) processStatus(req *BlockSyncRequest, res *BlockSyncResponse) error {
 	switch res.Status {
-	case 0: // Success
-		return bs.processBlocksResponse(req, res)
 	case 101: // Partial Response
 		panic("not handled")
 	case 201: // req.Start not found
-		return nil, fmt.Errorf("not found")
+		return fmt.Errorf("not found")
 	case 202: // Go Away
 		panic("not handled")
 	case 203: // Internal Error
-		return nil, fmt.Errorf("block sync peer errored: %s", res.Message)
+		return fmt.Errorf("block sync peer errored: %s", res.Message)
 	default:
-		return nil, fmt.Errorf("unrecognized response code")
+		return fmt.Errorf("unrecognized response code")
 	}
 }
 
@@ -236,17 +235,19 @@ func (bs *BlockSync) GetBlocks(ctx context.Context, tipset []cid.Cid, count int)
 	}
 
 	var err error
-	var res *BlockSyncResponse
 	for _, p := range perm {
-		res, err = bs.sendRequestToPeer(ctx, peers[p], req)
+		res, err := bs.sendRequestToPeer(ctx, peers[p], req)
 		if err != nil {
 			log.Warnf("BlockSync request failed for peer %s: %s", peers[p].String(), err)
 			continue
 		}
 
-		ts, err := bs.processStatus(req, res)
+		if res.Status == 0 {
-		if err == nil {
+			return bs.processBlocksResponse(req, res)
-			return ts, nil
+		}
+		err = bs.processStatus(req, res)
+		if err != nil {
+			log.Warnf("BlockSync peer %s response was an error: %s", peers[p].String(), err)
 		}
 	}
 	return nil, xerrors.Errorf("GetBlocks failed with all peers: %w", err)
@@ -298,25 +299,25 @@ func (bs *BlockSync) GetChainMessages(ctx context.Context, h *types.TipSet, coun
 		Options:       BSOptMessages,
 	}
 
-	res, err := bs.sendRequestToPeer(ctx, peers[perm[0]], req)
+	var err error
-	if err != nil {
+	for _, p := range perm {
-		return nil, err
+		res, err := bs.sendRequestToPeer(ctx, peers[p], req)
+		if err != nil {
+			log.Warnf("BlockSync request failed for peer %s: %s", peers[p].String(), err)
+			continue
+		}
+
+		if res.Status == 0 {
+			return res.Chain, nil
+		}
+		err = bs.processStatus(req, res)
+		if err != nil {
+			log.Warnf("BlockSync peer %s response was an error: %s", peers[p].String(), err)
+		}
 	}
 
-	switch res.Status {
+	// TODO: What if we have no peers (and err is nil)?
-	case 0: // Success
+	return nil, xerrors.Errorf("GetChainMessages failed with all peers: %w", err)
-		return res.Chain, nil
-	case 101: // Partial Response
-		panic("not handled")
-	case 201: // req.Start not found
-		return nil, fmt.Errorf("not found")
-	case 202: // Go Away
-		panic("not handled")
-	case 203: // Internal Error
-		return nil, fmt.Errorf("block sync peer errored: %s", res.Message)
-	default:
-		return nil, fmt.Errorf("unrecognized response code")
-	}
 }
 
 func bstsToFullTipSet(bts *BSTipSet) (*store.FullTipSet, error) {

node/builder.go

@@ -6,13 +6,7 @@ import (
 	"reflect"
 	"time"
 
-	"github.com/ipfs/go-filestore"
-	exchange "github.com/ipfs/go-ipfs-exchange-interface"
-
-	bserv "github.com/ipfs/go-blockservice"
-	"github.com/ipfs/go-datastore"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
-	ipld "github.com/ipfs/go-ipld-format"
 	ci "github.com/libp2p/go-libp2p-core/crypto"
 	"github.com/libp2p/go-libp2p-core/host"
 	"github.com/libp2p/go-libp2p-core/peer"
@@ -33,6 +27,7 @@ import (
 	"github.com/filecoin-project/go-lotus/node/hello"
 	"github.com/filecoin-project/go-lotus/node/impl"
 	"github.com/filecoin-project/go-lotus/node/modules"
+	"github.com/filecoin-project/go-lotus/node/modules/dtypes"
 	"github.com/filecoin-project/go-lotus/node/modules/helpers"
 	"github.com/filecoin-project/go-lotus/node/modules/lp2p"
 	"github.com/filecoin-project/go-lotus/node/modules/testing"
@@ -193,11 +188,11 @@ func Online() Option {
 
 			Override(new(*store.ChainStore), modules.ChainStore),
 
-			Override(new(blockstore.GCLocker), blockstore.NewGCLocker),
+			Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker),
-			Override(new(blockstore.GCBlockstore), blockstore.NewGCBlockstore),
+			Override(new(dtypes.ChainGCBlockstore), modules.ChainGCBlockstore),
-			Override(new(exchange.Interface), modules.Bitswap),
+			Override(new(dtypes.ChainExchange), modules.ChainExchange),
-			Override(new(bserv.BlockService), bserv.New),
+			Override(new(dtypes.ChainBlockService), modules.ChainBlockservice),
-			Override(new(ipld.DAGService), testing.MemoryClientDag),
+			Override(new(dtypes.ClientDAG), testing.MemoryClientDag),
 
 			// Filecoin services
 			Override(new(*chain.Syncer), chain.NewSyncer),
@@ -274,11 +269,11 @@ func Repo(r repo.Repo) Option {
 		Config(cfg),
 		Override(new(repo.LockedRepo), modules.LockedRepo(lr)), // module handles closing
 
-		Override(new(datastore.Batching), modules.Datastore),
+		Override(new(dtypes.MetadataDS), modules.Datastore),
-		Override(new(blockstore.Blockstore), modules.Blockstore),
+		Override(new(dtypes.ChainBlockstore), modules.ChainBlockstore),
 
-		Override(new(*filestore.Filestore), modules.ClientFstore),
+		Override(new(dtypes.ClientFilestore), modules.ClientFstore),
-		Override(new(ipld.DAGService), modules.ClientDAG),
+		Override(new(dtypes.ClientDAG), modules.ClientDAG),
 
 		Override(new(ci.PrivKey), pk),
 		Override(new(ci.PubKey), ci.PrivKey.GetPublic),

node/client/import.go

@@ -6,6 +6,8 @@ import (
 	"os"
 
 	"github.com/filecoin-project/go-lotus/api"
+	"github.com/filecoin-project/go-lotus/node/modules/dtypes"
+
 	"github.com/ipfs/go-filestore"
 	"go.uber.org/fx"
 
@@ -20,8 +22,8 @@ import (
 type LocalStorage struct {
 	fx.In
 
-	LocalDAG  ipld.DAGService
+	LocalDAG  dtypes.ClientDAG
-	Filestore *filestore.Filestore `optional:"true"`
+	Filestore dtypes.ClientFilestore `optional:"true"`
 }
 
 func (s *LocalStorage) ClientImport(ctx context.Context, path string) (cid.Cid, error) {

node/modules/chain.go

@@ -0,0 +1,108 @@
+package modules
+
+import (
+	"bytes"
+	"context"
+
+	"github.com/ipfs/go-bitswap"
+	"github.com/ipfs/go-bitswap/network"
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-car"
+	"github.com/ipfs/go-datastore"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	"github.com/libp2p/go-libp2p-core/host"
+	"github.com/libp2p/go-libp2p-core/routing"
+	"go.uber.org/fx"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-lotus/chain/store"
+	"github.com/filecoin-project/go-lotus/chain/types"
+	"github.com/filecoin-project/go-lotus/node/modules/dtypes"
+	"github.com/filecoin-project/go-lotus/node/modules/helpers"
+	"github.com/filecoin-project/go-lotus/node/repo"
+)
+
+func ChainExchange(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, rt routing.Routing, bs dtypes.ChainGCBlockstore) dtypes.ChainExchange {
+	bitswapNetwork := network.NewFromIpfsHost(host, rt)
+	exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, bs)
+	lc.Append(fx.Hook{
+		OnStop: func(ctx context.Context) error {
+			return exch.Close()
+		},
+	})
+
+	return exch
+}
+
+func ChainBlockstore(r repo.LockedRepo) (dtypes.ChainBlockstore, error) {
+	blocks, err := r.Datastore("/blocks")
+	if err != nil {
+		return nil, err
+	}
+
+	bs := blockstore.NewBlockstore(blocks)
+	return blockstore.NewIdStore(bs), nil
+}
+
+func ChainGCBlockstore(bs dtypes.ChainBlockstore, gcl dtypes.ChainGCLocker) dtypes.ChainGCBlockstore {
+	return blockstore.NewGCBlockstore(bs, gcl)
+}
+
+func ChainBlockservice(bs dtypes.ChainBlockstore, rem dtypes.ChainExchange) dtypes.ChainBlockService {
+	return blockservice.New(bs, rem)
+}
+
+func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS) *store.ChainStore {
+	chain := store.NewChainStore(bs, ds)
+
+	lc.Append(fx.Hook{
+		OnStart: func(ctx context.Context) error {
+			return chain.Load()
+		},
+	})
+
+	return chain
+}
+
+func ErrorGenesis() Genesis {
+	return func() (header *types.BlockHeader, e error) {
+		return nil, xerrors.New("No genesis block provided, provide the file with 'lotus daemon --genesis=[genesis file]'")
+	}
+}
+
+func LoadGenesis(genBytes []byte) func(dtypes.ChainBlockstore) Genesis {
+	return func(bs dtypes.ChainBlockstore) Genesis {
+		return func() (header *types.BlockHeader, e error) {
+			c, err := car.LoadCar(bs, bytes.NewReader(genBytes))
+			if err != nil {
+				return nil, err
+			}
+			if len(c.Roots) != 1 {
+				return nil, xerrors.New("expected genesis file to have one root")
+			}
+			root, err := bs.Get(c.Roots[0])
+			if err != nil {
+				return &types.BlockHeader{}, err
+			}
+
+			return types.DecodeBlock(root.RawData())
+		}
+	}
+}
+
+func SetGenesis(cs *store.ChainStore, g Genesis) error {
+	_, err := cs.GetGenesis()
+	if err == nil {
+		return nil // already set, noop
+	}
+	if err != datastore.ErrNotFound {
+		return err
+	}
+
+	genesis, err := g()
+	if err != nil {
+		return err
+	}
+
+	return cs.SetGenesis(genesis)
+}

node/modules/client.go

@@ -0,0 +1,47 @@
+package modules
+
+import (
+	"context"
+	"path/filepath"
+
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-datastore"
+	"github.com/ipfs/go-datastore/namespace"
+	"github.com/ipfs/go-filestore"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	offline "github.com/ipfs/go-ipfs-exchange-offline"
+	"github.com/ipfs/go-merkledag"
+	"go.uber.org/fx"
+
+	"github.com/filecoin-project/go-lotus/node/modules/dtypes"
+	"github.com/filecoin-project/go-lotus/node/repo"
+)
+
+func ClientFstore(r repo.LockedRepo) (dtypes.ClientFilestore, error) {
+	clientds, err := r.Datastore("/client")
+	if err != nil {
+		return nil, err
+	}
+	blocks := namespace.Wrap(clientds, datastore.NewKey("blocks"))
+
+	fm := filestore.NewFileManager(clientds, filepath.Dir(r.Path()))
+	fm.AllowFiles = true
+	// TODO: fm.AllowUrls (needs more code in client import)
+
+	bs := blockstore.NewBlockstore(blocks)
+	return filestore.NewFilestore(bs, fm), nil
+}
+
+func ClientDAG(lc fx.Lifecycle, fstore dtypes.ClientFilestore) dtypes.ClientDAG {
+	ibs := blockstore.NewIdStore((*filestore.Filestore)(fstore))
+	bsvc := blockservice.New(ibs, offline.Exchange(ibs))
+	dag := merkledag.NewDAGService(bsvc)
+
+	lc.Append(fx.Hook{
+		OnStop: func(_ context.Context) error {
+			return bsvc.Close()
+		},
+	})
+
+	return dag
+}

node/modules/core.go

@@ -1,44 +1,19 @@
 package modules
 
 import (
-	"bytes"
-	"context"
 	"crypto/rand"
 	"io"
 	"io/ioutil"
-	"path/filepath"
 
 	"github.com/gbrlsnchs/jwt/v3"
-	"github.com/ipfs/go-bitswap"
-	"github.com/ipfs/go-bitswap/network"
-	"github.com/ipfs/go-blockservice"
-	"github.com/ipfs/go-car"
-	"github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-datastore/namespace"
-	"github.com/ipfs/go-filestore"
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
-	exchange "github.com/ipfs/go-ipfs-exchange-interface"
-	offline "github.com/ipfs/go-ipfs-exchange-offline"
-	ipld "github.com/ipfs/go-ipld-format"
 	logging "github.com/ipfs/go-log"
-	"github.com/ipfs/go-merkledag"
-	"github.com/libp2p/go-libp2p-core/host"
 	"github.com/libp2p/go-libp2p-core/peerstore"
-	"github.com/libp2p/go-libp2p-core/routing"
 	record "github.com/libp2p/go-libp2p-record"
-	"github.com/mitchellh/go-homedir"
-	"go.uber.org/fx"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-lotus/api"
-	"github.com/filecoin-project/go-lotus/chain/address"
-	"github.com/filecoin-project/go-lotus/chain/store"
 	"github.com/filecoin-project/go-lotus/chain/types"
-	"github.com/filecoin-project/go-lotus/chain/wallet"
-	"github.com/filecoin-project/go-lotus/lib/sectorbuilder"
-	"github.com/filecoin-project/go-lotus/node/modules/helpers"
 	"github.com/filecoin-project/go-lotus/node/repo"
-	"github.com/filecoin-project/go-lotus/storage"
 )
 
 var log = logging.Logger("modules")
@@ -52,51 +27,6 @@ func RecordValidator(ps peerstore.Peerstore) record.Validator {
 	}
 }
 
-func Bitswap(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, rt routing.Routing, bs blockstore.GCBlockstore) exchange.Interface {
-	bitswapNetwork := network.NewFromIpfsHost(host, rt)
-	exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, bs)
-	lc.Append(fx.Hook{
-		OnStop: func(ctx context.Context) error {
-			return exch.Close()
-		},
-	})
-
-	return exch
-}
-
-func SetGenesis(cs *store.ChainStore, g Genesis) error {
-	_, err := cs.GetGenesis()
-	if err == nil {
-		return nil // already set, noop
-	}
-	if err != datastore.ErrNotFound {
-		return err
-	}
-
-	genesis, err := g()
-	if err != nil {
-		return err
-	}
-
-	return cs.SetGenesis(genesis)
-}
-
-func LockedRepo(lr repo.LockedRepo) func(lc fx.Lifecycle) repo.LockedRepo {
-	return func(lc fx.Lifecycle) repo.LockedRepo {
-		lc.Append(fx.Hook{
-			OnStop: func(_ context.Context) error {
-				return lr.Close()
-			},
-		})
-
-		return lr
-	}
-}
-
-func KeyStore(lr repo.LockedRepo) (types.KeyStore, error) {
-	return lr.KeyStore()
-}
-
 const JWTSecretName = "auth-jwt-private"
 
 type APIAlg jwt.HMACSHA
@@ -141,158 +71,3 @@ func APISecret(keystore types.KeyStore, lr repo.LockedRepo) (*APIAlg, error) {
 
 	return (*APIAlg)(jwt.NewHS256(key.PrivateKey)), nil
 }
-
-func Datastore(r repo.LockedRepo) (datastore.Batching, error) {
-	return r.Datastore("/metadata")
-}
-
-func Blockstore(r repo.LockedRepo) (blockstore.Blockstore, error) {
-	blocks, err := r.Datastore("/blocks")
-	if err != nil {
-		return nil, err
-	}
-
-	bs := blockstore.NewBlockstore(blocks)
-	return blockstore.NewIdStore(bs), nil
-}
-
-func ClientFstore(r repo.LockedRepo) (*filestore.Filestore, error) {
-	clientds, err := r.Datastore("/client")
-	if err != nil {
-		return nil, err
-	}
-	blocks := namespace.Wrap(clientds, datastore.NewKey("blocks"))
-
-	fm := filestore.NewFileManager(clientds, filepath.Dir(r.Path()))
-	fm.AllowFiles = true
-	// TODO: fm.AllowUrls (needs more code in client import)
-
-	bs := blockstore.NewBlockstore(blocks)
-	return filestore.NewFilestore(bs, fm), nil
-}
-
-func ClientDAG(lc fx.Lifecycle, fstore *filestore.Filestore) ipld.DAGService {
-	ibs := blockstore.NewIdStore(fstore)
-	bsvc := blockservice.New(ibs, offline.Exchange(ibs))
-	dag := merkledag.NewDAGService(bsvc)
-
-	lc.Append(fx.Hook{
-		OnStop: func(_ context.Context) error {
-			return bsvc.Close()
-		},
-	})
-
-	return dag
-}
-
-func ChainStore(lc fx.Lifecycle, bs blockstore.Blockstore, ds datastore.Batching) *store.ChainStore {
-	chain := store.NewChainStore(bs, ds)
-
-	lc.Append(fx.Hook{
-		OnStart: func(ctx context.Context) error {
-			return chain.Load()
-		},
-	})
-
-	return chain
-}
-
-func ErrorGenesis() Genesis {
-	return func() (header *types.BlockHeader, e error) {
-		return nil, xerrors.New("No genesis block provided, provide the file with 'lotus daemon --genesis=[genesis file]'")
-	}
-}
-
-func LoadGenesis(genBytes []byte) func(blockstore.Blockstore) Genesis {
-	return func(bs blockstore.Blockstore) Genesis {
-		return func() (header *types.BlockHeader, e error) {
-			c, err := car.LoadCar(bs, bytes.NewReader(genBytes))
-			if err != nil {
-				return nil, err
-			}
-			if len(c.Roots) != 1 {
-				return nil, xerrors.New("expected genesis file to have one root")
-			}
-			root, err := bs.Get(c.Roots[0])
-			if err != nil {
-				return &types.BlockHeader{}, err
-			}
-
-			return types.DecodeBlock(root.RawData())
-		}
-	}
-}
-
-func SectorBuilderConfig(storagePath string) func() (*sectorbuilder.SectorBuilderConfig, error) {
-	return func() (*sectorbuilder.SectorBuilderConfig, error) {
-		sp, err := homedir.Expand(storagePath)
-		if err != nil {
-			return nil, err
-		}
-
-		metadata := filepath.Join(sp, "meta")
-		sealed := filepath.Join(sp, "sealed")
-		staging := filepath.Join(sp, "staging")
-
-		// TODO: get the address of the miner actor
-		minerAddr, err := address.NewIDAddress(42)
-		if err != nil {
-			return nil, err
-		}
-
-		sb := &sectorbuilder.SectorBuilderConfig{
-			Miner:       minerAddr,
-			SectorSize:  1024,
-			MetadataDir: metadata,
-			SealedDir:   sealed,
-			StagedDir:   staging,
-		}
-
-		return sb, nil
-	}
-}
-
-func SectorBuilder(mctx helpers.MetricsCtx, lc fx.Lifecycle, sbc *sectorbuilder.SectorBuilderConfig) (*sectorbuilder.SectorBuilder, error) {
-	sb, err := sectorbuilder.New(sbc)
-	if err != nil {
-		return nil, err
-	}
-
-	ctx := helpers.LifecycleCtx(mctx, lc)
-
-	lc.Append(fx.Hook{
-		OnStart: func(context.Context) error {
-			sb.Run(ctx)
-			return nil
-		},
-	})
-
-	return sb, nil
-}
-
-func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds datastore.Batching, sb *sectorbuilder.SectorBuilder, w *wallet.Wallet) (*storage.Miner, error) {
-	maddrb, err := ds.Get(datastore.NewKey("miner-address"))
-	if err != nil {
-		return nil, err
-	}
-
-	maddr, err := address.NewFromBytes(maddrb)
-	if err != nil {
-		return nil, err
-	}
-
-	sm, err := storage.NewMiner(api, maddr, h, ds, sb, w)
-	if err != nil {
-		return nil, err
-	}
-
-	ctx := helpers.LifecycleCtx(mctx, lc)
-
-	lc.Append(fx.Hook{
-		OnStart: func(context.Context) error {
-			return sm.Run(ctx)
-		},
-	})
-
-	return sm, nil
-}

node/modules/dtypes/storage.go

@@ -0,0 +1,24 @@
+package dtypes
+
+import (
+	bserv "github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-datastore"
+	"github.com/ipfs/go-filestore"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	exchange "github.com/ipfs/go-ipfs-exchange-interface"
+	ipld "github.com/ipfs/go-ipld-format"
+)
+
+// MetadataDS stores metadata
+// dy default it's namespaced under /metadata in main repo datastore
+type MetadataDS datastore.Batching
+
+type ChainBlockstore blockstore.Blockstore
+
+type ChainGCLocker blockstore.GCLocker
+type ChainGCBlockstore blockstore.GCBlockstore
+type ChainExchange exchange.Interface
+type ChainBlockService bserv.BlockService
+
+type ClientFilestore *filestore.Filestore
+type ClientDAG ipld.DAGService

node/modules/lp2p/host.go

@@ -4,12 +4,11 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/ipfs/go-datastore"
 	nilrouting "github.com/ipfs/go-ipfs-routing/none"
 	"github.com/libp2p/go-libp2p"
-	host "github.com/libp2p/go-libp2p-core/host"
+	"github.com/libp2p/go-libp2p-core/host"
-	peer "github.com/libp2p/go-libp2p-core/peer"
+	"github.com/libp2p/go-libp2p-core/peer"
-	peerstore "github.com/libp2p/go-libp2p-core/peerstore"
+	"github.com/libp2p/go-libp2p-core/peerstore"
 	dht "github.com/libp2p/go-libp2p-kad-dht"
 	dhtopts "github.com/libp2p/go-libp2p-kad-dht/opts"
 	record "github.com/libp2p/go-libp2p-record"
@@ -17,6 +16,7 @@ import (
 	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
 	"go.uber.org/fx"
 
+	"github.com/filecoin-project/go-lotus/node/modules/dtypes"
 	"github.com/filecoin-project/go-lotus/node/modules/helpers"
 )
 
@@ -65,7 +65,7 @@ func MockHost(mn mocknet.Mocknet, id peer.ID, ps peerstore.Peerstore) (RawHost,
 }
 
 func DHTRouting(client bool) interface{} {
-	return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, host RawHost, dstore datastore.Batching, validator record.Validator) (BaseIpfsRouting, error) {
+	return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, host RawHost, dstore dtypes.MetadataDS, validator record.Validator) (BaseIpfsRouting, error) {
 		ctx := helpers.LifecycleCtx(mctx, lc)
 
 		d, err := dht.New(

node/modules/storage.go

@@ -0,0 +1,31 @@
+package modules
+
+import (
+	"context"
+
+	"go.uber.org/fx"
+
+	"github.com/filecoin-project/go-lotus/chain/types"
+	"github.com/filecoin-project/go-lotus/node/modules/dtypes"
+	"github.com/filecoin-project/go-lotus/node/repo"
+)
+
+func LockedRepo(lr repo.LockedRepo) func(lc fx.Lifecycle) repo.LockedRepo {
+	return func(lc fx.Lifecycle) repo.LockedRepo {
+		lc.Append(fx.Hook{
+			OnStop: func(_ context.Context) error {
+				return lr.Close()
+			},
+		})
+
+		return lr
+	}
+}
+
+func KeyStore(lr repo.LockedRepo) (types.KeyStore, error) {
+	return lr.KeyStore()
+}
+
+func Datastore(r repo.LockedRepo) (dtypes.MetadataDS, error) {
+	return r.Datastore("/metadata")
+}

node/modules/storageminer.go

@@ -0,0 +1,93 @@
+package modules
+
+import (
+	"context"
+	"path/filepath"
+
+	"github.com/ipfs/go-datastore"
+	"github.com/libp2p/go-libp2p-core/host"
+	"github.com/mitchellh/go-homedir"
+	"go.uber.org/fx"
+
+	"github.com/filecoin-project/go-lotus/api"
+	"github.com/filecoin-project/go-lotus/chain/address"
+	"github.com/filecoin-project/go-lotus/chain/wallet"
+	"github.com/filecoin-project/go-lotus/lib/sectorbuilder"
+	"github.com/filecoin-project/go-lotus/node/modules/dtypes"
+	"github.com/filecoin-project/go-lotus/node/modules/helpers"
+	"github.com/filecoin-project/go-lotus/storage"
+)
+
+func SectorBuilderConfig(storagePath string) func() (*sectorbuilder.SectorBuilderConfig, error) {
+	return func() (*sectorbuilder.SectorBuilderConfig, error) {
+		sp, err := homedir.Expand(storagePath)
+		if err != nil {
+			return nil, err
+		}
+
+		metadata := filepath.Join(sp, "meta")
+		sealed := filepath.Join(sp, "sealed")
+		staging := filepath.Join(sp, "staging")
+
+		// TODO: get the address of the miner actor
+		minerAddr, err := address.NewIDAddress(42)
+		if err != nil {
+			return nil, err
+		}
+
+		sb := &sectorbuilder.SectorBuilderConfig{
+			Miner:       minerAddr,
+			SectorSize:  1024,
+			MetadataDir: metadata,
+			SealedDir:   sealed,
+			StagedDir:   staging,
+		}
+
+		return sb, nil
+	}
+}
+
+func SectorBuilder(mctx helpers.MetricsCtx, lc fx.Lifecycle, sbc *sectorbuilder.SectorBuilderConfig) (*sectorbuilder.SectorBuilder, error) {
+	sb, err := sectorbuilder.New(sbc)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx := helpers.LifecycleCtx(mctx, lc)
+
+	lc.Append(fx.Hook{
+		OnStart: func(context.Context) error {
+			sb.Run(ctx)
+			return nil
+		},
+	})
+
+	return sb, nil
+}
+
+func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder, w *wallet.Wallet) (*storage.Miner, error) {
+	maddrb, err := ds.Get(datastore.NewKey("miner-address"))
+	if err != nil {
+		return nil, err
+	}
+
+	maddr, err := address.NewFromBytes(maddrb)
+	if err != nil {
+		return nil, err
+	}
+
+	sm, err := storage.NewMiner(api, maddr, h, ds, sb, w)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx := helpers.LifecycleCtx(mctx, lc)
+
+	lc.Append(fx.Hook{
+		OnStart: func(context.Context) error {
+			return sm.Run(ctx)
+		},
+	})
+
+	return sm, nil
+}

node/modules/testing.go

@@ -1,23 +0,0 @@
-package modules
-
-import (
-	"crypto/rand"
-	"io"
-	"io/ioutil"
-
-	"github.com/libp2p/go-libp2p-core/peer"
-	mh "github.com/multiformats/go-multihash"
-)
-
-// RandomPeerID generates random peer id
-func RandomPeerID() (peer.ID, error) {
-	b, err := ioutil.ReadAll(io.LimitReader(rand.Reader, 32))
-	if err != nil {
-		return "", err
-	}
-	hash, err := mh.Sum(b, mh.SHA2_256, -1)
-	if err != nil {
-		return "", err
-	}
-	return peer.ID(hash), nil
-}

node/modules/testing/genesis.go

@@ -18,6 +18,7 @@ import (
 	"github.com/filecoin-project/go-lotus/chain/types"
 	"github.com/filecoin-project/go-lotus/chain/wallet"
 	"github.com/filecoin-project/go-lotus/node/modules"
+	"github.com/filecoin-project/go-lotus/node/modules/dtypes"
 )
 
 var glog = logging.Logger("genesis")
@@ -44,8 +45,8 @@ func MakeGenesisMem(out io.Writer) func(bs blockstore.Blockstore, w *wallet.Wall
 	}
 }
 
-func MakeGenesis(outFile string) func(bs blockstore.Blockstore, w *wallet.Wallet) modules.Genesis {
+func MakeGenesis(outFile string) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
-	return func(bs blockstore.Blockstore, w *wallet.Wallet) modules.Genesis {
+	return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
 		return func() (*types.BlockHeader, error) {
 			glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network")
 			minerAddr, err := w.GenerateKey(types.KTSecp256k1)