Merge pull request #5679 from filecoin-project/next

1.5.1 staging branch

.circleci/config.yml

@@ -2,6 +2,7 @@ version: 2.1
 orbs:
   go: gotest/tools@0.0.13
   aws-cli: circleci/aws-cli@1.3.2
+  packer: salaxander/packer@0.0.3
 
 executors:
   golang:
@@ -277,6 +278,11 @@ jobs:
       - install-deps
       - prepare
       - run: make calibnet
+      - run: mkdir linux-calibnet && mv lotus lotus-miner lotus-worker linux-calibnet
+      - persist_to_workspace:
+          root: "."
+          paths:
+            - linux-calibnet
   build-lotus-soup:
     description: |
       Compile `lotus-soup` Testground test plan
@@ -289,7 +295,7 @@ jobs:
       - run: cd extern/filecoin-ffi && make
       - run:
           name: "go get lotus@master"
-          command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master
+          command: cd testplans/lotus-soup && go mod edit -replace=github.com/filecoin-project/lotus=../..
       - run:
           name: "build lotus-soup testplan"
           command: pushd testplans/lotus-soup && go build -tags=testground .
@@ -583,6 +589,22 @@ jobs:
               docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
             done
 
+  publish-packer:
+    description: build and push AWS IAM and DigitalOcean droplet.
+    executor:
+      name: packer/default
+      packer-version: 1.6.6
+    steps:
+      - checkout
+      - attach_workspace:
+          at: "."
+      - packer/build:
+          template: tools/packer/lotus.pkr.hcl
+          args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
+      - packer/build:
+          template: tools/packer/lotus.pkr.hcl
+          args: "-var ci_workspace_bins=./linux-calibnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
+
 workflows:
   version: 2.1
   ci:
@@ -683,3 +705,15 @@ workflows:
           path: .
           repo: lotus-dev
           tag: '${CIRCLE_SHA1:0:8}'
+      - publish-packer:
+          requires:
+            - build-all
+            - build-ntwk-calibration
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+$/
+         

api/api_common.go

@@ -11,8 +11,6 @@ import (
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
 	protocol "github.com/libp2p/go-libp2p-core/protocol"
-
-	"github.com/filecoin-project/lotus/build"
 )
 
 type Common interface {
@@ -33,6 +31,7 @@ type Common interface {
 	NetPubsubScores(context.Context) ([]PubsubScore, error)
 	NetAutoNatStatus(context.Context) (NatInfo, error)
 	NetAgentVersion(ctx context.Context, p peer.ID) (string, error)
+	NetPeerInfo(context.Context, peer.ID) (*ExtendedPeerInfo, error)
 
 	// NetBandwidthStats returns statistics about the nodes total bandwidth
 	// usage and current rate across all peers and protocols.
@@ -57,7 +56,7 @@ type Common interface {
 	ID(context.Context) (peer.ID, error)
 
 	// Version provides information about API provider
-	Version(context.Context) (Version, error)
+	Version(context.Context) (APIVersion, error)
 
 	LogList(context.Context) ([]string, error)
 	LogSetLevel(context.Context, string, string) error
@@ -71,15 +70,15 @@ type Common interface {
 	Closing(context.Context) (<-chan struct{}, error)
 }
 
-// Version provides various build-time information
+// APIVersion provides various build-time information
-type Version struct {
+type APIVersion struct {
 	Version string
 
 	// APIVersion is a binary encoded semver version of the remote implementing
 	// this api
 	//
 	// See APIVersion in build/version.go
-	APIVersion build.Version
+	APIVersion Version
 
 	// TODO: git commit / os / genesis cid?
 
@@ -87,7 +86,7 @@ type Version struct {
 	BlockDelay uint64
 }
 
-func (v Version) String() string {
+func (v APIVersion) String() string {
 	return fmt.Sprintf("%s+api%s", v.Version, v.APIVersion.String())
 }
 

api/api_full.go

@@ -32,6 +32,14 @@ import (
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 )
 
+//go:generate go run github.com/golang/mock/mockgen -destination=mocks/mock_full.go -package=mocks . FullNode
+
+// ChainIO abstracts operations for accessing raw IPLD objects.
+type ChainIO interface {
+	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
+	ChainHasObj(context.Context, cid.Cid) (bool, error)
+}
+
 // FullNode API is a low-level interface to the Filecoin network full node
 type FullNode interface {
 	Common
@@ -862,6 +870,8 @@ const (
 
 func (v SyncStateStage) String() string {
 	switch v {
+	case StageIdle:
+		return "idle"
 	case StageHeaders:
 		return "header sync"
 	case StagePersistHeaders:

api/api_storage.go

@@ -36,7 +36,7 @@ type StorageMiner interface {
 	MiningBase(context.Context) (*types.TipSet, error)
 
 	// Temp api for testing
-	PledgeSector(context.Context) error
+	PledgeSector(context.Context) (abi.SectorID, error)
 
 	// Get the status of a given sector by ID
 	SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (SectorInfo, error)
@@ -238,6 +238,9 @@ type AddressConfig struct {
 	PreCommitControl []address.Address
 	CommitControl    []address.Address
 	TerminateControl []address.Address
+
+	DisableOwnerFallback  bool
+	DisableWorkerFallback bool
 }
 
 // PendingDealInfo has info about pending deals and when they are due to be

api/api_worker.go

@@ -9,12 +9,10 @@ import (
 	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-
-	"github.com/filecoin-project/lotus/build"
 )
 
 type WorkerAPI interface {
-	Version(context.Context) (build.Version, error)
+	Version(context.Context) (Version, error)
 	// TODO: Info() (name, ...) ?
 
 	TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) // TaskType -> Weight

api/apibstore/apibstore.go

@@ -1,68 +0,0 @@
-package apibstore
-
-import (
-	"context"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/lotus/lib/blockstore"
-)
-
-type ChainIO interface {
-	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
-	ChainHasObj(context.Context, cid.Cid) (bool, error)
-}
-
-type apiBStore struct {
-	api ChainIO
-}
-
-func NewAPIBlockstore(cio ChainIO) blockstore.Blockstore {
-	return &apiBStore{
-		api: cio,
-	}
-}
-
-func (a *apiBStore) DeleteBlock(cid.Cid) error {
-	return xerrors.New("not supported")
-}
-
-func (a *apiBStore) Has(c cid.Cid) (bool, error) {
-	return a.api.ChainHasObj(context.TODO(), c)
-}
-
-func (a *apiBStore) Get(c cid.Cid) (blocks.Block, error) {
-	bb, err := a.api.ChainReadObj(context.TODO(), c)
-	if err != nil {
-		return nil, err
-	}
-	return blocks.NewBlockWithCid(bb, c)
-}
-
-func (a *apiBStore) GetSize(c cid.Cid) (int, error) {
-	bb, err := a.api.ChainReadObj(context.TODO(), c)
-	if err != nil {
-		return 0, err
-	}
-	return len(bb), nil
-}
-
-func (a *apiBStore) Put(blocks.Block) error {
-	return xerrors.New("not supported")
-}
-
-func (a *apiBStore) PutMany([]blocks.Block) error {
-	return xerrors.New("not supported")
-}
-
-func (a *apiBStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return nil, xerrors.New("not supported")
-}
-
-func (a *apiBStore) HashOnRead(enabled bool) {
-	return
-}
-
-var _ blockstore.Blockstore = &apiBStore{}

api/apistruct/struct.go

@@ -33,7 +33,6 @@ import (
 	"github.com/filecoin-project/specs-storage/storage"
 
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -60,12 +59,13 @@ type CommonStruct struct {
 		NetBandwidthStatsByPeer     func(ctx context.Context) (map[string]metrics.Stats, error)      `perm:"read"`
 		NetBandwidthStatsByProtocol func(ctx context.Context) (map[protocol.ID]metrics.Stats, error) `perm:"read"`
 		NetAgentVersion             func(ctx context.Context, p peer.ID) (string, error)             `perm:"read"`
+		NetPeerInfo                 func(context.Context, peer.ID) (*api.ExtendedPeerInfo, error)    `perm:"read"`
 		NetBlockAdd                 func(ctx context.Context, acl api.NetBlockList) error            `perm:"admin"`
 		NetBlockRemove              func(ctx context.Context, acl api.NetBlockList) error            `perm:"admin"`
 		NetBlockList                func(ctx context.Context) (api.NetBlockList, error)              `perm:"read"`
 
 		ID      func(context.Context) (peer.ID, error)        `perm:"read"`
-		Version func(context.Context) (api.Version, error) `perm:"read"`
+		Version func(context.Context) (api.APIVersion, error) `perm:"read"`
 
 		LogList     func(context.Context) ([]string, error)     `perm:"write"`
 		LogSetLevel func(context.Context, string, string) error `perm:"write"`
@@ -304,7 +304,7 @@ type StorageMinerStruct struct {
 		MarketPendingDeals        func(ctx context.Context) (api.PendingDealInfo, error)                                                                                                                       `perm:"write"`
 		MarketPublishPendingDeals func(ctx context.Context) error                                                                                                                                              `perm:"admin"`
 
-		PledgeSector func(context.Context) error `perm:"write"`
+		PledgeSector func(context.Context) (abi.SectorID, error) `perm:"write"`
 
 		SectorsStatus                 func(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) `perm:"read"`
 		SectorsList                   func(context.Context) ([]abi.SectorNumber, error)                                             `perm:"read"`
@@ -389,7 +389,7 @@ type WorkerStruct struct {
 	Internal struct {
 		// TODO: lower perms
 
-		Version func(context.Context) (build.Version, error) `perm:"admin"`
+		Version func(context.Context) (api.Version, error) `perm:"admin"`
 
 		TaskTypes func(context.Context) (map[sealtasks.TaskType]struct{}, error) `perm:"admin"`
 		Paths     func(context.Context) ([]stores.StoragePath, error)            `perm:"admin"`
@@ -540,13 +540,17 @@ func (c *CommonStruct) NetAgentVersion(ctx context.Context, p peer.ID) (string,
 	return c.Internal.NetAgentVersion(ctx, p)
 }
 
+func (c *CommonStruct) NetPeerInfo(ctx context.Context, p peer.ID) (*api.ExtendedPeerInfo, error) {
+	return c.Internal.NetPeerInfo(ctx, p)
+}
+
 // ID implements API.ID
 func (c *CommonStruct) ID(ctx context.Context) (peer.ID, error) {
 	return c.Internal.ID(ctx)
 }
 
 // Version implements API.Version
-func (c *CommonStruct) Version(ctx context.Context) (api.Version, error) {
+func (c *CommonStruct) Version(ctx context.Context) (api.APIVersion, error) {
 	return c.Internal.Version(ctx)
 }
 
@@ -1274,7 +1278,7 @@ func (c *StorageMinerStruct) ActorAddressConfig(ctx context.Context) (api.Addres
 	return c.Internal.ActorAddressConfig(ctx)
 }
 
-func (c *StorageMinerStruct) PledgeSector(ctx context.Context) error {
+func (c *StorageMinerStruct) PledgeSector(ctx context.Context) (abi.SectorID, error) {
 	return c.Internal.PledgeSector(ctx)
 }
 
@@ -1610,7 +1614,7 @@ func (c *StorageMinerStruct) CheckProvable(ctx context.Context, pp abi.Registere
 
 // WorkerStruct
 
-func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) {
+func (w *WorkerStruct) Version(ctx context.Context) (api.Version, error) {
 	return w.Internal.Version(ctx)
 }
 

api/docgen/docgen.go

@@ -113,7 +113,7 @@ func init() {
 	addExample(network.Connected)
 	addExample(dtypes.NetworkName("lotus"))
 	addExample(api.SyncStateStage(1))
-	addExample(build.FullAPIVersion)
+	addExample(api.FullAPIVersion)
 	addExample(api.PCHInbound)
 	addExample(time.Minute)
 	addExample(datatransfer.TransferID(3))
@@ -123,6 +123,8 @@ func init() {
 	addExample(retrievalmarket.DealStatusNew)
 	addExample(network.ReachabilityPublic)
 	addExample(build.NewestNetworkVersion)
+	addExample(map[string]int{"name": 42})
+	addExample(map[string]time.Time{"name": time.Unix(1615243938, 0).UTC()})
 	addExample(&types.ExecutionTrace{
 		Msg:    exampleValue("init", reflect.TypeOf(&types.Message{}), nil).(*types.Message),
 		MsgRct: exampleValue("init", reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt),

api/test/paych.go

@@ -15,7 +15,7 @@ import (
 	cbor "github.com/ipfs/go-ipld-cbor"
 
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/api/apibstore"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -132,7 +132,7 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) {
 		t.Fatal("Unable to settle payment channel")
 	}
 
-	creatorStore := adt.WrapStore(ctx, cbor.NewCborStore(apibstore.NewAPIBlockstore(paymentCreator)))
+	creatorStore := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(paymentCreator)))
 
 	// wait for the receiver to submit their vouchers
 	ev := events.NewEvents(ctx, paymentCreator)

api/test/tape.go

@@ -6,7 +6,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/network"
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
@@ -75,23 +74,9 @@ func testTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration, after bool
 		<-done
 	}()
 
-	err = miner.PledgeSector(ctx)
+	sid, err := miner.PledgeSector(ctx)
 	require.NoError(t, err)
 
-	// Wait till done.
-	var sectorNo abi.SectorNumber
-	for {
-		s, err := miner.SectorsList(ctx) // Note - the test builder doesn't import genesis sectors into FSM
-		require.NoError(t, err)
-		fmt.Printf("Sectors: %d\n", len(s))
-		if len(s) == 1 {
-			sectorNo = s[0]
-			break
-		}
-
-		build.Clock.Sleep(100 * time.Millisecond)
-	}
-
 	fmt.Printf("All sectors is fsm\n")
 
 	// If before, we expect the precommit to fail
@@ -103,7 +88,7 @@ func testTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration, after bool
 	}
 
 	for {
-		st, err := miner.SectorsStatus(ctx, sectorNo, false)
+		st, err := miner.SectorsStatus(ctx, sid.Number, false)
 		require.NoError(t, err)
 		if st.State == successState {
 			break

api/test/test.go

@@ -155,13 +155,13 @@ var MineNext = miner.MineReq{
 }
 
 func (ts *testSuite) testVersion(t *testing.T) {
-	build.RunningNodeType = build.NodeFull
+	api.RunningNodeType = api.NodeFull
 
 	ctx := context.Background()
 	apis, _ := ts.makeNodes(t, OneFull, OneMiner)
-	api := apis[0]
+	napi := apis[0]
 
-	v, err := api.Version(ctx)
+	v, err := napi.Version(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}

api/test/window_post.go

@@ -162,7 +162,7 @@ func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n,
 			log.Errorf("WAIT")
 		}
 		log.Errorf("PLEDGING %d", i)
-		err := miner.PledgeSector(ctx)
+		_, err := miner.PledgeSector(ctx)
 		require.NoError(t, err)
 	}
 

api/types.go

@@ -3,6 +3,7 @@ package api
 import (
 	"encoding/json"
 	"fmt"
+	"time"
 
 	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/filecoin-project/go-state-types/abi"
@@ -99,3 +100,18 @@ type NetBlockList struct {
 	IPAddrs   []string
 	IPSubnets []string
 }
+
+type ExtendedPeerInfo struct {
+	ID          peer.ID
+	Agent       string
+	Addrs       []string
+	Protocols   []string
+	ConnMgrMeta *ConnMgrInfo
+}
+
+type ConnMgrInfo struct {
+	FirstSeen time.Time
+	Value     int
+	Tags      map[string]int
+	Conns     map[string]time.Time
+}

api/version.go

@@ -0,0 +1,71 @@
+package api
+
+import (
+	"fmt"
+
+	xerrors "golang.org/x/xerrors"
+)
+
+type Version uint32
+
+func newVer(major, minor, patch uint8) Version {
+	return Version(uint32(major)<<16 | uint32(minor)<<8 | uint32(patch))
+}
+
+// Ints returns (major, minor, patch) versions
+func (ve Version) Ints() (uint32, uint32, uint32) {
+	v := uint32(ve)
+	return (v & majorOnlyMask) >> 16, (v & minorOnlyMask) >> 8, v & patchOnlyMask
+}
+
+func (ve Version) String() string {
+	vmj, vmi, vp := ve.Ints()
+	return fmt.Sprintf("%d.%d.%d", vmj, vmi, vp)
+}
+
+func (ve Version) EqMajorMinor(v2 Version) bool {
+	return ve&minorMask == v2&minorMask
+}
+
+type NodeType int
+
+const (
+	NodeUnknown NodeType = iota
+
+	NodeFull
+	NodeMiner
+	NodeWorker
+)
+
+var RunningNodeType NodeType
+
+func VersionForType(nodeType NodeType) (Version, error) {
+	switch nodeType {
+	case NodeFull:
+		return FullAPIVersion, nil
+	case NodeMiner:
+		return MinerAPIVersion, nil
+	case NodeWorker:
+		return WorkerAPIVersion, nil
+	default:
+		return Version(0), xerrors.Errorf("unknown node type %d", nodeType)
+	}
+}
+
+// semver versions of the rpc api exposed
+var (
+	FullAPIVersion   = newVer(1, 1, 0)
+	MinerAPIVersion  = newVer(1, 0, 1)
+	WorkerAPIVersion = newVer(1, 0, 0)
+)
+
+//nolint:varcheck,deadcode
+const (
+	majorMask = 0xff0000
+	minorMask = 0xffff00
+	patchMask = 0xffffff
+
+	majorOnlyMask = 0xff0000
+	minorOnlyMask = 0x00ff00
+	patchOnlyMask = 0x0000ff
+)

blockstore/api.go

@@ -0,0 +1,66 @@
+package blockstore
+
+import (
+	"context"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	"golang.org/x/xerrors"
+)
+
+type ChainIO interface {
+	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
+	ChainHasObj(context.Context, cid.Cid) (bool, error)
+}
+
+type apiBlockstore struct {
+	api ChainIO
+}
+
+// This blockstore is adapted in the constructor.
+var _ BasicBlockstore = (*apiBlockstore)(nil)
+
+func NewAPIBlockstore(cio ChainIO) Blockstore {
+	bs := &apiBlockstore{api: cio}
+	return Adapt(bs) // return an adapted blockstore.
+}
+
+func (a *apiBlockstore) DeleteBlock(cid.Cid) error {
+	return xerrors.New("not supported")
+}
+
+func (a *apiBlockstore) Has(c cid.Cid) (bool, error) {
+	return a.api.ChainHasObj(context.TODO(), c)
+}
+
+func (a *apiBlockstore) Get(c cid.Cid) (blocks.Block, error) {
+	bb, err := a.api.ChainReadObj(context.TODO(), c)
+	if err != nil {
+		return nil, err
+	}
+	return blocks.NewBlockWithCid(bb, c)
+}
+
+func (a *apiBlockstore) GetSize(c cid.Cid) (int, error) {
+	bb, err := a.api.ChainReadObj(context.TODO(), c)
+	if err != nil {
+		return 0, err
+	}
+	return len(bb), nil
+}
+
+func (a *apiBlockstore) Put(blocks.Block) error {
+	return xerrors.New("not supported")
+}
+
+func (a *apiBlockstore) PutMany([]blocks.Block) error {
+	return xerrors.New("not supported")
+}
+
+func (a *apiBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+	return nil, xerrors.New("not supported")
+}
+
+func (a *apiBlockstore) HashOnRead(enabled bool) {
+	return
+}

blockstore/badger/blockstore.go

@@ -16,7 +16,7 @@ import (
 	logger "github.com/ipfs/go-log/v2"
 	pool "github.com/libp2p/go-buffer-pool"
 
-	"github.com/filecoin-project/lotus/lib/blockstore"
+	"github.com/filecoin-project/lotus/blockstore"
 )
 
 var (
@@ -110,10 +110,7 @@ func Open(opts Options) (*Blockstore, error) {
 		return nil, fmt.Errorf("failed to open badger blockstore: %w", err)
 	}
 
-	bs := &Blockstore{
+	bs := &Blockstore{DB: db}
-		DB: db,
-	}
-
 	if p := opts.Prefix; p != "" {
 		bs.prefixing = true
 		bs.prefix = []byte(p)
@@ -134,6 +131,25 @@ func (b *Blockstore) Close() error {
 	return b.DB.Close()
 }
 
+// CollectGarbage runs garbage collection on the value log
+func (b *Blockstore) CollectGarbage() error {
+	if atomic.LoadInt64(&b.state) != stateOpen {
+		return ErrBlockstoreClosed
+	}
+
+	var err error
+	for err == nil {
+		err = b.DB.RunValueLogGC(0.125)
+	}
+
+	if err == badger.ErrNoRewrite {
+		// not really an error in this case
+		return nil
+	}
+
+	return err
+}
+
 // View implements blockstore.Viewer, which leverages zero-copy read-only
 // access to values.
 func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
@@ -321,6 +337,44 @@ func (b *Blockstore) DeleteBlock(cid cid.Cid) error {
 	})
 }
 
+func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
+	if atomic.LoadInt64(&b.state) != stateOpen {
+		return ErrBlockstoreClosed
+	}
+
+	batch := b.DB.NewWriteBatch()
+	defer batch.Cancel()
+
+	// toReturn tracks the byte slices to return to the pool, if we're using key
+	// prefixing. we can't return each slice to the pool after each Set, because
+	// badger holds on to the slice.
+	var toReturn [][]byte
+	if b.prefixing {
+		toReturn = make([][]byte, 0, len(cids))
+		defer func() {
+			for _, b := range toReturn {
+				KeyPool.Put(b)
+			}
+		}()
+	}
+
+	for _, cid := range cids {
+		k, pooled := b.PooledStorageKey(cid)
+		if pooled {
+			toReturn = append(toReturn, k)
+		}
+		if err := batch.Delete(k); err != nil {
+			return err
+		}
+	}
+
+	err := batch.Flush()
+	if err != nil {
+		err = fmt.Errorf("failed to delete blocks from badger blockstore: %w", err)
+	}
+	return err
+}
+
 // AllKeysChan implements Blockstore.AllKeysChan.
 func (b *Blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
 	if atomic.LoadInt64(&b.state) != stateOpen {

blockstore/badger/blockstore_test.go

@@ -6,8 +6,9 @@ import (
 	"testing"
 
 	blocks "github.com/ipfs/go-block-format"
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/lotus/blockstore"
 )
 
 func TestBadgerBlockstore(t *testing.T) {
@@ -60,8 +61,8 @@ func TestStorageKey(t *testing.T) {
 	require.Equal(t, k3, k2)
 }
 
-func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (bs blockstore.Blockstore, path string) {
+func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
-	return func(tb testing.TB) (bs blockstore.Blockstore, path string) {
+	return func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
 		tb.Helper()
 
 		path, err := ioutil.TempDir("", "")
@@ -82,8 +83,8 @@ func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (
 	}
 }
 
-func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB, path string) (bs blockstore.Blockstore, err error) {
+func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) {
-	return func(tb testing.TB, path string) (bs blockstore.Blockstore, err error) {
+	return func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) {
 		tb.Helper()
 		return Open(optsSupplier(path))
 	}

blockstore/badger/blockstore_test_suite.go

@@ -8,18 +8,19 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	u "github.com/ipfs/go-ipfs-util"
 
+	"github.com/filecoin-project/lotus/blockstore"
+
 	"github.com/stretchr/testify/require"
 )
 
 // TODO: move this to go-ipfs-blockstore.
 type Suite struct {
-	NewBlockstore  func(tb testing.TB) (bs blockstore.Blockstore, path string)
+	NewBlockstore  func(tb testing.TB) (bs blockstore.BasicBlockstore, path string)
-	OpenBlockstore func(tb testing.TB, path string) (bs blockstore.Blockstore, err error)
+	OpenBlockstore func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error)
 }
 
 func (s *Suite) RunTests(t *testing.T, prefix string) {
@@ -290,7 +291,7 @@ func (s *Suite) TestDelete(t *testing.T) {
 
 }
 
-func insertBlocks(t *testing.T, bs blockstore.Blockstore, count int) []cid.Cid {
+func insertBlocks(t *testing.T, bs blockstore.BasicBlockstore, count int) []cid.Cid {
 	keys := make([]cid.Cid, count)
 	for i := 0; i < count; i++ {
 		block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))

blockstore/blockstore.go

@@ -0,0 +1,95 @@
+package blockstore
+
+import (
+	cid "github.com/ipfs/go-cid"
+	ds "github.com/ipfs/go-datastore"
+	logging "github.com/ipfs/go-log/v2"
+
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+)
+
+var log = logging.Logger("blockstore")
+
+var ErrNotFound = blockstore.ErrNotFound
+
+// Blockstore is the blockstore interface used by Lotus. It is the union
+// of the basic go-ipfs blockstore, with other capabilities required by Lotus,
+// e.g. View or Sync.
+type Blockstore interface {
+	blockstore.Blockstore
+	blockstore.Viewer
+	BatchDeleter
+}
+
+// BasicBlockstore is an alias to the original IPFS Blockstore.
+type BasicBlockstore = blockstore.Blockstore
+
+type Viewer = blockstore.Viewer
+
+type BatchDeleter interface {
+	DeleteMany(cids []cid.Cid) error
+}
+
+// WrapIDStore wraps the underlying blockstore in an "identity" blockstore.
+// The ID store filters out all puts for blocks with CIDs using the "identity"
+// hash function. It also extracts inlined blocks from CIDs using the identity
+// hash function and returns them on get/has, ignoring the contents of the
+// blockstore.
+func WrapIDStore(bstore blockstore.Blockstore) Blockstore {
+	if is, ok := bstore.(*idstore); ok {
+		// already wrapped
+		return is
+	}
+
+	if bs, ok := bstore.(Blockstore); ok {
+		// we need to wrap our own because we don't want to neuter the DeleteMany method
+		// the underlying blockstore has implemented an (efficient) DeleteMany
+		return NewIDStore(bs)
+	}
+
+	// The underlying blockstore does not implement DeleteMany, so we need to shim it.
+	// This is less efficient as it'll iterate and perform single deletes.
+	return NewIDStore(Adapt(bstore))
+}
+
+// FromDatastore creates a new blockstore backed by the given datastore.
+func FromDatastore(dstore ds.Batching) Blockstore {
+	return WrapIDStore(blockstore.NewBlockstore(dstore))
+}
+
+type adaptedBlockstore struct {
+	blockstore.Blockstore
+}
+
+var _ Blockstore = (*adaptedBlockstore)(nil)
+
+func (a *adaptedBlockstore) View(cid cid.Cid, callback func([]byte) error) error {
+	blk, err := a.Get(cid)
+	if err != nil {
+		return err
+	}
+	return callback(blk.RawData())
+}
+
+func (a *adaptedBlockstore) DeleteMany(cids []cid.Cid) error {
+	for _, cid := range cids {
+		err := a.DeleteBlock(cid)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Adapt adapts a standard blockstore to a Lotus blockstore by
+// enriching it with the extra methods that Lotus requires (e.g. View, Sync).
+//
+// View proxies over to Get and calls the callback with the value supplied by Get.
+// Sync noops.
+func Adapt(bs blockstore.Blockstore) Blockstore {
+	if ret, ok := bs.(Blockstore); ok {
+		return ret
+	}
+	return &adaptedBlockstore{bs}
+}

blockstore/buffered.go

@@ -0,0 +1,174 @@
+package blockstore
+
+import (
+	"context"
+	"os"
+
+	block "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+)
+
+// buflog is a logger for the buffered blockstore. It is subscoped from the
+// blockstore logger.
+var buflog = log.Named("buf")
+
+type BufferedBlockstore struct {
+	read  Blockstore
+	write Blockstore
+}
+
+func NewBuffered(base Blockstore) *BufferedBlockstore {
+	var buf Blockstore
+	if os.Getenv("LOTUS_DISABLE_VM_BUF") == "iknowitsabadidea" {
+		buflog.Warn("VM BLOCKSTORE BUFFERING IS DISABLED")
+		buf = base
+	} else {
+		buf = NewMemory()
+	}
+
+	bs := &BufferedBlockstore{
+		read:  base,
+		write: buf,
+	}
+	return bs
+}
+
+func NewTieredBstore(r Blockstore, w Blockstore) *BufferedBlockstore {
+	return &BufferedBlockstore{
+		read:  r,
+		write: w,
+	}
+}
+
+var (
+	_ Blockstore = (*BufferedBlockstore)(nil)
+	_ Viewer     = (*BufferedBlockstore)(nil)
+)
+
+func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+	a, err := bs.read.AllKeysChan(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	b, err := bs.write.AllKeysChan(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	out := make(chan cid.Cid)
+	go func() {
+		defer close(out)
+		for a != nil || b != nil {
+			select {
+			case val, ok := <-a:
+				if !ok {
+					a = nil
+				} else {
+					select {
+					case out <- val:
+					case <-ctx.Done():
+						return
+					}
+				}
+			case val, ok := <-b:
+				if !ok {
+					b = nil
+				} else {
+					select {
+					case out <- val:
+					case <-ctx.Done():
+						return
+					}
+				}
+			}
+		}
+	}()
+
+	return out, nil
+}
+
+func (bs *BufferedBlockstore) DeleteBlock(c cid.Cid) error {
+	if err := bs.read.DeleteBlock(c); err != nil {
+		return err
+	}
+
+	return bs.write.DeleteBlock(c)
+}
+
+func (bs *BufferedBlockstore) DeleteMany(cids []cid.Cid) error {
+	if err := bs.read.DeleteMany(cids); err != nil {
+		return err
+	}
+
+	return bs.write.DeleteMany(cids)
+}
+
+func (bs *BufferedBlockstore) View(c cid.Cid, callback func([]byte) error) error {
+	// both stores are viewable.
+	if err := bs.write.View(c, callback); err == ErrNotFound {
+		// not found in write blockstore; fall through.
+	} else {
+		return err // propagate errors, or nil, i.e. found.
+	}
+	return bs.read.View(c, callback)
+}
+
+func (bs *BufferedBlockstore) Get(c cid.Cid) (block.Block, error) {
+	if out, err := bs.write.Get(c); err != nil {
+		if err != ErrNotFound {
+			return nil, err
+		}
+	} else {
+		return out, nil
+	}
+
+	return bs.read.Get(c)
+}
+
+func (bs *BufferedBlockstore) GetSize(c cid.Cid) (int, error) {
+	s, err := bs.read.GetSize(c)
+	if err == ErrNotFound || s == 0 {
+		return bs.write.GetSize(c)
+	}
+
+	return s, err
+}
+
+func (bs *BufferedBlockstore) Put(blk block.Block) error {
+	has, err := bs.read.Has(blk.Cid()) // TODO: consider dropping this check
+	if err != nil {
+		return err
+	}
+
+	if has {
+		return nil
+	}
+
+	return bs.write.Put(blk)
+}
+
+func (bs *BufferedBlockstore) Has(c cid.Cid) (bool, error) {
+	has, err := bs.write.Has(c)
+	if err != nil {
+		return false, err
+	}
+	if has {
+		return true, nil
+	}
+
+	return bs.read.Has(c)
+}
+
+func (bs *BufferedBlockstore) HashOnRead(hor bool) {
+	bs.read.HashOnRead(hor)
+	bs.write.HashOnRead(hor)
+}
+
+func (bs *BufferedBlockstore) PutMany(blks []block.Block) error {
+	return bs.write.PutMany(blks)
+}
+
+func (bs *BufferedBlockstore) Read() Blockstore {
+	return bs.read
+}

blockstore/cached.go

@@ -0,0 +1,25 @@
+package blockstore
+
+import (
+	"context"
+
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+)
+
+type CacheOpts = blockstore.CacheOpts
+
+func DefaultCacheOpts() CacheOpts {
+	return CacheOpts{
+		HasBloomFilterSize:   0,
+		HasBloomFilterHashes: 0,
+		HasARCCacheSize:      512 << 10,
+	}
+}
+
+func CachedBlockstore(ctx context.Context, bs Blockstore, opts CacheOpts) (Blockstore, error) {
+	cached, err := blockstore.CachedBlockstore(ctx, bs, opts)
+	if err != nil {
+		return nil, err
+	}
+	return WrapIDStore(cached), nil
+}

blockstore/doc.go

@@ -0,0 +1,9 @@
+// Package blockstore and subpackages contain most of the blockstore
+// implementations used by Lotus.
+//
+// Blockstores not ultimately constructed out of the building blocks in this
+// package may not work properly.
+//
+// This package re-exports parts of the go-ipfs-blockstore package such that
+// no other package needs to import it directly, for ergonomics and traceability.
+package blockstore

blockstore/fallback.go

@@ -9,48 +9,61 @@ import (
 
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
-	logging "github.com/ipfs/go-log"
 )
 
-var log = logging.Logger("blockstore")
+// UnwrapFallbackStore takes a blockstore, and returns the underlying blockstore
-
+// if it was a FallbackStore. Otherwise, it just returns the supplied store
-type FallbackStore struct {
+// unmodified.
-	blockstore.Blockstore
+func UnwrapFallbackStore(bs Blockstore) (Blockstore, bool) {
-
+	if fbs, ok := bs.(*FallbackStore); ok {
-	fallbackGetBlock func(context.Context, cid.Cid) (blocks.Block, error)
+		return fbs.Blockstore, true
-	lk               sync.RWMutex
+	}
+	return bs, false
 }
 
-func (fbs *FallbackStore) SetFallback(fg func(context.Context, cid.Cid) (blocks.Block, error)) {
+// FallbackStore is a read-through store that queries another (potentially
+// remote) source if the block is not found locally. If the block is found
+// during the fallback, it stores it in the local store.
+type FallbackStore struct {
+	Blockstore
+
+	lk sync.RWMutex
+	// missFn is the function that will be invoked on a local miss to pull the
+	// block from elsewhere.
+	missFn func(context.Context, cid.Cid) (blocks.Block, error)
+}
+
+var _ Blockstore = (*FallbackStore)(nil)
+
+func (fbs *FallbackStore) SetFallback(missFn func(context.Context, cid.Cid) (blocks.Block, error)) {
 	fbs.lk.Lock()
 	defer fbs.lk.Unlock()
 
-	fbs.fallbackGetBlock = fg
+	fbs.missFn = missFn
 }
 
 func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
-	log.Errorw("fallbackstore: Block not found locally, fetching from the network", "cid", c)
+	log.Warnf("fallbackstore: block not found locally, fetching from the network; cid: %s", c)
 	fbs.lk.RLock()
 	defer fbs.lk.RUnlock()
 
-	if fbs.fallbackGetBlock == nil {
+	if fbs.missFn == nil {
 		// FallbackStore wasn't configured yet (chainstore/bitswap aren't up yet)
 		// Wait for a bit and retry
 		fbs.lk.RUnlock()
 		time.Sleep(5 * time.Second)
 		fbs.lk.RLock()
 
-		if fbs.fallbackGetBlock == nil {
+		if fbs.missFn == nil {
-			log.Errorw("fallbackstore: fallbackGetBlock not configured yet")
+			log.Errorw("fallbackstore: missFn not configured yet")
-			return nil, blockstore.ErrNotFound
+			return nil, ErrNotFound
 		}
 	}
 
 	ctx, cancel := context.WithTimeout(context.TODO(), 120*time.Second)
 	defer cancel()
 
-	b, err := fbs.fallbackGetBlock(ctx, c)
+	b, err := fbs.missFn(ctx, c)
 	if err != nil {
 		return nil, err
 	}
@@ -69,7 +82,7 @@ func (fbs *FallbackStore) Get(c cid.Cid) (blocks.Block, error) {
 	switch err {
 	case nil:
 		return b, nil
-	case blockstore.ErrNotFound:
+	case ErrNotFound:
 		return fbs.getFallback(c)
 	default:
 		return b, err
@@ -81,7 +94,7 @@ func (fbs *FallbackStore) GetSize(c cid.Cid) (int, error) {
 	switch err {
 	case nil:
 		return sz, nil
-	case blockstore.ErrNotFound:
+	case ErrNotFound:
 		b, err := fbs.getFallback(c)
 		if err != nil {
 			return 0, err
@@ -91,5 +104,3 @@ func (fbs *FallbackStore) GetSize(c cid.Cid) (int, error) {
 		return sz, err
 	}
 }
-
-var _ blockstore.Blockstore = &FallbackStore{}

blockstore/idstore.go

@@ -0,0 +1,174 @@
+package blockstore
+
+import (
+	"context"
+	"io"
+
+	"golang.org/x/xerrors"
+
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
+	mh "github.com/multiformats/go-multihash"
+)
+
+var _ Blockstore = (*idstore)(nil)
+
+type idstore struct {
+	bs Blockstore
+}
+
+func NewIDStore(bs Blockstore) Blockstore {
+	return &idstore{bs: bs}
+}
+
+func decodeCid(cid cid.Cid) (inline bool, data []byte, err error) {
+	if cid.Prefix().MhType != mh.IDENTITY {
+		return false, nil, nil
+	}
+
+	dmh, err := mh.Decode(cid.Hash())
+	if err != nil {
+		return false, nil, err
+	}
+
+	if dmh.Code == mh.IDENTITY {
+		return true, dmh.Digest, nil
+	}
+
+	return false, nil, err
+}
+
+func (b *idstore) Has(cid cid.Cid) (bool, error) {
+	inline, _, err := decodeCid(cid)
+	if err != nil {
+		return false, xerrors.Errorf("error decoding Cid: %w", err)
+	}
+
+	if inline {
+		return true, nil
+	}
+
+	return b.bs.Has(cid)
+}
+
+func (b *idstore) Get(cid cid.Cid) (blocks.Block, error) {
+	inline, data, err := decodeCid(cid)
+	if err != nil {
+		return nil, xerrors.Errorf("error decoding Cid: %w", err)
+	}
+
+	if inline {
+		return blocks.NewBlockWithCid(data, cid)
+	}
+
+	return b.bs.Get(cid)
+}
+
+func (b *idstore) GetSize(cid cid.Cid) (int, error) {
+	inline, data, err := decodeCid(cid)
+	if err != nil {
+		return 0, xerrors.Errorf("error decoding Cid: %w", err)
+	}
+
+	if inline {
+		return len(data), err
+	}
+
+	return b.bs.GetSize(cid)
+}
+
+func (b *idstore) View(cid cid.Cid, cb func([]byte) error) error {
+	inline, data, err := decodeCid(cid)
+	if err != nil {
+		return xerrors.Errorf("error decoding Cid: %w", err)
+	}
+
+	if inline {
+		return cb(data)
+	}
+
+	return b.bs.View(cid, cb)
+}
+
+func (b *idstore) Put(blk blocks.Block) error {
+	inline, _, err := decodeCid(blk.Cid())
+	if err != nil {
+		return xerrors.Errorf("error decoding Cid: %w", err)
+	}
+
+	if inline {
+		return nil
+	}
+
+	return b.bs.Put(blk)
+}
+
+func (b *idstore) PutMany(blks []blocks.Block) error {
+	toPut := make([]blocks.Block, 0, len(blks))
+	for _, blk := range blks {
+		inline, _, err := decodeCid(blk.Cid())
+		if err != nil {
+			return xerrors.Errorf("error decoding Cid: %w", err)
+		}
+
+		if inline {
+			continue
+		}
+		toPut = append(toPut, blk)
+	}
+
+	if len(toPut) > 0 {
+		return b.bs.PutMany(toPut)
+	}
+
+	return nil
+}
+
+func (b *idstore) DeleteBlock(cid cid.Cid) error {
+	inline, _, err := decodeCid(cid)
+	if err != nil {
+		return xerrors.Errorf("error decoding Cid: %w", err)
+	}
+
+	if inline {
+		return nil
+	}
+
+	return b.bs.DeleteBlock(cid)
+}
+
+func (b *idstore) DeleteMany(cids []cid.Cid) error {
+	toDelete := make([]cid.Cid, 0, len(cids))
+	for _, cid := range cids {
+		inline, _, err := decodeCid(cid)
+		if err != nil {
+			return xerrors.Errorf("error decoding Cid: %w", err)
+		}
+
+		if inline {
+			continue
+		}
+		toDelete = append(toDelete, cid)
+	}
+
+	if len(toDelete) > 0 {
+		return b.bs.DeleteMany(toDelete)
+	}
+
+	return nil
+}
+
+func (b *idstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+	return b.bs.AllKeysChan(ctx)
+}
+
+func (b *idstore) HashOnRead(enabled bool) {
+	b.bs.HashOnRead(enabled)
+}
+
+func (b *idstore) Close() error {
+	if c, ok := b.bs.(io.Closer); ok {
+		return c.Close()
+	}
+	return nil
+}

blockstore/ipfs.go

@@ -1,4 +1,4 @@
-package ipfsbstore
+package blockstore
 
 import (
 	"bytes"
@@ -16,16 +16,16 @@ import (
 	iface "github.com/ipfs/interface-go-ipfs-core"
 	"github.com/ipfs/interface-go-ipfs-core/options"
 	"github.com/ipfs/interface-go-ipfs-core/path"
-
-	"github.com/filecoin-project/lotus/lib/blockstore"
 )
 
-type IpfsBstore struct {
+type IPFSBlockstore struct {
 	ctx             context.Context
 	api, offlineAPI iface.CoreAPI
 }
 
-func NewIpfsBstore(ctx context.Context, onlineMode bool) (*IpfsBstore, error) {
+var _ BasicBlockstore = (*IPFSBlockstore)(nil)
+
+func NewLocalIPFSBlockstore(ctx context.Context, onlineMode bool) (Blockstore, error) {
 	localApi, err := httpapi.NewLocalApi()
 	if err != nil {
 		return nil, xerrors.Errorf("getting local ipfs api: %w", err)
@@ -34,6 +34,7 @@ func NewIpfsBstore(ctx context.Context, onlineMode bool) (*IpfsBstore, error) {
 	if err != nil {
 		return nil, xerrors.Errorf("setting offline mode: %s", err)
 	}
+
 	offlineAPI := api
 	if onlineMode {
 		offlineAPI, err = localApi.WithOptions(options.Api.Offline(true))
@@ -42,14 +43,16 @@ func NewIpfsBstore(ctx context.Context, onlineMode bool) (*IpfsBstore, error) {
 		}
 	}
 
-	return &IpfsBstore{
+	bs := &IPFSBlockstore{
 		ctx:        ctx,
 		api:        api,
 		offlineAPI: offlineAPI,
-	}, nil
+	}
+
+	return Adapt(bs), nil
 }
 
-func NewRemoteIpfsBstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineMode bool) (*IpfsBstore, error) {
+func NewRemoteIPFSBlockstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineMode bool) (Blockstore, error) {
 	httpApi, err := httpapi.NewApi(maddr)
 	if err != nil {
 		return nil, xerrors.Errorf("setting remote ipfs api: %w", err)
@@ -58,6 +61,7 @@ func NewRemoteIpfsBstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineM
 	if err != nil {
 		return nil, xerrors.Errorf("applying offline mode: %s", err)
 	}
+
 	offlineAPI := api
 	if onlineMode {
 		offlineAPI, err = httpApi.WithOptions(options.Api.Offline(true))
@@ -66,18 +70,20 @@ func NewRemoteIpfsBstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineM
 		}
 	}
 
-	return &IpfsBstore{
+	bs := &IPFSBlockstore{
 		ctx:        ctx,
 		api:        api,
 		offlineAPI: offlineAPI,
-	}, nil
+	}
+
+	return Adapt(bs), nil
 }
 
-func (i *IpfsBstore) DeleteBlock(cid cid.Cid) error {
+func (i *IPFSBlockstore) DeleteBlock(cid cid.Cid) error {
 	return xerrors.Errorf("not supported")
 }
 
-func (i *IpfsBstore) Has(cid cid.Cid) (bool, error) {
+func (i *IPFSBlockstore) Has(cid cid.Cid) (bool, error) {
 	_, err := i.offlineAPI.Block().Stat(i.ctx, path.IpldPath(cid))
 	if err != nil {
 		// The underlying client is running in Offline mode.
@@ -93,7 +99,7 @@ func (i *IpfsBstore) Has(cid cid.Cid) (bool, error) {
 	return true, nil
 }
 
-func (i *IpfsBstore) Get(cid cid.Cid) (blocks.Block, error) {
+func (i *IPFSBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
 	rd, err := i.api.Block().Get(i.ctx, path.IpldPath(cid))
 	if err != nil {
 		return nil, xerrors.Errorf("getting ipfs block: %w", err)
@@ -107,7 +113,7 @@ func (i *IpfsBstore) Get(cid cid.Cid) (blocks.Block, error) {
 	return blocks.NewBlockWithCid(data, cid)
 }
 
-func (i *IpfsBstore) GetSize(cid cid.Cid) (int, error) {
+func (i *IPFSBlockstore) GetSize(cid cid.Cid) (int, error) {
 	st, err := i.api.Block().Stat(i.ctx, path.IpldPath(cid))
 	if err != nil {
 		return 0, xerrors.Errorf("getting ipfs block: %w", err)
@@ -116,7 +122,7 @@ func (i *IpfsBstore) GetSize(cid cid.Cid) (int, error) {
 	return st.Size(), nil
 }
 
-func (i *IpfsBstore) Put(block blocks.Block) error {
+func (i *IPFSBlockstore) Put(block blocks.Block) error {
 	mhd, err := multihash.Decode(block.Cid().Hash())
 	if err != nil {
 		return err
@@ -128,7 +134,7 @@ func (i *IpfsBstore) Put(block blocks.Block) error {
 	return err
 }
 
-func (i *IpfsBstore) PutMany(blocks []blocks.Block) error {
+func (i *IPFSBlockstore) PutMany(blocks []blocks.Block) error {
 	// TODO: could be done in parallel
 
 	for _, block := range blocks {
@@ -140,12 +146,10 @@ func (i *IpfsBstore) PutMany(blocks []blocks.Block) error {
 	return nil
 }
 
-func (i *IpfsBstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+func (i *IPFSBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
 	return nil, xerrors.Errorf("not supported")
 }
 
-func (i *IpfsBstore) HashOnRead(enabled bool) {
+func (i *IPFSBlockstore) HashOnRead(enabled bool) {
 	return // TODO: We could technically support this, but..
 }
-
-var _ blockstore.Blockstore = &IpfsBstore{}

blockstore/mem.go

@@ -7,20 +7,32 @@ import (
 	"github.com/ipfs/go-cid"
 )
 
-// MemStore is a terminal blockstore that keeps blocks in memory.
+// NewMemory returns a temporary memory-backed blockstore.
-type MemStore map[cid.Cid]blocks.Block
+func NewMemory() MemBlockstore {
+	return make(MemBlockstore)
+}
 
-func (m MemStore) DeleteBlock(k cid.Cid) error {
+// MemBlockstore is a terminal blockstore that keeps blocks in memory.
+type MemBlockstore map[cid.Cid]blocks.Block
+
+func (m MemBlockstore) DeleteBlock(k cid.Cid) error {
 	delete(m, k)
 	return nil
 }
 
-func (m MemStore) Has(k cid.Cid) (bool, error) {
+func (m MemBlockstore) DeleteMany(ks []cid.Cid) error {
+	for _, k := range ks {
+		delete(m, k)
+	}
+	return nil
+}
+
+func (m MemBlockstore) Has(k cid.Cid) (bool, error) {
 	_, ok := m[k]
 	return ok, nil
 }
 
-func (m MemStore) View(k cid.Cid, callback func([]byte) error) error {
+func (m MemBlockstore) View(k cid.Cid, callback func([]byte) error) error {
 	b, ok := m[k]
 	if !ok {
 		return ErrNotFound
@@ -28,7 +40,7 @@ func (m MemStore) View(k cid.Cid, callback func([]byte) error) error {
 	return callback(b.RawData())
 }
 
-func (m MemStore) Get(k cid.Cid) (blocks.Block, error) {
+func (m MemBlockstore) Get(k cid.Cid) (blocks.Block, error) {
 	b, ok := m[k]
 	if !ok {
 		return nil, ErrNotFound
@@ -37,7 +49,7 @@ func (m MemStore) Get(k cid.Cid) (blocks.Block, error) {
 }
 
 // GetSize returns the CIDs mapped BlockSize
-func (m MemStore) GetSize(k cid.Cid) (int, error) {
+func (m MemBlockstore) GetSize(k cid.Cid) (int, error) {
 	b, ok := m[k]
 	if !ok {
 		return 0, ErrNotFound
@@ -46,7 +58,7 @@ func (m MemStore) GetSize(k cid.Cid) (int, error) {
 }
 
 // Put puts a given block to the underlying datastore
-func (m MemStore) Put(b blocks.Block) error {
+func (m MemBlockstore) Put(b blocks.Block) error {
 	// Convert to a basic block for safety, but try to reuse the existing
 	// block if it's already a basic block.
 	k := b.Cid()
@@ -64,7 +76,7 @@ func (m MemStore) Put(b blocks.Block) error {
 
 // PutMany puts a slice of blocks at the same time using batching
 // capabilities of the underlying datastore whenever possible.
-func (m MemStore) PutMany(bs []blocks.Block) error {
+func (m MemBlockstore) PutMany(bs []blocks.Block) error {
 	for _, b := range bs {
 		_ = m.Put(b) // can't fail
 	}
@@ -74,7 +86,7 @@ func (m MemStore) PutMany(bs []blocks.Block) error {
 // AllKeysChan returns a channel from which
 // the CIDs in the Blockstore can be read. It should respect
 // the given context, closing the channel if it becomes Done.
-func (m MemStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+func (m MemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
 	ch := make(chan cid.Cid, len(m))
 	for k := range m {
 		ch <- k
@@ -85,6 +97,6 @@ func (m MemStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
 
 // HashOnRead specifies if every read block should be
 // rehashed to make sure it matches its CID.
-func (m MemStore) HashOnRead(enabled bool) {
+func (m MemBlockstore) HashOnRead(enabled bool) {
 	// no-op
 }

blockstore/metrics.go

@@ -0,0 +1,154 @@
+package blockstore
+
+import (
+	"time"
+
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
+)
+
+//
+// Currently unused, but kept in repo in case we introduce one of the candidate
+// cache implementations (Freecache, Ristretto), both of which report these
+// metrics.
+//
+
+// CacheMetricsEmitInterval is the interval at which metrics are emitted onto
+// OpenCensus.
+var CacheMetricsEmitInterval = 5 * time.Second
+
+var (
+	CacheName, _ = tag.NewKey("cache_name")
+)
+
+// CacheMeasures groups all metrics emitted by the blockstore caches.
+var CacheMeasures = struct {
+	HitRatio       *stats.Float64Measure
+	Hits           *stats.Int64Measure
+	Misses         *stats.Int64Measure
+	Entries        *stats.Int64Measure
+	QueriesServed  *stats.Int64Measure
+	Adds           *stats.Int64Measure
+	Updates        *stats.Int64Measure
+	Evictions      *stats.Int64Measure
+	CostAdded      *stats.Int64Measure
+	CostEvicted    *stats.Int64Measure
+	SetsDropped    *stats.Int64Measure
+	SetsRejected   *stats.Int64Measure
+	QueriesDropped *stats.Int64Measure
+}{
+	HitRatio:       stats.Float64("blockstore/cache/hit_ratio", "Hit ratio of blockstore cache", stats.UnitDimensionless),
+	Hits:           stats.Int64("blockstore/cache/hits", "Total number of hits at blockstore cache", stats.UnitDimensionless),
+	Misses:         stats.Int64("blockstore/cache/misses", "Total number of misses at blockstore cache", stats.UnitDimensionless),
+	Entries:        stats.Int64("blockstore/cache/entry_count", "Total number of entries currently in the blockstore cache", stats.UnitDimensionless),
+	QueriesServed:  stats.Int64("blockstore/cache/queries_served", "Total number of queries served by the blockstore cache", stats.UnitDimensionless),
+	Adds:           stats.Int64("blockstore/cache/adds", "Total number of adds to blockstore cache", stats.UnitDimensionless),
+	Updates:        stats.Int64("blockstore/cache/updates", "Total number of updates in blockstore cache", stats.UnitDimensionless),
+	Evictions:      stats.Int64("blockstore/cache/evictions", "Total number of evictions from blockstore cache", stats.UnitDimensionless),
+	CostAdded:      stats.Int64("blockstore/cache/cost_added", "Total cost (byte size) of entries added into blockstore cache", stats.UnitBytes),
+	CostEvicted:    stats.Int64("blockstore/cache/cost_evicted", "Total cost (byte size) of entries evicted by blockstore cache", stats.UnitBytes),
+	SetsDropped:    stats.Int64("blockstore/cache/sets_dropped", "Total number of sets dropped by blockstore cache", stats.UnitDimensionless),
+	SetsRejected:   stats.Int64("blockstore/cache/sets_rejected", "Total number of sets rejected by blockstore cache", stats.UnitDimensionless),
+	QueriesDropped: stats.Int64("blockstore/cache/queries_dropped", "Total number of queries dropped by blockstore cache", stats.UnitDimensionless),
+}
+
+// CacheViews groups all cache-related default views.
+var CacheViews = struct {
+	HitRatio       *view.View
+	Hits           *view.View
+	Misses         *view.View
+	Entries        *view.View
+	QueriesServed  *view.View
+	Adds           *view.View
+	Updates        *view.View
+	Evictions      *view.View
+	CostAdded      *view.View
+	CostEvicted    *view.View
+	SetsDropped    *view.View
+	SetsRejected   *view.View
+	QueriesDropped *view.View
+}{
+	HitRatio: &view.View{
+		Measure:     CacheMeasures.HitRatio,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	Hits: &view.View{
+		Measure:     CacheMeasures.Hits,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	Misses: &view.View{
+		Measure:     CacheMeasures.Misses,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	Entries: &view.View{
+		Measure:     CacheMeasures.Entries,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	QueriesServed: &view.View{
+		Measure:     CacheMeasures.QueriesServed,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	Adds: &view.View{
+		Measure:     CacheMeasures.Adds,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	Updates: &view.View{
+		Measure:     CacheMeasures.Updates,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	Evictions: &view.View{
+		Measure:     CacheMeasures.Evictions,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	CostAdded: &view.View{
+		Measure:     CacheMeasures.CostAdded,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	CostEvicted: &view.View{
+		Measure:     CacheMeasures.CostEvicted,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	SetsDropped: &view.View{
+		Measure:     CacheMeasures.SetsDropped,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	SetsRejected: &view.View{
+		Measure:     CacheMeasures.SetsRejected,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	QueriesDropped: &view.View{
+		Measure:     CacheMeasures.QueriesDropped,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+}
+
+// DefaultViews exports all default views for this package.
+var DefaultViews = []*view.View{
+	CacheViews.HitRatio,
+	CacheViews.Hits,
+	CacheViews.Misses,
+	CacheViews.Entries,
+	CacheViews.QueriesServed,
+	CacheViews.Adds,
+	CacheViews.Updates,
+	CacheViews.Evictions,
+	CacheViews.CostAdded,
+	CacheViews.CostEvicted,
+	CacheViews.SetsDropped,
+	CacheViews.SetsRejected,
+	CacheViews.QueriesDropped,
+}

blockstore/splitstore/markset.go

@@ -0,0 +1,38 @@
+package splitstore
+
+import (
+	"path/filepath"
+
+	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
+)
+
+// MarkSet is a utility to keep track of seen CID, and later query for them.
+//
+// * If the expected dataset is large, it can be backed by a datastore (e.g. bbolt).
+// * If a probabilistic result is acceptable, it can be backed by a bloom filter (default).
+type MarkSet interface {
+	Mark(cid.Cid) error
+	Has(cid.Cid) (bool, error)
+	Close() error
+}
+
+// markBytes is deliberately a non-nil empty byte slice for serialization.
+var markBytes = []byte{}
+
+type MarkSetEnv interface {
+	Create(name string, sizeHint int64) (MarkSet, error)
+	Close() error
+}
+
+func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) {
+	switch mtype {
+	case "", "bloom":
+		return NewBloomMarkSetEnv()
+	case "bolt":
+		return NewBoltMarkSetEnv(filepath.Join(path, "markset.bolt"))
+	default:
+		return nil, xerrors.Errorf("unknown mark set type %s", mtype)
+	}
+}

blockstore/splitstore/markset_bloom.go

@@ -0,0 +1,77 @@
+package splitstore
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+
+	"golang.org/x/xerrors"
+
+	bbloom "github.com/ipfs/bbloom"
+	cid "github.com/ipfs/go-cid"
+)
+
+const (
+	BloomFilterMinSize     = 10_000_000
+	BloomFilterProbability = 0.01
+)
+
+type BloomMarkSetEnv struct{}
+
+var _ MarkSetEnv = (*BloomMarkSetEnv)(nil)
+
+type BloomMarkSet struct {
+	salt []byte
+	bf   *bbloom.Bloom
+}
+
+var _ MarkSet = (*BloomMarkSet)(nil)
+
+func NewBloomMarkSetEnv() (*BloomMarkSetEnv, error) {
+	return &BloomMarkSetEnv{}, nil
+}
+
+func (e *BloomMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
+	size := int64(BloomFilterMinSize)
+	for size < sizeHint {
+		size += BloomFilterMinSize
+	}
+
+	salt := make([]byte, 4)
+	_, err := rand.Read(salt)
+	if err != nil {
+		return nil, xerrors.Errorf("error reading salt: %w", err)
+	}
+
+	bf, err := bbloom.New(float64(size), BloomFilterProbability)
+	if err != nil {
+		return nil, xerrors.Errorf("error creating bloom filter: %w", err)
+	}
+
+	return &BloomMarkSet{salt: salt, bf: bf}, nil
+}
+
+func (e *BloomMarkSetEnv) Close() error {
+	return nil
+}
+
+func (s *BloomMarkSet) saltedKey(cid cid.Cid) []byte {
+	hash := cid.Hash()
+	key := make([]byte, len(s.salt)+len(hash))
+	n := copy(key, s.salt)
+	copy(key[n:], hash)
+	rehash := sha256.Sum256(key)
+	return rehash[:]
+}
+
+func (s *BloomMarkSet) Mark(cid cid.Cid) error {
+	s.bf.Add(s.saltedKey(cid))
+	return nil
+}
+
+func (s *BloomMarkSet) Has(cid cid.Cid) (bool, error) {
+	return s.bf.Has(s.saltedKey(cid)), nil
+}
+
+func (s *BloomMarkSet) Close() error {
+	return nil
+}

blockstore/splitstore/markset_bolt.go

@@ -0,0 +1,81 @@
+package splitstore
+
+import (
+	"time"
+
+	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
+	bolt "go.etcd.io/bbolt"
+)
+
+type BoltMarkSetEnv struct {
+	db *bolt.DB
+}
+
+var _ MarkSetEnv = (*BoltMarkSetEnv)(nil)
+
+type BoltMarkSet struct {
+	db       *bolt.DB
+	bucketId []byte
+}
+
+var _ MarkSet = (*BoltMarkSet)(nil)
+
+func NewBoltMarkSetEnv(path string) (*BoltMarkSetEnv, error) {
+	db, err := bolt.Open(path, 0644,
+		&bolt.Options{
+			Timeout: 1 * time.Second,
+			NoSync:  true,
+		})
+	if err != nil {
+		return nil, err
+	}
+
+	return &BoltMarkSetEnv{db: db}, nil
+}
+
+func (e *BoltMarkSetEnv) Create(name string, hint int64) (MarkSet, error) {
+	bucketId := []byte(name)
+	err := e.db.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists(bucketId)
+		if err != nil {
+			return xerrors.Errorf("error creating bolt db bucket %s: %w", name, err)
+		}
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &BoltMarkSet{db: e.db, bucketId: bucketId}, nil
+}
+
+func (e *BoltMarkSetEnv) Close() error {
+	return e.db.Close()
+}
+
+func (s *BoltMarkSet) Mark(cid cid.Cid) error {
+	return s.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket(s.bucketId)
+		return b.Put(cid.Hash(), markBytes)
+	})
+}
+
+func (s *BoltMarkSet) Has(cid cid.Cid) (result bool, err error) {
+	err = s.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(s.bucketId)
+		v := b.Get(cid.Hash())
+		result = v != nil
+		return nil
+	})
+
+	return result, err
+}
+
+func (s *BoltMarkSet) Close() error {
+	return s.db.Update(func(tx *bolt.Tx) error {
+		return tx.DeleteBucket(s.bucketId)
+	})
+}

blockstore/splitstore/markset_test.go

@@ -0,0 +1,138 @@
+package splitstore
+
+import (
+	"io/ioutil"
+	"testing"
+
+	cid "github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+)
+
+func TestBoltMarkSet(t *testing.T) {
+	testMarkSet(t, "bolt")
+}
+
+func TestBloomMarkSet(t *testing.T) {
+	testMarkSet(t, "bloom")
+}
+
+func testMarkSet(t *testing.T, lsType string) {
+	t.Helper()
+
+	path, err := ioutil.TempDir("", "sweep-test.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	env, err := OpenMarkSetEnv(path, lsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer env.Close() //nolint:errcheck
+
+	hotSet, err := env.Create("hot", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	coldSet, err := env.Create("cold", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	mustHave := func(s MarkSet, cid cid.Cid) {
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !has {
+			t.Fatal("mark not found")
+		}
+	}
+
+	mustNotHave := func(s MarkSet, cid cid.Cid) {
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("unexpected mark")
+		}
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	hotSet.Mark(k1)  //nolint
+	hotSet.Mark(k2)  //nolint
+	coldSet.Mark(k3) //nolint
+
+	mustHave(hotSet, k1)
+	mustHave(hotSet, k2)
+	mustNotHave(hotSet, k3)
+	mustNotHave(hotSet, k4)
+
+	mustNotHave(coldSet, k1)
+	mustNotHave(coldSet, k2)
+	mustHave(coldSet, k3)
+	mustNotHave(coldSet, k4)
+
+	// close them and reopen to redo the dance
+
+	err = hotSet.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = coldSet.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	hotSet, err = env.Create("hot", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	coldSet, err = env.Create("cold", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	hotSet.Mark(k3)  //nolint
+	hotSet.Mark(k4)  //nolint
+	coldSet.Mark(k1) //nolint
+
+	mustNotHave(hotSet, k1)
+	mustNotHave(hotSet, k2)
+	mustHave(hotSet, k3)
+	mustHave(hotSet, k4)
+
+	mustHave(coldSet, k1)
+	mustNotHave(coldSet, k2)
+	mustNotHave(coldSet, k3)
+	mustNotHave(coldSet, k4)
+
+	err = hotSet.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = coldSet.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

blockstore/splitstore/splitstore_test.go

@@ -0,0 +1,255 @@
+package splitstore
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/types/mock"
+
+	cid "github.com/ipfs/go-cid"
+	datastore "github.com/ipfs/go-datastore"
+	dssync "github.com/ipfs/go-datastore/sync"
+	logging "github.com/ipfs/go-log/v2"
+)
+
+func init() {
+	CompactionThreshold = 5
+	CompactionCold = 1
+	CompactionBoundary = 2
+	logging.SetLogLevel("splitstore", "DEBUG")
+}
+
+func testSplitStore(t *testing.T, cfg *Config) {
+	chain := &mockChain{}
+	// genesis
+	genBlock := mock.MkBlock(nil, 0, 0)
+	genTs := mock.TipSet(genBlock)
+	chain.push(genTs)
+
+	// the myriads of stores
+	ds := dssync.MutexWrap(datastore.NewMapDatastore())
+	hot := blockstore.NewMemorySync()
+	cold := blockstore.NewMemorySync()
+
+	// put the genesis block to cold store
+	blk, err := genBlock.ToStorageBlock()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = cold.Put(blk)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// open the splitstore
+	ss, err := Open("", ds, hot, cold, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close() //nolint
+
+	err = ss.Start(chain)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// make some tipsets, but not enough to cause compaction
+	mkBlock := func(curTs *types.TipSet, i int) *types.TipSet {
+		blk := mock.MkBlock(curTs, uint64(i), uint64(i))
+		sblk, err := blk.ToStorageBlock()
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = ss.Put(sblk)
+		if err != nil {
+			t.Fatal(err)
+		}
+		ts := mock.TipSet(blk)
+		chain.push(ts)
+
+		return ts
+	}
+
+	mkGarbageBlock := func(curTs *types.TipSet, i int) {
+		blk := mock.MkBlock(curTs, uint64(i), uint64(i))
+		sblk, err := blk.ToStorageBlock()
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = ss.Put(sblk)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	waitForCompaction := func() {
+		for atomic.LoadInt32(&ss.compacting) == 1 {
+			time.Sleep(100 * time.Millisecond)
+		}
+	}
+
+	curTs := genTs
+	for i := 1; i < 5; i++ {
+		curTs = mkBlock(curTs, i)
+		waitForCompaction()
+	}
+
+	mkGarbageBlock(genTs, 1)
+
+	// count objects in the cold and hot stores
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	countBlocks := func(bs blockstore.Blockstore) int {
+		count := 0
+		ch, err := bs.AllKeysChan(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for range ch {
+			count++
+		}
+		return count
+	}
+
+	coldCnt := countBlocks(cold)
+	hotCnt := countBlocks(hot)
+
+	if coldCnt != 1 {
+		t.Errorf("expected %d blocks, but got %d", 1, coldCnt)
+	}
+
+	if hotCnt != 5 {
+		t.Errorf("expected %d blocks, but got %d", 5, hotCnt)
+	}
+
+	// trigger a compaction
+	for i := 5; i < 10; i++ {
+		curTs = mkBlock(curTs, i)
+		waitForCompaction()
+	}
+
+	coldCnt = countBlocks(cold)
+	hotCnt = countBlocks(hot)
+
+	if !cfg.EnableFullCompaction {
+		if coldCnt != 5 {
+			t.Errorf("expected %d cold blocks, but got %d", 5, coldCnt)
+		}
+
+		if hotCnt != 5 {
+			t.Errorf("expected %d hot blocks, but got %d", 5, hotCnt)
+		}
+	}
+
+	if cfg.EnableFullCompaction && !cfg.EnableGC {
+		if coldCnt != 3 {
+			t.Errorf("expected %d cold blocks, but got %d", 3, coldCnt)
+		}
+
+		if hotCnt != 7 {
+			t.Errorf("expected %d hot blocks, but got %d", 7, hotCnt)
+		}
+	}
+
+	if cfg.EnableFullCompaction && cfg.EnableGC {
+		if coldCnt != 2 {
+			t.Errorf("expected %d cold blocks, but got %d", 2, coldCnt)
+		}
+
+		if hotCnt != 7 {
+			t.Errorf("expected %d hot blocks, but got %d", 7, hotCnt)
+		}
+	}
+}
+
+func TestSplitStoreSimpleCompaction(t *testing.T) {
+	testSplitStore(t, &Config{TrackingStoreType: "mem"})
+}
+
+func TestSplitStoreFullCompactionWithoutGC(t *testing.T) {
+	testSplitStore(t, &Config{
+		TrackingStoreType:    "mem",
+		EnableFullCompaction: true,
+	})
+}
+
+func TestSplitStoreFullCompactionWithGC(t *testing.T) {
+	testSplitStore(t, &Config{
+		TrackingStoreType:    "mem",
+		EnableFullCompaction: true,
+		EnableGC:             true,
+	})
+}
+
+type mockChain struct {
+	sync.Mutex
+	tipsets  []*types.TipSet
+	listener func(revert []*types.TipSet, apply []*types.TipSet) error
+}
+
+func (c *mockChain) push(ts *types.TipSet) {
+	c.Lock()
+	c.tipsets = append(c.tipsets, ts)
+	c.Unlock()
+
+	if c.listener != nil {
+		err := c.listener(nil, []*types.TipSet{ts})
+		if err != nil {
+			log.Errorf("mockchain: error dispatching listener: %s", err)
+		}
+	}
+}
+
+func (c *mockChain) GetTipsetByHeight(_ context.Context, epoch abi.ChainEpoch, _ *types.TipSet, _ bool) (*types.TipSet, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	iEpoch := int(epoch)
+	if iEpoch > len(c.tipsets) {
+		return nil, fmt.Errorf("bad epoch %d", epoch)
+	}
+
+	return c.tipsets[iEpoch-1], nil
+}
+
+func (c *mockChain) GetHeaviestTipSet() *types.TipSet {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.tipsets[len(c.tipsets)-1]
+}
+
+func (c *mockChain) SubscribeHeadChanges(change func(revert []*types.TipSet, apply []*types.TipSet) error) {
+	c.listener = change
+}
+
+func (c *mockChain) WalkSnapshot(_ context.Context, ts *types.TipSet, epochs abi.ChainEpoch, _ bool, _ bool, f func(cid.Cid) error) error {
+	c.Lock()
+	defer c.Unlock()
+
+	start := int(ts.Height()) - 1
+	end := start - int(epochs)
+	if end < 0 {
+		end = -1
+	}
+	for i := start; i > end; i-- {
+		ts := c.tipsets[i]
+		for _, cid := range ts.Cids() {
+			err := f(cid)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

blockstore/splitstore/tracking.go

@@ -0,0 +1,109 @@
+package splitstore
+
+import (
+	"path/filepath"
+	"sync"
+
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	cid "github.com/ipfs/go-cid"
+)
+
+// TrackingStore is a persistent store that tracks blocks that are added
+// to the hotstore, tracking the epoch at which they are written.
+type TrackingStore interface {
+	Put(cid.Cid, abi.ChainEpoch) error
+	PutBatch([]cid.Cid, abi.ChainEpoch) error
+	Get(cid.Cid) (abi.ChainEpoch, error)
+	Delete(cid.Cid) error
+	DeleteBatch([]cid.Cid) error
+	ForEach(func(cid.Cid, abi.ChainEpoch) error) error
+	Sync() error
+	Close() error
+}
+
+// OpenTrackingStore opens a tracking store of the specified type in the
+// specified path.
+func OpenTrackingStore(path string, ttype string) (TrackingStore, error) {
+	switch ttype {
+	case "", "bolt":
+		return OpenBoltTrackingStore(filepath.Join(path, "tracker.bolt"))
+	case "mem":
+		return NewMemTrackingStore(), nil
+	default:
+		return nil, xerrors.Errorf("unknown tracking store type %s", ttype)
+	}
+}
+
+// NewMemTrackingStore creates an in-memory tracking store.
+// This is only useful for test or situations where you don't want to open the
+// real tracking store (eg concurrent read only access on a node's datastore)
+func NewMemTrackingStore() *MemTrackingStore {
+	return &MemTrackingStore{tab: make(map[cid.Cid]abi.ChainEpoch)}
+}
+
+// MemTrackingStore is a simple in-memory tracking store
+type MemTrackingStore struct {
+	sync.Mutex
+	tab map[cid.Cid]abi.ChainEpoch
+}
+
+var _ TrackingStore = (*MemTrackingStore)(nil)
+
+func (s *MemTrackingStore) Put(cid cid.Cid, epoch abi.ChainEpoch) error {
+	s.Lock()
+	defer s.Unlock()
+	s.tab[cid] = epoch
+	return nil
+}
+
+func (s *MemTrackingStore) PutBatch(cids []cid.Cid, epoch abi.ChainEpoch) error {
+	s.Lock()
+	defer s.Unlock()
+	for _, cid := range cids {
+		s.tab[cid] = epoch
+	}
+	return nil
+}
+
+func (s *MemTrackingStore) Get(cid cid.Cid) (abi.ChainEpoch, error) {
+	s.Lock()
+	defer s.Unlock()
+	epoch, ok := s.tab[cid]
+	if ok {
+		return epoch, nil
+	}
+	return 0, xerrors.Errorf("missing tracking epoch for %s", cid)
+}
+
+func (s *MemTrackingStore) Delete(cid cid.Cid) error {
+	s.Lock()
+	defer s.Unlock()
+	delete(s.tab, cid)
+	return nil
+}
+
+func (s *MemTrackingStore) DeleteBatch(cids []cid.Cid) error {
+	s.Lock()
+	defer s.Unlock()
+	for _, cid := range cids {
+		delete(s.tab, cid)
+	}
+	return nil
+}
+
+func (s *MemTrackingStore) ForEach(f func(cid.Cid, abi.ChainEpoch) error) error {
+	s.Lock()
+	defer s.Unlock()
+	for cid, epoch := range s.tab {
+		err := f(cid, epoch)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *MemTrackingStore) Sync() error  { return nil }
+func (s *MemTrackingStore) Close() error { return nil }

blockstore/splitstore/tracking_bolt.go

@@ -0,0 +1,120 @@
+package splitstore
+
+import (
+	"time"
+
+	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
+	bolt "go.etcd.io/bbolt"
+
+	"github.com/filecoin-project/go-state-types/abi"
+)
+
+type BoltTrackingStore struct {
+	db       *bolt.DB
+	bucketId []byte
+}
+
+var _ TrackingStore = (*BoltTrackingStore)(nil)
+
+func OpenBoltTrackingStore(path string) (*BoltTrackingStore, error) {
+	opts := &bolt.Options{
+		Timeout: 1 * time.Second,
+		NoSync:  true,
+	}
+	db, err := bolt.Open(path, 0644, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	bucketId := []byte("tracker")
+	err = db.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists(bucketId)
+		if err != nil {
+			return xerrors.Errorf("error creating bolt db bucket %s: %w", string(bucketId), err)
+		}
+		return nil
+	})
+
+	if err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+
+	return &BoltTrackingStore{db: db, bucketId: bucketId}, nil
+}
+
+func (s *BoltTrackingStore) Put(cid cid.Cid, epoch abi.ChainEpoch) error {
+	val := epochToBytes(epoch)
+	return s.db.Batch(func(tx *bolt.Tx) error {
+		b := tx.Bucket(s.bucketId)
+		return b.Put(cid.Hash(), val)
+	})
+}
+
+func (s *BoltTrackingStore) PutBatch(cids []cid.Cid, epoch abi.ChainEpoch) error {
+	val := epochToBytes(epoch)
+	return s.db.Batch(func(tx *bolt.Tx) error {
+		b := tx.Bucket(s.bucketId)
+		for _, cid := range cids {
+			err := b.Put(cid.Hash(), val)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
+func (s *BoltTrackingStore) Get(cid cid.Cid) (epoch abi.ChainEpoch, err error) {
+	err = s.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(s.bucketId)
+		val := b.Get(cid.Hash())
+		if val == nil {
+			return xerrors.Errorf("missing tracking epoch for %s", cid)
+		}
+		epoch = bytesToEpoch(val)
+		return nil
+	})
+	return epoch, err
+}
+
+func (s *BoltTrackingStore) Delete(cid cid.Cid) error {
+	return s.db.Batch(func(tx *bolt.Tx) error {
+		b := tx.Bucket(s.bucketId)
+		return b.Delete(cid.Hash())
+	})
+}
+
+func (s *BoltTrackingStore) DeleteBatch(cids []cid.Cid) error {
+	return s.db.Batch(func(tx *bolt.Tx) error {
+		b := tx.Bucket(s.bucketId)
+		for _, cid := range cids {
+			err := b.Delete(cid.Hash())
+			if err != nil {
+				return xerrors.Errorf("error deleting %s", cid)
+			}
+		}
+		return nil
+	})
+}
+
+func (s *BoltTrackingStore) ForEach(f func(cid.Cid, abi.ChainEpoch) error) error {
+	return s.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(s.bucketId)
+		return b.ForEach(func(k, v []byte) error {
+			cid := cid.NewCidV1(cid.Raw, k)
+			epoch := bytesToEpoch(v)
+			return f(cid, epoch)
+		})
+	})
+}
+
+func (s *BoltTrackingStore) Sync() error {
+	return s.db.Sync()
+}
+
+func (s *BoltTrackingStore) Close() error {
+	return s.db.Close()
+}

blockstore/splitstore/tracking_test.go

@@ -0,0 +1,130 @@
+package splitstore
+
+import (
+	"io/ioutil"
+	"testing"
+
+	cid "github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+
+	"github.com/filecoin-project/go-state-types/abi"
+)
+
+func TestBoltTrackingStore(t *testing.T) {
+	testTrackingStore(t, "bolt")
+}
+
+func testTrackingStore(t *testing.T, tsType string) {
+	t.Helper()
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	mustHave := func(s TrackingStore, cid cid.Cid, epoch abi.ChainEpoch) {
+		val, err := s.Get(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if val != epoch {
+			t.Fatal("epoch mismatch")
+		}
+	}
+
+	mustNotHave := func(s TrackingStore, cid cid.Cid) {
+		_, err := s.Get(cid)
+		if err == nil {
+			t.Fatal("expected error")
+		}
+	}
+
+	path, err := ioutil.TempDir("", "snoop-test.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s, err := OpenTrackingStore(path, tsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	s.Put(k1, 1) //nolint
+	s.Put(k2, 2) //nolint
+	s.Put(k3, 3) //nolint
+	s.Put(k4, 4) //nolint
+
+	mustHave(s, k1, 1)
+	mustHave(s, k2, 2)
+	mustHave(s, k3, 3)
+	mustHave(s, k4, 4)
+
+	s.Delete(k1) // nolint
+	s.Delete(k2) // nolint
+
+	mustNotHave(s, k1)
+	mustNotHave(s, k2)
+	mustHave(s, k3, 3)
+	mustHave(s, k4, 4)
+
+	s.PutBatch([]cid.Cid{k1}, 1) //nolint
+	s.PutBatch([]cid.Cid{k2}, 2) //nolint
+
+	mustHave(s, k1, 1)
+	mustHave(s, k2, 2)
+	mustHave(s, k3, 3)
+	mustHave(s, k4, 4)
+
+	allKeys := map[string]struct{}{
+		k1.String(): {},
+		k2.String(): {},
+		k3.String(): {},
+		k4.String(): {},
+	}
+
+	err = s.ForEach(func(k cid.Cid, _ abi.ChainEpoch) error {
+		_, ok := allKeys[k.String()]
+		if !ok {
+			t.Fatal("unexpected key")
+		}
+
+		delete(allKeys, k.String())
+		return nil
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(allKeys) != 0 {
+		t.Fatal("not all keys were returned")
+	}
+
+	// no close and reopen and ensure the keys still exist
+	err = s.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s, err = OpenTrackingStore(path, tsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(s, k1, 1)
+	mustHave(s, k2, 2)
+	mustHave(s, k3, 3)
+	mustHave(s, k4, 4)
+
+	s.Close() //nolint:errcheck
+}

blockstore/sync.go

@@ -0,0 +1,81 @@
+package blockstore
+
+import (
+	"context"
+	"sync"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+)
+
+// NewMemorySync returns a thread-safe in-memory blockstore.
+func NewMemorySync() *SyncBlockstore {
+	return &SyncBlockstore{bs: make(MemBlockstore)}
+}
+
+// SyncBlockstore is a terminal blockstore that is a synchronized version
+// of MemBlockstore.
+type SyncBlockstore struct {
+	mu sync.RWMutex
+	bs MemBlockstore // specifically use a memStore to save indirection overhead.
+}
+
+func (m *SyncBlockstore) DeleteBlock(k cid.Cid) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.bs.DeleteBlock(k)
+}
+
+func (m *SyncBlockstore) DeleteMany(ks []cid.Cid) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.bs.DeleteMany(ks)
+}
+
+func (m *SyncBlockstore) Has(k cid.Cid) (bool, error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.bs.Has(k)
+}
+
+func (m *SyncBlockstore) View(k cid.Cid, callback func([]byte) error) error {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	return m.bs.View(k, callback)
+}
+
+func (m *SyncBlockstore) Get(k cid.Cid) (blocks.Block, error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.bs.Get(k)
+}
+
+func (m *SyncBlockstore) GetSize(k cid.Cid) (int, error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.bs.GetSize(k)
+}
+
+func (m *SyncBlockstore) Put(b blocks.Block) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.bs.Put(b)
+}
+
+func (m *SyncBlockstore) PutMany(bs []blocks.Block) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.bs.PutMany(bs)
+}
+
+func (m *SyncBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	// this blockstore implementation doesn't do any async work.
+	return m.bs.AllKeysChan(ctx)
+}
+
+func (m *SyncBlockstore) HashOnRead(enabled bool) {
+	// noop
+}

blockstore/timed.go

@@ -1,4 +1,4 @@
-package timedbs
+package blockstore
 
 import (
 	"context"
@@ -10,37 +10,37 @@ import (
 	"github.com/ipfs/go-cid"
 	"github.com/raulk/clock"
 	"go.uber.org/multierr"
-
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 )
 
-// TimedCacheBS is a blockstore that keeps blocks for at least the specified
+// TimedCacheBlockstore is a blockstore that keeps blocks for at least the
-// caching interval before discarding them. Garbage collection must be started
+// specified caching interval before discarding them. Garbage collection must
-// and stopped by calling Start/Stop.
+// be started and stopped by calling Start/Stop.
 //
 // Under the covers, it's implemented with an active and an inactive blockstore
 // that are rotated every cache time interval. This means all blocks will be
 // stored at most 2x the cache interval.
-type TimedCacheBS struct {
+//
+// Create a new instance by calling the NewTimedCacheBlockstore constructor.
+type TimedCacheBlockstore struct {
 	mu               sync.RWMutex
-	active, inactive blockstore.MemStore
+	active, inactive MemBlockstore
 	clock            clock.Clock
 	interval         time.Duration
 	closeCh          chan struct{}
 	doneRotatingCh   chan struct{}
 }
 
-func NewTimedCacheBS(cacheTime time.Duration) *TimedCacheBS {
+func NewTimedCacheBlockstore(interval time.Duration) *TimedCacheBlockstore {
-	return &TimedCacheBS{
+	b := &TimedCacheBlockstore{
-		active:   blockstore.NewTemporary(),
+		active:   NewMemory(),
-		inactive: blockstore.NewTemporary(),
+		inactive: NewMemory(),
-		interval: cacheTime,
+		interval: interval,
-		clock:    build.Clock,
+		clock:    clock.New(),
 	}
+	return b
 }
 
-func (t *TimedCacheBS) Start(ctx context.Context) error {
+func (t *TimedCacheBlockstore) Start(_ context.Context) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	if t.closeCh != nil {
@@ -65,11 +65,11 @@ func (t *TimedCacheBS) Start(ctx context.Context) error {
 	return nil
 }
 
-func (t *TimedCacheBS) Stop(ctx context.Context) error {
+func (t *TimedCacheBlockstore) Stop(_ context.Context) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	if t.closeCh == nil {
-		return fmt.Errorf("not started started")
+		return fmt.Errorf("not started")
 	}
 	select {
 	case <-t.closeCh:
@@ -80,15 +80,15 @@ func (t *TimedCacheBS) Stop(ctx context.Context) error {
 	return nil
 }
 
-func (t *TimedCacheBS) rotate() {
+func (t *TimedCacheBlockstore) rotate() {
-	newBs := blockstore.NewTemporary()
+	newBs := NewMemory()
 
 	t.mu.Lock()
 	t.inactive, t.active = t.active, newBs
 	t.mu.Unlock()
 }
 
-func (t *TimedCacheBS) Put(b blocks.Block) error {
+func (t *TimedCacheBlockstore) Put(b blocks.Block) error {
 	// Don't check the inactive set here. We want to keep this block for at
 	// least one interval.
 	t.mu.Lock()
@@ -96,33 +96,43 @@ func (t *TimedCacheBS) Put(b blocks.Block) error {
 	return t.active.Put(b)
 }
 
-func (t *TimedCacheBS) PutMany(bs []blocks.Block) error {
+func (t *TimedCacheBlockstore) PutMany(bs []blocks.Block) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	return t.active.PutMany(bs)
 }
 
-func (t *TimedCacheBS) Get(k cid.Cid) (blocks.Block, error) {
+func (t *TimedCacheBlockstore) View(k cid.Cid, callback func([]byte) error) error {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+	err := t.active.View(k, callback)
+	if err == ErrNotFound {
+		err = t.inactive.View(k, callback)
+	}
+	return err
+}
+
+func (t *TimedCacheBlockstore) Get(k cid.Cid) (blocks.Block, error) {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 	b, err := t.active.Get(k)
-	if err == blockstore.ErrNotFound {
+	if err == ErrNotFound {
 		b, err = t.inactive.Get(k)
 	}
 	return b, err
 }
 
-func (t *TimedCacheBS) GetSize(k cid.Cid) (int, error) {
+func (t *TimedCacheBlockstore) GetSize(k cid.Cid) (int, error) {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 	size, err := t.active.GetSize(k)
-	if err == blockstore.ErrNotFound {
+	if err == ErrNotFound {
 		size, err = t.inactive.GetSize(k)
 	}
 	return size, err
 }
 
-func (t *TimedCacheBS) Has(k cid.Cid) (bool, error) {
+func (t *TimedCacheBlockstore) Has(k cid.Cid) (bool, error) {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 	if has, err := t.active.Has(k); err != nil {
@@ -133,17 +143,23 @@ func (t *TimedCacheBS) Has(k cid.Cid) (bool, error) {
 	return t.inactive.Has(k)
 }
 
-func (t *TimedCacheBS) HashOnRead(_ bool) {
+func (t *TimedCacheBlockstore) HashOnRead(_ bool) {
 	// no-op
 }
 
-func (t *TimedCacheBS) DeleteBlock(k cid.Cid) error {
+func (t *TimedCacheBlockstore) DeleteBlock(k cid.Cid) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	return multierr.Combine(t.active.DeleteBlock(k), t.inactive.DeleteBlock(k))
 }
 
-func (t *TimedCacheBS) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+func (t *TimedCacheBlockstore) DeleteMany(ks []cid.Cid) error {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	return multierr.Combine(t.active.DeleteMany(ks), t.inactive.DeleteMany(ks))
+}
+
+func (t *TimedCacheBlockstore) AllKeysChan(_ context.Context) (<-chan cid.Cid, error) {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 

blockstore/timed_test.go

@@ -1,4 +1,4 @@
-package timedbs
+package blockstore
 
 import (
 	"context"
@@ -12,8 +12,8 @@ import (
 	"github.com/ipfs/go-cid"
 )
 
-func TestTimedBSSimple(t *testing.T) {
+func TestTimedCacheBlockstoreSimple(t *testing.T) {
-	tc := NewTimedCacheBS(10 * time.Millisecond)
+	tc := NewTimedCacheBlockstore(10 * time.Millisecond)
 	mClock := clock.NewMock()
 	mClock.Set(time.Now())
 	tc.clock = mClock

blockstore/union.go

@@ -0,0 +1,119 @@
+package blockstore
+
+import (
+	"context"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+)
+
+type unionBlockstore []Blockstore
+
+// Union returns an unioned blockstore.
+//
+// * Reads return from the first blockstore that has the value, querying in the
+//   supplied order.
+// * Writes (puts and deltes) are broadcast to all stores.
+//
+func Union(stores ...Blockstore) Blockstore {
+	return unionBlockstore(stores)
+}
+
+func (m unionBlockstore) Has(cid cid.Cid) (has bool, err error) {
+	for _, bs := range m {
+		if has, err = bs.Has(cid); has || err != nil {
+			break
+		}
+	}
+	return has, err
+}
+
+func (m unionBlockstore) Get(cid cid.Cid) (blk blocks.Block, err error) {
+	for _, bs := range m {
+		if blk, err = bs.Get(cid); err == nil || err != ErrNotFound {
+			break
+		}
+	}
+	return blk, err
+}
+
+func (m unionBlockstore) View(cid cid.Cid, callback func([]byte) error) (err error) {
+	for _, bs := range m {
+		if err = bs.View(cid, callback); err == nil || err != ErrNotFound {
+			break
+		}
+	}
+	return err
+}
+
+func (m unionBlockstore) GetSize(cid cid.Cid) (size int, err error) {
+	for _, bs := range m {
+		if size, err = bs.GetSize(cid); err == nil || err != ErrNotFound {
+			break
+		}
+	}
+	return size, err
+}
+
+func (m unionBlockstore) Put(block blocks.Block) (err error) {
+	for _, bs := range m {
+		if err = bs.Put(block); err != nil {
+			break
+		}
+	}
+	return err
+}
+
+func (m unionBlockstore) PutMany(blks []blocks.Block) (err error) {
+	for _, bs := range m {
+		if err = bs.PutMany(blks); err != nil {
+			break
+		}
+	}
+	return err
+}
+
+func (m unionBlockstore) DeleteBlock(cid cid.Cid) (err error) {
+	for _, bs := range m {
+		if err = bs.DeleteBlock(cid); err != nil {
+			break
+		}
+	}
+	return err
+}
+
+func (m unionBlockstore) DeleteMany(cids []cid.Cid) (err error) {
+	for _, bs := range m {
+		if err = bs.DeleteMany(cids); err != nil {
+			break
+		}
+	}
+	return err
+}
+
+func (m unionBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+	// this does not deduplicate; this interface needs to be revisited.
+	outCh := make(chan cid.Cid)
+
+	go func() {
+		defer close(outCh)
+
+		for _, bs := range m {
+			ch, err := bs.AllKeysChan(ctx)
+			if err != nil {
+				return
+			}
+			for cid := range ch {
+				outCh <- cid
+			}
+		}
+	}()
+
+	return outCh, nil
+}
+
+func (m unionBlockstore) HashOnRead(enabled bool) {
+	for _, bs := range m {
+		bs.HashOnRead(enabled)
+	}
+}

blockstore/union_test.go

@@ -0,0 +1,102 @@
+package blockstore
+
+import (
+	"context"
+	"testing"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	b0 = blocks.NewBlock([]byte("abc"))
+	b1 = blocks.NewBlock([]byte("foo"))
+	b2 = blocks.NewBlock([]byte("bar"))
+)
+
+func TestUnionBlockstore_Get(t *testing.T) {
+	m1 := NewMemory()
+	m2 := NewMemory()
+
+	_ = m1.Put(b1)
+	_ = m2.Put(b2)
+
+	u := Union(m1, m2)
+
+	v1, err := u.Get(b1.Cid())
+	require.NoError(t, err)
+	require.Equal(t, b1.RawData(), v1.RawData())
+
+	v2, err := u.Get(b2.Cid())
+	require.NoError(t, err)
+	require.Equal(t, b2.RawData(), v2.RawData())
+}
+
+func TestUnionBlockstore_Put_PutMany_Delete_AllKeysChan(t *testing.T) {
+	m1 := NewMemory()
+	m2 := NewMemory()
+
+	u := Union(m1, m2)
+
+	err := u.Put(b0)
+	require.NoError(t, err)
+
+	var has bool
+
+	// write was broadcasted to all stores.
+	has, _ = m1.Has(b0.Cid())
+	require.True(t, has)
+
+	has, _ = m2.Has(b0.Cid())
+	require.True(t, has)
+
+	has, _ = u.Has(b0.Cid())
+	require.True(t, has)
+
+	// put many.
+	err = u.PutMany([]blocks.Block{b1, b2})
+	require.NoError(t, err)
+
+	// write was broadcasted to all stores.
+	has, _ = m1.Has(b1.Cid())
+	require.True(t, has)
+
+	has, _ = m1.Has(b2.Cid())
+	require.True(t, has)
+
+	has, _ = m2.Has(b1.Cid())
+	require.True(t, has)
+
+	has, _ = m2.Has(b2.Cid())
+	require.True(t, has)
+
+	// also in the union store.
+	has, _ = u.Has(b1.Cid())
+	require.True(t, has)
+
+	has, _ = u.Has(b2.Cid())
+	require.True(t, has)
+
+	// deleted from all stores.
+	err = u.DeleteBlock(b1.Cid())
+	require.NoError(t, err)
+
+	has, _ = u.Has(b1.Cid())
+	require.False(t, has)
+
+	has, _ = m1.Has(b1.Cid())
+	require.False(t, has)
+
+	has, _ = m2.Has(b1.Cid())
+	require.False(t, has)
+
+	// check that AllKeysChan returns b0 and b2, twice (once per backing store)
+	ch, err := u.AllKeysChan(context.Background())
+	require.NoError(t, err)
+
+	var i int
+	for range ch {
+		i++
+	}
+	require.Equal(t, 4, i)
+}

build/tools.go

@@ -4,5 +4,6 @@ package build
 
 import (
 	_ "github.com/GeertJohan/go.rice/rice"
+	_ "github.com/golang/mock/mockgen"
 	_ "github.com/whyrusleeping/bencher"
 )

build/version.go

@@ -1,11 +1,5 @@
 package build
 
-import (
-	"fmt"
-
-	"golang.org/x/xerrors"
-)
-
 var CurrentCommit string
 var BuildType int
 
@@ -40,67 +34,3 @@ const BuildVersion = "1.5.0"
 func UserVersion() string {
 	return BuildVersion + buildType() + CurrentCommit
 }
-
-type Version uint32
-
-func newVer(major, minor, patch uint8) Version {
-	return Version(uint32(major)<<16 | uint32(minor)<<8 | uint32(patch))
-}
-
-// Ints returns (major, minor, patch) versions
-func (ve Version) Ints() (uint32, uint32, uint32) {
-	v := uint32(ve)
-	return (v & majorOnlyMask) >> 16, (v & minorOnlyMask) >> 8, v & patchOnlyMask
-}
-
-func (ve Version) String() string {
-	vmj, vmi, vp := ve.Ints()
-	return fmt.Sprintf("%d.%d.%d", vmj, vmi, vp)
-}
-
-func (ve Version) EqMajorMinor(v2 Version) bool {
-	return ve&minorMask == v2&minorMask
-}
-
-type NodeType int
-
-const (
-	NodeUnknown NodeType = iota
-
-	NodeFull
-	NodeMiner
-	NodeWorker
-)
-
-var RunningNodeType NodeType
-
-func VersionForType(nodeType NodeType) (Version, error) {
-	switch nodeType {
-	case NodeFull:
-		return FullAPIVersion, nil
-	case NodeMiner:
-		return MinerAPIVersion, nil
-	case NodeWorker:
-		return WorkerAPIVersion, nil
-	default:
-		return Version(0), xerrors.Errorf("unknown node type %d", nodeType)
-	}
-}
-
-// semver versions of the rpc api exposed
-var (
-	FullAPIVersion   = newVer(1, 1, 0)
-	MinerAPIVersion  = newVer(1, 0, 1)
-	WorkerAPIVersion = newVer(1, 0, 0)
-)
-
-//nolint:varcheck,deadcode
-const (
-	majorMask = 0xff0000
-	minorMask = 0xffff00
-	patchMask = 0xffffff
-
-	majorOnlyMask = 0xff0000
-	minorOnlyMask = 0x00ff00
-	patchOnlyMask = 0x0000ff
-)

chain/actors/adt/diff_adt_test.go

@@ -16,7 +16,7 @@ import (
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
 	adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt"
 
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
+	bstore "github.com/filecoin-project/lotus/blockstore"
 )
 
 func TestDiffAdtArray(t *testing.T) {
@@ -295,7 +295,7 @@ func (t *TestDiffArray) Remove(key uint64, val *typegen.Deferred) error {
 
 func newContextStore() Store {
 	ctx := context.Background()
-	bs := bstore.NewTemporarySync()
+	bs := bstore.NewMemorySync()
 	store := cbornode.NewCborStore(bs)
 	return WrapStore(ctx, store)
 }

chain/events/state/mock/api.go

@@ -5,8 +5,8 @@ import (
 	"sync"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
 )

chain/events/state/predicates.go

@@ -3,6 +3,7 @@ package state
 import (
 	"context"
 
+	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 
 	"github.com/filecoin-project/go-address"
@@ -10,7 +11,7 @@ import (
 	"github.com/filecoin-project/go-state-types/big"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
-	"github.com/filecoin-project/lotus/api/apibstore"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
@@ -23,7 +24,7 @@ type UserData interface{}
 
 // ChainAPI abstracts out calls made by this class to external APIs
 type ChainAPI interface {
-	apibstore.ChainIO
+	api.ChainIO
 	StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error)
 }
 
@@ -36,7 +37,7 @@ type StatePredicates struct {
 func NewStatePredicates(api ChainAPI) *StatePredicates {
 	return &StatePredicates{
 		api: api,
-		cst: cbor.NewCborStore(apibstore.NewAPIBlockstore(api)),
+		cst: cbor.NewCborStore(blockstore.NewAPIBlockstore(api)),
 	}
 }
 

chain/events/state/predicates_test.go

@@ -23,9 +23,9 @@ import (
 	adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt"
 	tutils "github.com/filecoin-project/specs-actors/v2/support/testing"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 var dummyCid cid.Cid
@@ -36,7 +36,7 @@ func init() {
 
 func TestMarketPredicates(t *testing.T) {
 	ctx := context.Background()
-	bs := bstore.NewTemporarySync()
+	bs := bstore.NewMemorySync()
 	store := adt2.WrapStore(ctx, cbornode.NewCborStore(bs))
 
 	oldDeal1 := &market2.DealState{
@@ -334,7 +334,7 @@ func TestMarketPredicates(t *testing.T) {
 
 func TestMinerSectorChange(t *testing.T) {
 	ctx := context.Background()
-	bs := bstore.NewTemporarySync()
+	bs := bstore.NewMemorySync()
 	store := adt2.WrapStore(ctx, cbornode.NewCborStore(bs))
 
 	nextID := uint64(0)

chain/gen/gen.go

@@ -27,6 +27,7 @@ import (
 	proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
 	"github.com/filecoin-project/lotus/chain/beacon"
@@ -40,7 +41,6 @@ import (
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/lotus/genesis"
 	"github.com/filecoin-project/lotus/journal"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/lib/sigs"
 	"github.com/filecoin-project/lotus/node/repo"
 )
@@ -125,7 +125,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
 		return nil, xerrors.Errorf("failed to get metadata datastore: %w", err)
 	}
 
-	bs, err := lr.Blockstore(context.TODO(), repo.BlockstoreChain)
+	bs, err := lr.Blockstore(context.TODO(), repo.UniversalBlockstore)
 	if err != nil {
 		return nil, err
 	}

chain/gen/genesis/f00_system.go

@@ -8,8 +8,8 @@ import (
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 func SetupSystemActor(bs bstore.Blockstore) (*types.Actor, error) {

chain/gen/genesis/f01_init.go

@@ -16,9 +16,9 @@ import (
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"golang.org/x/xerrors"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/genesis"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesis.Actor, rootVerifier genesis.Actor, remainder genesis.Actor) (int64, *types.Actor, map[address.Address]address.Address, error) {

chain/gen/genesis/f02_reward.go

@@ -9,9 +9,9 @@ import (
 	reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 func SetupRewardActor(bs bstore.Blockstore, qaPower big.Int) (*types.Actor, error) {

chain/gen/genesis/f03_cron.go

@@ -7,8 +7,8 @@ import (
 	"github.com/filecoin-project/specs-actors/actors/builtin/cron"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 func SetupCronActor(bs bstore.Blockstore) (*types.Actor, error) {

chain/gen/genesis/f04_power.go

@@ -9,8 +9,8 @@ import (
 	power0 "github.com/filecoin-project/specs-actors/actors/builtin/power"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 func SetupStoragePowerActor(bs bstore.Blockstore) (*types.Actor, error) {

chain/gen/genesis/f05_market.go

@@ -8,8 +8,8 @@ import (
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 func SetupStorageMarketActor(bs bstore.Blockstore) (*types.Actor, error) {

chain/gen/genesis/f06_vreg.go

@@ -10,8 +10,8 @@ import (
 	verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 var RootVerifierID address.Address

chain/gen/genesis/genesis.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
+
 	"github.com/filecoin-project/lotus/journal"
 
 	"github.com/ipfs/go-cid"
@@ -26,13 +27,13 @@ import (
 	verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
 	adt0 "github.com/filecoin-project/specs-actors/actors/util/adt"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/state"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/genesis"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/lib/sigs"
 )
 
@@ -233,14 +234,37 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
 
 	}
 
-	vregroot, err := address.NewIDAddress(80)
+	switch template.VerifregRootKey.Type {
+	case genesis.TAccount:
+		var ainfo genesis.AccountMeta
+		if err := json.Unmarshal(template.VerifregRootKey.Meta, &ainfo); err != nil {
+			return nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
+		}
+		st, err := cst.Put(ctx, &account0.State{Address: ainfo.Owner})
 		if err != nil {
 			return nil, nil, err
 		}
 
-	if err = createMultisigAccount(ctx, bs, cst, state, vregroot, template.VerifregRootKey, keyIDs); err != nil {
+		_, ok := keyIDs[ainfo.Owner]
+		if ok {
+			return nil, nil, fmt.Errorf("rootkey account has already been declared, cannot be assigned 80: %s", ainfo.Owner)
+		}
+
+		err = state.SetActor(builtin.RootVerifierAddress, &types.Actor{
+			Code:    builtin0.AccountActorCodeID,
+			Balance: template.VerifregRootKey.Balance,
+			Head:    st,
+		})
+		if err != nil {
+			return nil, nil, xerrors.Errorf("setting verifreg rootkey account: %w", err)
+		}
+	case genesis.TMultisig:
+		if err = createMultisigAccount(ctx, bs, cst, state, builtin.RootVerifierAddress, template.VerifregRootKey, keyIDs); err != nil {
 			return nil, nil, xerrors.Errorf("failed to set up verified registry signer: %w", err)
 		}
+	default:
+		return nil, nil, xerrors.Errorf("unknown account type for verifreg rootkey: %w", err)
+	}
 
 	// Setup the first verifier as ID-address 81
 	// TODO: remove this
@@ -300,8 +324,36 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
 
 	template.RemainderAccount.Balance = remainingFil
 
-	if err := createMultisigAccount(ctx, bs, cst, state, builtin.ReserveAddress, template.RemainderAccount, keyIDs); err != nil {
+	switch template.RemainderAccount.Type {
-		return nil, nil, xerrors.Errorf("failed to set up remainder account: %w", err)
+	case genesis.TAccount:
+		var ainfo genesis.AccountMeta
+		if err := json.Unmarshal(template.RemainderAccount.Meta, &ainfo); err != nil {
+			return nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
+		}
+		st, err := cst.Put(ctx, &account0.State{Address: ainfo.Owner})
+		if err != nil {
+			return nil, nil, err
+		}
+
+		_, ok := keyIDs[ainfo.Owner]
+		if ok {
+			return nil, nil, fmt.Errorf("remainder account has already been declared, cannot be assigned 90: %s", ainfo.Owner)
+		}
+
+		err = state.SetActor(builtin.ReserveAddress, &types.Actor{
+			Code:    builtin0.AccountActorCodeID,
+			Balance: template.RemainderAccount.Balance,
+			Head:    st,
+		})
+		if err != nil {
+			return nil, nil, xerrors.Errorf("setting remainder account: %w", err)
+		}
+	case genesis.TMultisig:
+		if err = createMultisigAccount(ctx, bs, cst, state, builtin.ReserveAddress, template.RemainderAccount, keyIDs); err != nil {
+			return nil, nil, xerrors.Errorf("failed to set up remainder: %w", err)
+		}
+	default:
+		return nil, nil, xerrors.Errorf("unknown account type for remainder: %w", err)
 	}
 
 	return state, keyIDs, nil
@@ -406,7 +458,7 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci
 		StateBase:      stateroot,
 		Epoch:          0,
 		Rand:           &fakeRand{},
-		Bstore:         cs.Blockstore(),
+		Bstore:         cs.StateBlockstore(),
 		Syscalls:       mkFakedSigSyscalls(cs.VMSys()),
 		CircSupplyCalc: nil,
 		NtwkVersion:    genesisNetworkVersion,

chain/gen/genesis/miners.go

@@ -70,7 +70,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 		StateBase:      sroot,
 		Epoch:          0,
 		Rand:           &fakeRand{},
-		Bstore:         cs.Blockstore(),
+		Bstore:         cs.StateBlockstore(),
 		Syscalls:       mkFakedSigSyscalls(cs.VMSys()),
 		CircSupplyCalc: csc,
 		NtwkVersion:    genesisNetworkVersion,

chain/gen/mining.go

@@ -79,7 +79,7 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w api.WalletA
 		}
 	}
 
-	store := sm.ChainStore().Store(ctx)
+	store := sm.ChainStore().ActorStore(ctx)
 	blsmsgroot, err := toArray(store, blsMsgCids)
 	if err != nil {
 		return nil, xerrors.Errorf("building bls amt: %w", err)

chain/stmgr/call.go

@@ -59,7 +59,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
 		StateBase:      bstate,
 		Epoch:          bheight,
 		Rand:           store.NewChainRand(sm.cs, ts.Cids()),
-		Bstore:         sm.cs.Blockstore(),
+		Bstore:         sm.cs.StateBlockstore(),
 		Syscalls:       sm.cs.VMSys(),
 		CircSupplyCalc: sm.GetVMCirculatingSupply,
 		NtwkVersion:    sm.GetNtwkVersion,
@@ -174,7 +174,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 		StateBase:      state,
 		Epoch:          ts.Height() + 1,
 		Rand:           r,
-		Bstore:         sm.cs.Blockstore(),
+		Bstore:         sm.cs.StateBlockstore(),
 		Syscalls:       sm.cs.VMSys(),
 		CircSupplyCalc: sm.GetVMCirculatingSupply,
 		NtwkVersion:    sm.GetNtwkVersion,

chain/stmgr/forks.go

@@ -15,6 +15,7 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/network"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -24,8 +25,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
-	"github.com/filecoin-project/lotus/lib/bufbstore"
 	builtin0 "github.com/filecoin-project/specs-actors/actors/builtin"
 	miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	multisig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig"
@@ -505,7 +504,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
 			}
 		case builtin0.StorageMinerActorCodeID:
 			var st miner0.State
-			if err := sm.ChainStore().Store(ctx).Get(ctx, act.Head, &st); err != nil {
+			if err := sm.ChainStore().ActorStore(ctx).Get(ctx, act.Head, &st); err != nil {
 				return xerrors.Errorf("failed to load miner state: %w", err)
 			}
 
@@ -549,7 +548,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
 		return cid.Undef, xerrors.Errorf("failed to load power actor: %w", err)
 	}
 
-	cst := cbor.NewCborStore(sm.ChainStore().Blockstore())
+	cst := cbor.NewCborStore(sm.ChainStore().StateBlockstore())
 	if err := cst.Get(ctx, powAct.Head, &ps); err != nil {
 		return cid.Undef, xerrors.Errorf("failed to get power actor state: %w", err)
 	}
@@ -583,7 +582,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
 			}
 		case builtin0.StorageMinerActorCodeID:
 			var st miner0.State
-			if err := sm.ChainStore().Store(ctx).Get(ctx, act.Head, &st); err != nil {
+			if err := sm.ChainStore().ActorStore(ctx).Get(ctx, act.Head, &st); err != nil {
 				return xerrors.Errorf("failed to load miner state: %w", err)
 			}
 
@@ -592,7 +591,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
 				return xerrors.Errorf("failed to get miner info: %w", err)
 			}
 
-			sectorsArr, err := adt0.AsArray(sm.ChainStore().Store(ctx), st.Sectors)
+			sectorsArr, err := adt0.AsArray(sm.ChainStore().ActorStore(ctx), st.Sectors)
 			if err != nil {
 				return xerrors.Errorf("failed to load sectors array: %w", err)
 			}
@@ -612,11 +611,11 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
 			lbact, err := lbtree.GetActor(addr)
 			if err == nil {
 				var lbst miner0.State
-				if err := sm.ChainStore().Store(ctx).Get(ctx, lbact.Head, &lbst); err != nil {
+				if err := sm.ChainStore().ActorStore(ctx).Get(ctx, lbact.Head, &lbst); err != nil {
 					return xerrors.Errorf("failed to load miner state: %w", err)
 				}
 
-				lbsectors, err := adt0.AsArray(sm.ChainStore().Store(ctx), lbst.Sectors)
+				lbsectors, err := adt0.AsArray(sm.ChainStore().ActorStore(ctx), lbst.Sectors)
 				if err != nil {
 					return xerrors.Errorf("failed to load lb sectors array: %w", err)
 				}
@@ -712,7 +711,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
 }
 
 func UpgradeIgnition(ctx context.Context, sm *StateManager, _ MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
-	store := sm.cs.Store(ctx)
+	store := sm.cs.ActorStore(ctx)
 
 	if build.UpgradeLiftoffHeight <= epoch {
 		return cid.Undef, xerrors.Errorf("liftoff height must be beyond ignition height")
@@ -768,7 +767,7 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, _ MigrationCache, cb
 
 func UpgradeRefuel(ctx context.Context, sm *StateManager, _ MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
 
-	store := sm.cs.Store(ctx)
+	store := sm.cs.ActorStore(ctx)
 	tree, err := sm.StateTree(root)
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("getting state tree: %w", err)
@@ -793,7 +792,7 @@ func UpgradeRefuel(ctx context.Context, sm *StateManager, _ MigrationCache, cb E
 }
 
 func UpgradeActorsV2(ctx context.Context, sm *StateManager, _ MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
-	buf := bufbstore.NewTieredBstore(sm.cs.Blockstore(), bstore.NewTemporarySync())
+	buf := blockstore.NewTieredBstore(sm.cs.StateBlockstore(), blockstore.NewMemorySync())
 	store := store.ActorStore(ctx, buf)
 
 	info, err := store.Put(ctx, new(types.StateInfo0))
@@ -844,7 +843,7 @@ func UpgradeLiftoff(ctx context.Context, sm *StateManager, _ MigrationCache, cb
 		return cid.Undef, xerrors.Errorf("getting state tree: %w", err)
 	}
 
-	err = setNetworkName(ctx, sm.cs.Store(ctx), tree, "mainnet")
+	err = setNetworkName(ctx, sm.cs.ActorStore(ctx), tree, "mainnet")
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("setting network name: %w", err)
 	}
@@ -853,7 +852,7 @@ func UpgradeLiftoff(ctx context.Context, sm *StateManager, _ MigrationCache, cb
 }
 
 func UpgradeCalico(ctx context.Context, sm *StateManager, _ MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
-	store := sm.cs.Store(ctx)
+	store := sm.cs.ActorStore(ctx)
 	var stateRoot types.StateRoot
 	if err := store.Get(ctx, root, &stateRoot); err != nil {
 		return cid.Undef, xerrors.Errorf("failed to decode state root: %w", err)
@@ -1010,7 +1009,7 @@ func upgradeActorsV3Common(
 	root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet,
 	config nv10.Config,
 ) (cid.Cid, error) {
-	buf := bufbstore.NewTieredBstore(sm.cs.Blockstore(), bstore.NewTemporarySync())
+	buf := blockstore.NewTieredBstore(sm.cs.StateBlockstore(), blockstore.NewMemorySync())
 	store := store.ActorStore(ctx, buf)
 
 	// Load the state root.
@@ -1240,7 +1239,7 @@ func resetGenesisMsigs0(ctx context.Context, sm *StateManager, store adt0.Store,
 		return xerrors.Errorf("getting genesis tipset: %w", err)
 	}
 
-	cst := cbor.NewCborStore(sm.cs.Blockstore())
+	cst := cbor.NewCborStore(sm.cs.StateBlockstore())
 	genesisTree, err := state.LoadStateTree(cst, gts.ParentState())
 	if err != nil {
 		return xerrors.Errorf("loading state tree: %w", err)

chain/stmgr/forks_test.go

@@ -125,7 +125,7 @@ func TestForkHeightTriggers(t *testing.T) {
 			Height:  testForkHeight,
 			Migration: func(ctx context.Context, sm *StateManager, cache MigrationCache, cb ExecCallback,
 				root cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
-				cst := ipldcbor.NewCborStore(sm.ChainStore().Blockstore())
+				cst := ipldcbor.NewCborStore(sm.ChainStore().StateBlockstore())
 
 				st, err := sm.StateTree(root)
 				if err != nil {

chain/stmgr/read.go

@@ -22,7 +22,7 @@ func (sm *StateManager) ParentStateTsk(tsk types.TipSetKey) (*state.StateTree, e
 }
 
 func (sm *StateManager) ParentState(ts *types.TipSet) (*state.StateTree, error) {
-	cst := cbor.NewCborStore(sm.cs.Blockstore())
+	cst := cbor.NewCborStore(sm.cs.StateBlockstore())
 	state, err := state.LoadStateTree(cst, sm.parentState(ts))
 	if err != nil {
 		return nil, xerrors.Errorf("load state tree: %w", err)
@@ -32,7 +32,7 @@ func (sm *StateManager) ParentState(ts *types.TipSet) (*state.StateTree, error)
 }
 
 func (sm *StateManager) StateTree(st cid.Cid) (*state.StateTree, error) {
-	cst := cbor.NewCborStore(sm.cs.Blockstore())
+	cst := cbor.NewCborStore(sm.cs.StateBlockstore())
 	state, err := state.LoadStateTree(cst, st)
 	if err != nil {
 		return nil, xerrors.Errorf("load state tree: %w", err)

chain/stmgr/stmgr.go

@@ -286,7 +286,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
 			StateBase:      base,
 			Epoch:          epoch,
 			Rand:           r,
-			Bstore:         sm.cs.Blockstore(),
+			Bstore:         sm.cs.StateBlockstore(),
 			Syscalls:       sm.cs.VMSys(),
 			CircSupplyCalc: sm.GetVMCirculatingSupply,
 			NtwkVersion:    sm.GetNtwkVersion,
@@ -430,7 +430,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
 		return cid.Cid{}, cid.Cid{}, err
 	}
 
-	rectarr := blockadt.MakeEmptyArray(sm.cs.Store(ctx))
+	rectarr := blockadt.MakeEmptyArray(sm.cs.ActorStore(ctx))
 	for i, receipt := range receipts {
 		if err := rectarr.Set(uint64(i), receipt); err != nil {
 			return cid.Undef, cid.Undef, xerrors.Errorf("failed to build receipts amt: %w", err)
@@ -515,7 +515,7 @@ func (sm *StateManager) ResolveToKeyAddress(ctx context.Context, addr address.Ad
 		ts = sm.cs.GetHeaviestTipSet()
 	}
 
-	cst := cbor.NewCborStore(sm.cs.Blockstore())
+	cst := cbor.NewCborStore(sm.cs.StateBlockstore())
 
 	// First try to resolve the actor in the parent state, so we don't have to compute anything.
 	tree, err := state.LoadStateTree(cst, ts.ParentState())
@@ -556,7 +556,7 @@ func (sm *StateManager) GetBlsPublicKey(ctx context.Context, addr address.Addres
 }
 
 func (sm *StateManager) LookupID(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) {
-	cst := cbor.NewCborStore(sm.cs.Blockstore())
+	cst := cbor.NewCborStore(sm.cs.StateBlockstore())
 	state, err := state.LoadStateTree(cst, sm.parentState(ts))
 	if err != nil {
 		return address.Undef, xerrors.Errorf("load state tree: %w", err)
@@ -882,7 +882,7 @@ func (sm *StateManager) MarketBalance(ctx context.Context, addr address.Address,
 		return api.MarketBalance{}, err
 	}
 
-	mstate, err := market.Load(sm.cs.Store(ctx), act)
+	mstate, err := market.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return api.MarketBalance{}, err
 	}
@@ -966,7 +966,7 @@ func (sm *StateManager) setupGenesisVestingSchedule(ctx context.Context) error {
 		return xerrors.Errorf("getting genesis tipset state: %w", err)
 	}
 
-	cst := cbor.NewCborStore(sm.cs.Blockstore())
+	cst := cbor.NewCborStore(sm.cs.StateBlockstore())
 	sTree, err := state.LoadStateTree(cst, st)
 	if err != nil {
 		return xerrors.Errorf("loading state tree: %w", err)
@@ -1325,7 +1325,7 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha
 			unCirc = big.Add(unCirc, actor.Balance)
 
 		case a == market.Address:
-			mst, err := market.Load(sm.cs.Store(ctx), actor)
+			mst, err := market.Load(sm.cs.ActorStore(ctx), actor)
 			if err != nil {
 				return err
 			}
@@ -1342,7 +1342,7 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha
 			circ = big.Add(circ, actor.Balance)
 
 		case builtin.IsStorageMinerActor(actor.Code):
-			mst, err := miner.Load(sm.cs.Store(ctx), actor)
+			mst, err := miner.Load(sm.cs.ActorStore(ctx), actor)
 			if err != nil {
 				return err
 			}
@@ -1359,7 +1359,7 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha
 			}
 
 		case builtin.IsMultisigActor(actor.Code):
-			mst, err := multisig.Load(sm.cs.Store(ctx), actor)
+			mst, err := multisig.Load(sm.cs.ActorStore(ctx), actor)
 			if err != nil {
 				return err
 			}
@@ -1413,7 +1413,7 @@ func (sm *StateManager) GetPaychState(ctx context.Context, addr address.Address,
 		return nil, nil, err
 	}
 
-	actState, err := paych.Load(sm.cs.Store(ctx), act)
+	actState, err := paych.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -1431,7 +1431,7 @@ func (sm *StateManager) GetMarketState(ctx context.Context, ts *types.TipSet) (m
 		return nil, err
 	}
 
-	actState, err := market.Load(sm.cs.Store(ctx), act)
+	actState, err := market.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return nil, err
 	}

chain/stmgr/utils.go

@@ -48,7 +48,7 @@ func GetNetworkName(ctx context.Context, sm *StateManager, st cid.Cid) (dtypes.N
 	if err != nil {
 		return "", err
 	}
-	ias, err := init_.Load(sm.cs.Store(ctx), act)
+	ias, err := init_.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return "", err
 	}
@@ -65,7 +65,7 @@ func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr
 	if err != nil {
 		return address.Undef, xerrors.Errorf("(get sset) failed to load miner actor: %w", err)
 	}
-	mas, err := miner.Load(sm.cs.Store(ctx), act)
+	mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return address.Undef, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
 	}
@@ -75,7 +75,7 @@ func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr
 		return address.Undef, xerrors.Errorf("failed to load actor info: %w", err)
 	}
 
-	return vm.ResolveToKeyAddr(state, sm.cs.Store(ctx), info.Worker)
+	return vm.ResolveToKeyAddr(state, sm.cs.ActorStore(ctx), info.Worker)
 }
 
 func GetPower(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (power.Claim, power.Claim, bool, error) {
@@ -88,7 +88,7 @@ func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr addres
 		return power.Claim{}, power.Claim{}, false, xerrors.Errorf("(get sset) failed to load power actor state: %w", err)
 	}
 
-	pas, err := power.Load(sm.cs.Store(ctx), act)
+	pas, err := power.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return power.Claim{}, power.Claim{}, false, err
 	}
@@ -123,7 +123,7 @@ func PreCommitInfo(ctx context.Context, sm *StateManager, maddr address.Address,
 		return nil, xerrors.Errorf("(get sset) failed to load miner actor: %w", err)
 	}
 
-	mas, err := miner.Load(sm.cs.Store(ctx), act)
+	mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
 	}
@@ -137,7 +137,7 @@ func MinerSectorInfo(ctx context.Context, sm *StateManager, maddr address.Addres
 		return nil, xerrors.Errorf("(get sset) failed to load miner actor: %w", err)
 	}
 
-	mas, err := miner.Load(sm.cs.Store(ctx), act)
+	mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
 	}
@@ -151,7 +151,7 @@ func GetSectorsForWinningPoSt(ctx context.Context, nv network.Version, pv ffiwra
 		return nil, xerrors.Errorf("failed to load miner actor: %w", err)
 	}
 
-	mas, err := miner.Load(sm.cs.Store(ctx), act)
+	mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to load miner actor state: %w", err)
 	}
@@ -249,7 +249,7 @@ func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, ma
 		return false, xerrors.Errorf("failed to load power actor: %w", err)
 	}
 
-	spas, err := power.Load(sm.cs.Store(ctx), act)
+	spas, err := power.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return false, xerrors.Errorf("failed to load power actor state: %w", err)
 	}
@@ -272,7 +272,7 @@ func GetStorageDeal(ctx context.Context, sm *StateManager, dealID abi.DealID, ts
 		return nil, xerrors.Errorf("failed to load market actor: %w", err)
 	}
 
-	state, err := market.Load(sm.cs.Store(ctx), act)
+	state, err := market.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to load market actor state: %w", err)
 	}
@@ -320,7 +320,7 @@ func ListMinerActors(ctx context.Context, sm *StateManager, ts *types.TipSet) ([
 		return nil, xerrors.Errorf("failed to load power actor: %w", err)
 	}
 
-	powState, err := power.Load(sm.cs.Store(ctx), act)
+	powState, err := power.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to load power actor state: %w", err)
 	}
@@ -353,7 +353,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
 		StateBase:      base,
 		Epoch:          height,
 		Rand:           r,
-		Bstore:         sm.cs.Blockstore(),
+		Bstore:         sm.cs.StateBlockstore(),
 		Syscalls:       sm.cs.VMSys(),
 		CircSupplyCalc: sm.GetVMCirculatingSupply,
 		NtwkVersion:    sm.GetNtwkVersion,
@@ -474,7 +474,7 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule
 		return nil, xerrors.Errorf("failed to load miner actor: %w", err)
 	}
 
-	mas, err := miner.Load(sm.cs.Store(ctx), act)
+	mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to load miner actor state: %w", err)
 	}
@@ -623,7 +623,7 @@ func minerHasMinPower(ctx context.Context, sm *StateManager, addr address.Addres
 		return false, xerrors.Errorf("loading power actor state: %w", err)
 	}
 
-	ps, err := power.Load(sm.cs.Store(ctx), pact)
+	ps, err := power.Load(sm.cs.ActorStore(ctx), pact)
 	if err != nil {
 		return false, err
 	}
@@ -654,7 +654,7 @@ func MinerEligibleToMine(ctx context.Context, sm *StateManager, addr address.Add
 		return false, xerrors.Errorf("loading power actor state: %w", err)
 	}
 
-	pstate, err := power.Load(sm.cs.Store(ctx), pact)
+	pstate, err := power.Load(sm.cs.ActorStore(ctx), pact)
 	if err != nil {
 		return false, err
 	}
@@ -664,7 +664,7 @@ func MinerEligibleToMine(ctx context.Context, sm *StateManager, addr address.Add
 		return false, xerrors.Errorf("loading miner actor state: %w", err)
 	}
 
-	mstate, err := miner.Load(sm.cs.Store(ctx), mact)
+	mstate, err := miner.Load(sm.cs.ActorStore(ctx), mact)
 	if err != nil {
 		return false, err
 	}
@@ -696,7 +696,7 @@ func MinerEligibleToMine(ctx context.Context, sm *StateManager, addr address.Add
 }
 
 func CheckTotalFIL(ctx context.Context, sm *StateManager, ts *types.TipSet) (abi.TokenAmount, error) {
-	str, err := state.LoadStateTree(sm.ChainStore().Store(ctx), ts.ParentState())
+	str, err := state.LoadStateTree(sm.ChainStore().ActorStore(ctx), ts.ParentState())
 	if err != nil {
 		return abi.TokenAmount{}, err
 	}

chain/store/index_test.go

@@ -6,10 +6,10 @@ import (
 	"testing"
 
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/gen"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types/mock"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	datastore "github.com/ipfs/go-datastore"
 	syncds "github.com/ipfs/go-datastore/sync"
 	"github.com/stretchr/testify/assert"
@@ -30,7 +30,7 @@ func TestIndexSeeks(t *testing.T) {
 
 	ctx := context.TODO()
 
-	nbs := blockstore.NewTemporarySync()
+	nbs := blockstore.NewMemorySync()
 	cs := store.NewChainStore(nbs, nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil, nil)
 	defer cs.Close() //nolint:errcheck
 

chain/store/store.go

@@ -23,12 +23,12 @@ import (
 	blockadt "github.com/filecoin-project/specs-actors/actors/util/adt"
 
 	"github.com/filecoin-project/lotus/api"
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/journal"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/metrics"
 
 	"go.opencensus.io/stats"
@@ -81,7 +81,7 @@ func init() {
 }
 
 // ReorgNotifee represents a callback that gets called upon reorgs.
-type ReorgNotifee func(rev, app []*types.TipSet) error
+type ReorgNotifee = func(rev, app []*types.TipSet) error
 
 // Journal event types.
 const (
@@ -107,11 +107,11 @@ type HeadChangeEvt struct {
 //   1. a tipset cache
 //   2. a block => messages references cache.
 type ChainStore struct {
-	bs      bstore.Blockstore
+	chainBlockstore bstore.Blockstore
-	localbs bstore.Blockstore
+	stateBlockstore bstore.Blockstore
-	ds      dstore.Batching
+	metadataDs      dstore.Batching
 
-	localviewer bstore.Viewer
+	chainLocalBlockstore bstore.Blockstore
 
 	heaviestLk sync.Mutex
 	heaviest   *types.TipSet
@@ -139,32 +139,31 @@ type ChainStore struct {
 	wg       sync.WaitGroup
 }
 
-// localbs is guaranteed to fail Get* if requested block isn't stored locally
+func NewChainStore(chainBs bstore.Blockstore, stateBs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder, j journal.Journal) *ChainStore {
-func NewChainStore(bs bstore.Blockstore, localbs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder, j journal.Journal) *ChainStore {
+	c, _ := lru.NewARC(DefaultMsgMetaCacheSize)
-	mmCache, _ := lru.NewARC(DefaultMsgMetaCacheSize)
+	tsc, _ := lru.NewARC(DefaultTipSetCacheSize)
-	tsCache, _ := lru.NewARC(DefaultTipSetCacheSize)
 	if j == nil {
 		j = journal.NilJournal()
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
+	// unwraps the fallback store in case one is configured.
+	// some methods _need_ to operate on a local blockstore only.
+	localbs, _ := bstore.UnwrapFallbackStore(chainBs)
 	cs := &ChainStore{
-		bs:       bs,
+		chainBlockstore:      chainBs,
-		localbs:  localbs,
+		stateBlockstore:      stateBs,
-		ds:       ds,
+		chainLocalBlockstore: localbs,
+		metadataDs:           ds,
 		bestTips:             pubsub.New(64),
 		tipsets:              make(map[abi.ChainEpoch][]cid.Cid),
-		mmCache:  mmCache,
+		mmCache:              c,
-		tsCache:  tsCache,
+		tsCache:              tsc,
 		vmcalls:              vmcalls,
 		cancelFn:             cancel,
 		journal:              j,
 	}
 
-	if v, ok := localbs.(bstore.Viewer); ok {
-		cs.localviewer = v
-	}
-
 	cs.evtTypes = [1]journal.EventType{
 		evtTypeHeadChange: j.RegisterEventType("sync", "head_change"),
 	}
@@ -216,7 +215,7 @@ func (cs *ChainStore) Close() error {
 }
 
 func (cs *ChainStore) Load() error {
-	head, err := cs.ds.Get(chainHeadKey)
+	head, err := cs.metadataDs.Get(chainHeadKey)
 	if err == dstore.ErrNotFound {
 		log.Warn("no previous chain state found")
 		return nil
@@ -246,7 +245,7 @@ func (cs *ChainStore) writeHead(ts *types.TipSet) error {
 		return xerrors.Errorf("failed to marshal tipset: %w", err)
 	}
 
-	if err := cs.ds.Put(chainHeadKey, data); err != nil {
+	if err := cs.metadataDs.Put(chainHeadKey, data); err != nil {
 		return xerrors.Errorf("failed to write chain head to datastore: %w", err)
 	}
 
@@ -306,13 +305,13 @@ func (cs *ChainStore) SubscribeHeadChanges(f ReorgNotifee) {
 func (cs *ChainStore) IsBlockValidated(ctx context.Context, blkid cid.Cid) (bool, error) {
 	key := blockValidationCacheKeyPrefix.Instance(blkid.String())
 
-	return cs.ds.Has(key)
+	return cs.metadataDs.Has(key)
 }
 
 func (cs *ChainStore) MarkBlockAsValidated(ctx context.Context, blkid cid.Cid) error {
 	key := blockValidationCacheKeyPrefix.Instance(blkid.String())
 
-	if err := cs.ds.Put(key, []byte{0}); err != nil {
+	if err := cs.metadataDs.Put(key, []byte{0}); err != nil {
 		return xerrors.Errorf("cache block validation: %w", err)
 	}
 
@@ -322,7 +321,7 @@ func (cs *ChainStore) MarkBlockAsValidated(ctx context.Context, blkid cid.Cid) e
 func (cs *ChainStore) UnmarkBlockAsValidated(ctx context.Context, blkid cid.Cid) error {
 	key := blockValidationCacheKeyPrefix.Instance(blkid.String())
 
-	if err := cs.ds.Delete(key); err != nil {
+	if err := cs.metadataDs.Delete(key); err != nil {
 		return xerrors.Errorf("removing from valid block cache: %w", err)
 	}
 
@@ -339,7 +338,7 @@ func (cs *ChainStore) SetGenesis(b *types.BlockHeader) error {
 		return err
 	}
 
-	return cs.ds.Put(dstore.NewKey("0"), b.Cid().Bytes())
+	return cs.metadataDs.Put(dstore.NewKey("0"), b.Cid().Bytes())
 }
 
 func (cs *ChainStore) PutTipSet(ctx context.Context, ts *types.TipSet) error {
@@ -594,7 +593,7 @@ func (cs *ChainStore) takeHeaviestTipSet(ctx context.Context, ts *types.TipSet)
 // FlushValidationCache removes all results of block validation from the
 // chain metadata store. Usually the first step after a new chain import.
 func (cs *ChainStore) FlushValidationCache() error {
-	return FlushValidationCache(cs.ds)
+	return FlushValidationCache(cs.metadataDs)
 }
 
 func FlushValidationCache(ds datastore.Batching) error {
@@ -653,7 +652,7 @@ func (cs *ChainStore) SetHead(ts *types.TipSet) error {
 // Contains returns whether our BlockStore has all blocks in the supplied TipSet.
 func (cs *ChainStore) Contains(ts *types.TipSet) (bool, error) {
 	for _, c := range ts.Cids() {
-		has, err := cs.bs.Has(c)
+		has, err := cs.chainBlockstore.Has(c)
 		if err != nil {
 			return false, err
 		}
@@ -668,16 +667,8 @@ func (cs *ChainStore) Contains(ts *types.TipSet) (bool, error) {
 // GetBlock fetches a BlockHeader with the supplied CID. It returns
 // blockstore.ErrNotFound if the block was not found in the BlockStore.
 func (cs *ChainStore) GetBlock(c cid.Cid) (*types.BlockHeader, error) {
-	if cs.localviewer == nil {
-		sb, err := cs.localbs.Get(c)
-		if err != nil {
-			return nil, err
-		}
-		return types.DecodeBlock(sb.RawData())
-	}
-
 	var blk *types.BlockHeader
-	err := cs.localviewer.View(c, func(b []byte) (err error) {
+	err := cs.chainLocalBlockstore.View(c, func(b []byte) (err error) {
 		blk, err = types.DecodeBlock(b)
 		return err
 	})
@@ -851,7 +842,7 @@ func (cs *ChainStore) PersistBlockHeaders(b ...*types.BlockHeader) error {
 			end = len(b)
 		}
 
-		err = multierr.Append(err, cs.bs.PutMany(sbs[start:end]))
+		err = multierr.Append(err, cs.chainLocalBlockstore.PutMany(sbs[start:end]))
 	}
 
 	return err
@@ -875,7 +866,7 @@ func PutMessage(bs bstore.Blockstore, m storable) (cid.Cid, error) {
 }
 
 func (cs *ChainStore) PutMessage(m storable) (cid.Cid, error) {
-	return PutMessage(cs.bs, m)
+	return PutMessage(cs.chainBlockstore, m)
 }
 
 func (cs *ChainStore) expandTipset(b *types.BlockHeader) (*types.TipSet, error) {
@@ -936,7 +927,7 @@ func (cs *ChainStore) AddBlock(ctx context.Context, b *types.BlockHeader) error
 }
 
 func (cs *ChainStore) GetGenesis() (*types.BlockHeader, error) {
-	data, err := cs.ds.Get(dstore.NewKey("0"))
+	data, err := cs.metadataDs.Get(dstore.NewKey("0"))
 	if err != nil {
 		return nil, err
 	}
@@ -962,17 +953,8 @@ func (cs *ChainStore) GetCMessage(c cid.Cid) (types.ChainMsg, error) {
 }
 
 func (cs *ChainStore) GetMessage(c cid.Cid) (*types.Message, error) {
-	if cs.localviewer == nil {
-		sb, err := cs.localbs.Get(c)
-		if err != nil {
-			log.Errorf("get message get failed: %s: %s", c, err)
-			return nil, err
-		}
-		return types.DecodeMessage(sb.RawData())
-	}
-
 	var msg *types.Message
-	err := cs.localviewer.View(c, func(b []byte) (err error) {
+	err := cs.chainLocalBlockstore.View(c, func(b []byte) (err error) {
 		msg, err = types.DecodeMessage(b)
 		return err
 	})
@@ -980,17 +962,8 @@ func (cs *ChainStore) GetMessage(c cid.Cid) (*types.Message, error) {
 }
 
 func (cs *ChainStore) GetSignedMessage(c cid.Cid) (*types.SignedMessage, error) {
-	if cs.localviewer == nil {
-		sb, err := cs.localbs.Get(c)
-		if err != nil {
-			log.Errorf("get message get failed: %s: %s", c, err)
-			return nil, err
-		}
-		return types.DecodeSignedMessage(sb.RawData())
-	}
-
 	var msg *types.SignedMessage
-	err := cs.localviewer.View(c, func(b []byte) (err error) {
+	err := cs.chainLocalBlockstore.View(c, func(b []byte) (err error) {
 		msg, err = types.DecodeSignedMessage(b)
 		return err
 	})
@@ -1000,7 +973,7 @@ func (cs *ChainStore) GetSignedMessage(c cid.Cid) (*types.SignedMessage, error)
 func (cs *ChainStore) readAMTCids(root cid.Cid) ([]cid.Cid, error) {
 	ctx := context.TODO()
 	// block headers use adt0, for now.
-	a, err := blockadt.AsArray(cs.Store(ctx), root)
+	a, err := blockadt.AsArray(cs.ActorStore(ctx), root)
 	if err != nil {
 		return nil, xerrors.Errorf("amt load: %w", err)
 	}
@@ -1124,7 +1097,7 @@ func (cs *ChainStore) ReadMsgMetaCids(mmc cid.Cid) ([]cid.Cid, []cid.Cid, error)
 		return mmcids.bls, mmcids.secpk, nil
 	}
 
-	cst := cbor.NewCborStore(cs.localbs)
+	cst := cbor.NewCborStore(cs.chainLocalBlockstore)
 	var msgmeta types.MsgMeta
 	if err := cst.Get(context.TODO(), mmc, &msgmeta); err != nil {
 		return nil, nil, xerrors.Errorf("failed to load msgmeta (%s): %w", mmc, err)
@@ -1194,7 +1167,7 @@ func (cs *ChainStore) MessagesForBlock(b *types.BlockHeader) ([]*types.Message,
 func (cs *ChainStore) GetParentReceipt(b *types.BlockHeader, i int) (*types.MessageReceipt, error) {
 	ctx := context.TODO()
 	// block headers use adt0, for now.
-	a, err := blockadt.AsArray(cs.Store(ctx), b.ParentMessageReceipts)
+	a, err := blockadt.AsArray(cs.ActorStore(ctx), b.ParentMessageReceipts)
 	if err != nil {
 		return nil, xerrors.Errorf("amt load: %w", err)
 	}
@@ -1237,16 +1210,26 @@ func (cs *ChainStore) LoadSignedMessagesFromCids(cids []cid.Cid) ([]*types.Signe
 	return msgs, nil
 }
 
-func (cs *ChainStore) Blockstore() bstore.Blockstore {
+// ChainBlockstore returns the chain blockstore. Currently the chain and state
-	return cs.bs
+// // stores are both backed by the same physical store, albeit with different
+// // caching policies, but in the future they will segregate.
+func (cs *ChainStore) ChainBlockstore() bstore.Blockstore {
+	return cs.chainBlockstore
+}
+
+// StateBlockstore returns the state blockstore. Currently the chain and state
+// stores are both backed by the same physical store, albeit with different
+// caching policies, but in the future they will segregate.
+func (cs *ChainStore) StateBlockstore() bstore.Blockstore {
+	return cs.stateBlockstore
 }
 
 func ActorStore(ctx context.Context, bs bstore.Blockstore) adt.Store {
 	return adt.WrapStore(ctx, cbor.NewCborStore(bs))
 }
 
-func (cs *ChainStore) Store(ctx context.Context) adt.Store {
+func (cs *ChainStore) ActorStore(ctx context.Context) adt.Store {
-	return ActorStore(ctx, cs.bs)
+	return ActorStore(ctx, cs.stateBlockstore)
 }
 
 func (cs *ChainStore) VMSys() vm.SyscallBuilder {
@@ -1444,8 +1427,9 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo
 		return xerrors.Errorf("failed to write car header: %s", err)
 	}
 
-	return cs.WalkSnapshot(ctx, ts, inclRecentRoots, skipOldMsgs, func(c cid.Cid) error {
+	unionBs := bstore.Union(cs.stateBlockstore, cs.chainBlockstore)
-		blk, err := cs.bs.Get(c)
+	return cs.WalkSnapshot(ctx, ts, inclRecentRoots, skipOldMsgs, true, func(c cid.Cid) error {
+		blk, err := unionBs.Get(c)
 		if err != nil {
 			return xerrors.Errorf("writing object to car, bs.Get: %w", err)
 		}
@@ -1458,7 +1442,7 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo
 	})
 }
 
-func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, skipOldMsgs bool, cb func(cid.Cid) error) error {
+func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, skipOldMsgs, skipMsgReceipts bool, cb func(cid.Cid) error) error {
 	if ts == nil {
 		ts = cs.GetHeaviestTipSet()
 	}
@@ -1478,7 +1462,7 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
 			return err
 		}
 
-		data, err := cs.bs.Get(blk)
+		data, err := cs.chainBlockstore.Get(blk)
 		if err != nil {
 			return xerrors.Errorf("getting block: %w", err)
 		}
@@ -1498,7 +1482,7 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
 		var cids []cid.Cid
 		if !skipOldMsgs || b.Height > ts.Height()-inclRecentRoots {
 			if walked.Visit(b.Messages) {
-				mcids, err := recurseLinks(cs.bs, walked, b.Messages, []cid.Cid{b.Messages})
+				mcids, err := recurseLinks(cs.chainBlockstore, walked, b.Messages, []cid.Cid{b.Messages})
 				if err != nil {
 					return xerrors.Errorf("recursing messages failed: %w", err)
 				}
@@ -1519,13 +1503,17 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
 
 		if b.Height == 0 || b.Height > ts.Height()-inclRecentRoots {
 			if walked.Visit(b.ParentStateRoot) {
-				cids, err := recurseLinks(cs.bs, walked, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot})
+				cids, err := recurseLinks(cs.stateBlockstore, walked, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot})
 				if err != nil {
 					return xerrors.Errorf("recursing genesis state failed: %w", err)
 				}
 
 				out = append(out, cids...)
 			}
+
+			if !skipMsgReceipts && walked.Visit(b.ParentMessageReceipts) {
+				out = append(out, b.ParentMessageReceipts)
+			}
 		}
 
 		for _, c := range out {
@@ -1561,7 +1549,12 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
 }
 
 func (cs *ChainStore) Import(r io.Reader) (*types.TipSet, error) {
-	header, err := car.LoadCar(cs.Blockstore(), r)
+	// TODO: writing only to the state blockstore is incorrect.
+	//  At this time, both the state and chain blockstores are backed by the
+	//  universal store. When we physically segregate the stores, we will need
+	//  to route state objects to the state blockstore, and chain objects to
+	//  the chain blockstore.
+	header, err := car.LoadCar(cs.StateBlockstore(), r)
 	if err != nil {
 		return nil, xerrors.Errorf("loadcar failed: %w", err)
 	}

chain/store/store_test.go

@@ -11,12 +11,12 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/crypto"
 
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
 	"github.com/filecoin-project/lotus/chain/gen"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
@@ -52,7 +52,7 @@ func BenchmarkGetRandomness(b *testing.B) {
 		b.Fatal(err)
 	}
 
-	bs, err := lr.Blockstore(context.TODO(), repo.BlockstoreChain)
+	bs, err := lr.Blockstore(context.TODO(), repo.UniversalBlockstore)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -104,7 +104,7 @@ func TestChainExportImport(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	nbs := blockstore.NewTemporary()
+	nbs := blockstore.NewMemory()
 	cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), nil, nil)
 	defer cs.Close() //nolint:errcheck
 
@@ -139,7 +139,7 @@ func TestChainExportImportFull(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	nbs := blockstore.NewTemporary()
+	nbs := blockstore.NewMemory()
 	cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), nil, nil)
 	defer cs.Close() //nolint:errcheck
 

chain/store/weight.go

@@ -28,7 +28,7 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn
 
 	tpow := big2.Zero()
 	{
-		cst := cbor.NewCborStore(cs.Blockstore())
+		cst := cbor.NewCborStore(cs.StateBlockstore())
 		state, err := state.LoadStateTree(cst, ts.ParentState())
 		if err != nil {
 			return types.NewInt(0), xerrors.Errorf("load state tree: %w", err)
@@ -39,7 +39,7 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn
 			return types.NewInt(0), xerrors.Errorf("get power actor: %w", err)
 		}
 
-		powState, err := power.Load(cs.Store(ctx), act)
+		powState, err := power.Load(cs.ActorStore(ctx), act)
 		if err != nil {
 			return types.NewInt(0), xerrors.Errorf("failed to load power actor state: %w", err)
 		}

chain/sub/incoming.go

@@ -7,13 +7,13 @@ import (
 	"time"
 
 	address "github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain"
 	"github.com/filecoin-project/lotus/chain/messagepool"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/lib/sigs"
 	"github.com/filecoin-project/lotus/metrics"
 	"github.com/filecoin-project/lotus/node/impl/client"
@@ -101,7 +101,7 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
 					[]tag.Mutator{tag.Insert(metrics.MinerID, blk.Header.Miner.String())},
 					metrics.BlockDelay.M(delay),
 				)
-				log.Warnf("Received block with large delay %d from miner %s", delay, blk.Header.Miner)
+				log.Warnw("received block with large delay from miner", "block", blk.Cid(), "delay", delay, "miner", blk.Header.Miner)
 			}
 
 			if s.InformNewBlock(msg.ReceivedFrom, &types.FullBlock{
@@ -392,7 +392,7 @@ func (bv *BlockValidator) isChainNearSynced() bool {
 func (bv *BlockValidator) validateMsgMeta(ctx context.Context, msg *types.BlockMsg) error {
 	// TODO there has to be a simpler way to do this without the blockstore dance
 	// block headers use adt0
-	store := blockadt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewTemporary()))
+	store := blockadt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewMemory()))
 	bmArr := blockadt.MakeEmptyArray(store)
 	smArr := blockadt.MakeEmptyArray(store)
 

chain/sync.go

@@ -44,6 +44,7 @@ import (
 	proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof"
 
 	"github.com/filecoin-project/lotus/api"
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
 	"github.com/filecoin-project/lotus/chain/beacon"
@@ -54,7 +55,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/lib/sigs"
 	"github.com/filecoin-project/lotus/metrics"
 )
@@ -321,7 +321,7 @@ func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error {
 
 	// We use a temporary bstore here to avoid writing intermediate pieces
 	// into the blockstore.
-	blockstore := bstore.NewTemporary()
+	blockstore := bstore.NewMemory()
 	cst := cbor.NewCborStore(blockstore)
 
 	var bcids, scids []cid.Cid
@@ -354,7 +354,7 @@ func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error {
 	}
 
 	// Finally, flush.
-	return vm.Copy(context.TODO(), blockstore, syncer.store.Blockstore(), smroot)
+	return vm.Copy(context.TODO(), blockstore, syncer.store.ChainBlockstore(), smroot)
 }
 
 func (syncer *Syncer) LocalPeer() peer.ID {
@@ -640,7 +640,7 @@ func (syncer *Syncer) minerIsValid(ctx context.Context, maddr address.Address, b
 		return xerrors.Errorf("failed to load power actor: %w", err)
 	}
 
-	powState, err := power.Load(syncer.store.Store(ctx), act)
+	powState, err := power.Load(syncer.store.ActorStore(ctx), act)
 	if err != nil {
 		return xerrors.Errorf("failed to load power actor state: %w", err)
 	}
@@ -1055,7 +1055,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock
 		return err
 	}
 
-	st, err := state.LoadStateTree(syncer.store.Store(ctx), stateroot)
+	st, err := state.LoadStateTree(syncer.store.ActorStore(ctx), stateroot)
 	if err != nil {
 		return xerrors.Errorf("failed to load base state tree: %w", err)
 	}
@@ -1102,7 +1102,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock
 	}
 
 	// Validate message arrays in a temporary blockstore.
-	tmpbs := bstore.NewTemporary()
+	tmpbs := bstore.NewMemory()
 	tmpstore := blockadt.WrapStore(ctx, cbor.NewCborStore(tmpbs))
 
 	bmArr := blockadt.MakeEmptyArray(tmpstore)
@@ -1172,7 +1172,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock
 	}
 
 	// Finally, flush.
-	return vm.Copy(ctx, tmpbs, syncer.store.Blockstore(), mrcid)
+	return vm.Copy(ctx, tmpbs, syncer.store.ChainBlockstore(), mrcid)
 }
 
 func (syncer *Syncer) verifyBlsAggregate(ctx context.Context, sig *crypto.Signature, msgs []cid.Cid, pubks [][]byte) error {
@@ -1553,7 +1553,7 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS
 
 		for bsi := 0; bsi < len(bstout); bsi++ {
 			// temp storage so we don't persist data we dont want to
-			bs := bstore.NewTemporary()
+			bs := bstore.NewMemory()
 			blks := cbor.NewCborStore(bs)
 
 			this := headers[i-bsi]
@@ -1574,7 +1574,7 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS
 				return err
 			}
 
-			if err := copyBlockstore(ctx, bs, syncer.store.Blockstore()); err != nil {
+			if err := copyBlockstore(ctx, bs, syncer.store.ChainBlockstore()); err != nil {
 				return xerrors.Errorf("message processing failed: %w", err)
 			}
 		}

chain/vm/vm.go

@@ -28,6 +28,7 @@ import (
 	"github.com/filecoin-project/go-state-types/exitcode"
 	"github.com/filecoin-project/go-state-types/network"
 
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/aerrors"
@@ -36,9 +37,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin/reward"
 	"github.com/filecoin-project/lotus/chain/state"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/blockstore"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
-	"github.com/filecoin-project/lotus/lib/bufbstore"
 )
 
 const MaxCallDepth = 4096
@@ -208,7 +206,7 @@ type VM struct {
 	cstate         *state.StateTree
 	base           cid.Cid
 	cst            *cbor.BasicIpldStore
-	buf            *bufbstore.BufferedBS
+	buf            *blockstore.BufferedBlockstore
 	blockHeight    abi.ChainEpoch
 	areg           *ActorRegistry
 	rand           Rand
@@ -224,7 +222,7 @@ type VMOpts struct {
 	StateBase      cid.Cid
 	Epoch          abi.ChainEpoch
 	Rand           Rand
-	Bstore         bstore.Blockstore
+	Bstore         blockstore.Blockstore
 	Syscalls       SyscallBuilder
 	CircSupplyCalc CircSupplyCalculator
 	NtwkVersion    NtwkVersionGetter // TODO: stebalien: In what cases do we actually need this? It seems like even when creating new networks we want to use the 'global'/build-default version getter
@@ -233,7 +231,7 @@ type VMOpts struct {
 }
 
 func NewVM(ctx context.Context, opts *VMOpts) (*VM, error) {
-	buf := bufbstore.NewBufferedBstore(opts.Bstore)
+	buf := blockstore.NewBuffered(opts.Bstore)
 	cst := cbor.NewCborStore(buf)
 	state, err := state.LoadStateTree(cst, opts.StateBase)
 	if err != nil {

cli/auth.go

@@ -9,6 +9,7 @@ import (
 	"github.com/filecoin-project/go-jsonrpc/auth"
 
 	"github.com/filecoin-project/lotus/api/apistruct"
+	cliutil "github.com/filecoin-project/lotus/cli/util"
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
@@ -127,7 +128,7 @@ var authApiInfoToken = &cli.Command{
 
 		// TODO: Log in audit log when it is implemented
 
-		fmt.Printf("%s=%s:%s\n", envForRepo(t), string(token), ainfo.Addr)
+		fmt.Printf("%s=%s:%s\n", cliutil.EnvForRepo(t), string(token), ainfo.Addr)
 		return nil
 	},
 }

cli/backup.go

@@ -51,7 +51,10 @@ func BackupCmd(repoFlag string, rt repo.RepoType, getApi BackupApiFn) *cli.Comma
 			return xerrors.Errorf("getting metadata datastore: %w", err)
 		}
 
-		bds := backupds.Wrap(mds)
+		bds, err := backupds.Wrap(mds, backupds.NoLogdir)
+		if err != nil {
+			return err
+		}
 
 		fpath, err := homedir.Expand(cctx.Args().First())
 		if err != nil {

cli/chain.go

@@ -642,7 +642,10 @@ var chainListCmd = &cli.Command{
 						gasUsed += r.GasUsed
 					}
 
-					fmt.Printf("\ttipset: \t%d msgs, %d / %d (%0.2f%%)\n", len(msgs), gasUsed, limitSum, 100*float64(gasUsed)/float64(limitSum))
+					gasEfficiency := 100 * float64(gasUsed) / float64(limitSum)
+					gasCapacity := 100 * float64(limitSum) / float64(build.BlockGasLimit)
+
+					fmt.Printf("\ttipset: \t%d msgs, %d (%0.2f%%) / %d (%0.2f%%)\n", len(msgs), gasUsed, gasEfficiency, limitSum, gasCapacity)
 				}
 				fmt.Println()
 			}

cli/client.go

@@ -1195,6 +1195,11 @@ var clientListAsksCmd = &cli.Command{
 		&cli.BoolFlag{
 			Name: "by-ping",
 		},
+		&cli.StringFlag{
+			Name:  "output-format",
+			Value: "text",
+			Usage: "Either 'text' or 'csv'",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		api, closer, err := GetFullNodeAPI(cctx)
@@ -1214,11 +1219,16 @@ var clientListAsksCmd = &cli.Command{
 				return asks[i].Ping < asks[j].Ping
 			})
 		}
+		pfmt := "%s: min:%s max:%s price:%s/GiB/Epoch verifiedPrice:%s/GiB/Epoch ping:%s\n"
+		if cctx.String("output-format") == "csv" {
+			fmt.Printf("Miner,Min,Max,Price,VerifiedPrice,Ping\n")
+			pfmt = "%s,%s,%s,%s,%s,%s\n"
+		}
 
 		for _, a := range asks {
 			ask := a.Ask
 
-			fmt.Printf("%s: min:%s max:%s price:%s/GiB/Epoch verifiedPrice:%s/GiB/Epoch ping:%s\n", ask.Miner,
+			fmt.Printf(pfmt, ask.Miner,
 				types.SizeStr(types.NewInt(uint64(ask.MinPieceSize))),
 				types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize))),
 				types.FIL(ask.Price),
@@ -1237,7 +1247,13 @@ type QueriedAsk struct {
 }
 
 func GetAsks(ctx context.Context, api lapi.FullNode) ([]QueriedAsk, error) {
+	isTTY := true
+	if fileInfo, _ := os.Stdout.Stat(); (fileInfo.Mode() & os.ModeCharDevice) == 0 {
+		isTTY = false
+	}
+	if isTTY {
 		color.Blue(".. getting miner list")
+	}
 	miners, err := api.StateListMiners(ctx, types.EmptyTSK)
 	if err != nil {
 		return nil, xerrors.Errorf("getting miner list: %w", err)
@@ -1282,14 +1298,18 @@ loop:
 	for {
 		select {
 		case <-time.After(150 * time.Millisecond):
+			if isTTY {
 				fmt.Printf("\r* Found %d miners with power", atomic.LoadInt64(&found))
+			}
 		case <-done:
 			break loop
 		}
 	}
+	if isTTY {
 		fmt.Printf("\r* Found %d miners with power\n", atomic.LoadInt64(&found))
 
 		color.Blue(".. querying asks")
+	}
 
 	var asks []QueriedAsk
 	var queried, got int64
@@ -1349,12 +1369,16 @@ loop2:
 	for {
 		select {
 		case <-time.After(150 * time.Millisecond):
+			if isTTY {
 				fmt.Printf("\r* Queried %d asks, got %d responses", atomic.LoadInt64(&queried), atomic.LoadInt64(&got))
+			}
 		case <-done:
 			break loop2
 		}
 	}
+	if isTTY {
 		fmt.Printf("\r* Queried %d asks, got %d responses\n", atomic.LoadInt64(&queried), atomic.LoadInt64(&got))
+	}
 
 	sort.Slice(asks, func(i, j int) bool {
 		return asks[i].Ask.Price.LessThan(asks[j].Ask.Price)

cli/cmd.go

@@ -1,34 +1,17 @@
 package cli
 
 import (
-	"context"
-	"fmt"
-	"net/http"
-	"net/url"
-	"os"
-	"os/signal"
 	"strings"
-	"syscall"
 
 	logging "github.com/ipfs/go-log/v2"
-	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-jsonrpc"
 
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/api/client"
 	cliutil "github.com/filecoin-project/lotus/cli/util"
-	"github.com/filecoin-project/lotus/node/repo"
 )
 
 var log = logging.Logger("cli")
 
-const (
-	metadataTraceContext = "traceContext"
-)
-
 // custom CLI error
 
 type ErrCmdFailed struct {
@@ -46,253 +29,31 @@ func NewCliError(s string) error {
 // ApiConnector returns API instance
 type ApiConnector func() api.FullNode
 
-// The flag passed on the command line with the listen address of the API
+func GetFullNodeServices(ctx *cli.Context) (ServicesAPI, error) {
-// server (only used by the tests)
+	if tn, ok := ctx.App.Metadata["test-services"]; ok {
-func flagForAPI(t repo.RepoType) string {
+		return tn.(ServicesAPI), nil
-	switch t {
-	case repo.FullNode:
-		return "api-url"
-	case repo.StorageMiner:
-		return "miner-api-url"
-	case repo.Worker:
-		return "worker-api-url"
-	default:
-		panic(fmt.Sprintf("Unknown repo type: %v", t))
-	}
-}
-
-func flagForRepo(t repo.RepoType) string {
-	switch t {
-	case repo.FullNode:
-		return "repo"
-	case repo.StorageMiner:
-		return "miner-repo"
-	case repo.Worker:
-		return "worker-repo"
-	default:
-		panic(fmt.Sprintf("Unknown repo type: %v", t))
-	}
-}
-
-func envForRepo(t repo.RepoType) string {
-	switch t {
-	case repo.FullNode:
-		return "FULLNODE_API_INFO"
-	case repo.StorageMiner:
-		return "MINER_API_INFO"
-	case repo.Worker:
-		return "WORKER_API_INFO"
-	default:
-		panic(fmt.Sprintf("Unknown repo type: %v", t))
-	}
-}
-
-// TODO remove after deprecation period
-func envForRepoDeprecation(t repo.RepoType) string {
-	switch t {
-	case repo.FullNode:
-		return "FULLNODE_API_INFO"
-	case repo.StorageMiner:
-		return "STORAGE_API_INFO"
-	case repo.Worker:
-		return "WORKER_API_INFO"
-	default:
-		panic(fmt.Sprintf("Unknown repo type: %v", t))
-	}
-}
-
-func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (cliutil.APIInfo, error) {
-	// Check if there was a flag passed with the listen address of the API
-	// server (only used by the tests)
-	apiFlag := flagForAPI(t)
-	if ctx.IsSet(apiFlag) {
-		strma := ctx.String(apiFlag)
-		strma = strings.TrimSpace(strma)
-
-		return cliutil.APIInfo{Addr: strma}, nil
 	}
 
-	envKey := envForRepo(t)
+	api, c, err := GetFullNodeAPI(ctx)
-	env, ok := os.LookupEnv(envKey)
-	if !ok {
-		// TODO remove after deprecation period
-		envKey = envForRepoDeprecation(t)
-		env, ok = os.LookupEnv(envKey)
-		if ok {
-			log.Warnf("Use deprecation env(%s) value, please use env(%s) instead.", envKey, envForRepo(t))
-		}
-	}
-	if ok {
-		return cliutil.ParseApiInfo(env), nil
-	}
-
-	repoFlag := flagForRepo(t)
-
-	p, err := homedir.Expand(ctx.String(repoFlag))
 	if err != nil {
-		return cliutil.APIInfo{}, xerrors.Errorf("could not expand home dir (%s): %w", repoFlag, err)
+		return nil, err
 	}
 
-	r, err := repo.NewFS(p)
+	return &ServicesImpl{api: api, closer: c}, nil
-	if err != nil {
-		return cliutil.APIInfo{}, xerrors.Errorf("could not open repo at path: %s; %w", p, err)
-	}
-
-	ma, err := r.APIEndpoint()
-	if err != nil {
-		return cliutil.APIInfo{}, xerrors.Errorf("could not get api endpoint: %w", err)
-	}
-
-	token, err := r.APIToken()
-	if err != nil {
-		log.Warnf("Couldn't load CLI token, capabilities may be limited: %v", err)
-	}
-
-	return cliutil.APIInfo{
-		Addr:  ma.String(),
-		Token: token,
-	}, nil
 }
 
-func GetRawAPI(ctx *cli.Context, t repo.RepoType) (string, http.Header, error) {
+var GetAPIInfo = cliutil.GetAPIInfo
-	ainfo, err := GetAPIInfo(ctx, t)
+var GetRawAPI = cliutil.GetRawAPI
-	if err != nil {
+var GetAPI = cliutil.GetAPI
-		return "", nil, xerrors.Errorf("could not get API info: %w", err)
-	}
 
-	addr, err := ainfo.DialArgs()
+var DaemonContext = cliutil.DaemonContext
-	if err != nil {
+var ReqContext = cliutil.ReqContext
-		return "", nil, xerrors.Errorf("could not get DialArgs: %w", err)
-	}
 
-	return addr, ainfo.AuthHeader(), nil
+var GetFullNodeAPI = cliutil.GetFullNodeAPI
-}
+var GetGatewayAPI = cliutil.GetGatewayAPI
 
-func GetAPI(ctx *cli.Context) (api.Common, jsonrpc.ClientCloser, error) {
+var GetStorageMinerAPI = cliutil.GetStorageMinerAPI
-	ti, ok := ctx.App.Metadata["repoType"]
+var GetWorkerAPI = cliutil.GetWorkerAPI
-	if !ok {
-		log.Errorf("unknown repo type, are you sure you want to use GetAPI?")
-		ti = repo.FullNode
-	}
-	t, ok := ti.(repo.RepoType)
-	if !ok {
-		log.Errorf("repoType type does not match the type of repo.RepoType")
-	}
-
-	if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
-		return tn.(api.StorageMiner), func() {}, nil
-	}
-	if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
-		return tn.(api.FullNode), func() {}, nil
-	}
-
-	addr, headers, err := GetRawAPI(ctx, t)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return client.NewCommonRPC(ctx.Context, addr, headers)
-}
-
-func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error) {
-	if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
-		return tn.(api.FullNode), func() {}, nil
-	}
-
-	addr, headers, err := GetRawAPI(ctx, repo.FullNode)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return client.NewFullNodeRPC(ctx.Context, addr, headers)
-}
-
-type GetStorageMinerOptions struct {
-	PreferHttp bool
-}
-
-type GetStorageMinerOption func(*GetStorageMinerOptions)
-
-func StorageMinerUseHttp(opts *GetStorageMinerOptions) {
-	opts.PreferHttp = true
-}
-
-func GetStorageMinerAPI(ctx *cli.Context, opts ...GetStorageMinerOption) (api.StorageMiner, jsonrpc.ClientCloser, error) {
-	var options GetStorageMinerOptions
-	for _, opt := range opts {
-		opt(&options)
-	}
-
-	if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
-		return tn.(api.StorageMiner), func() {}, nil
-	}
-
-	addr, headers, err := GetRawAPI(ctx, repo.StorageMiner)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	if options.PreferHttp {
-		u, err := url.Parse(addr)
-		if err != nil {
-			return nil, nil, xerrors.Errorf("parsing miner api URL: %w", err)
-		}
-
-		switch u.Scheme {
-		case "ws":
-			u.Scheme = "http"
-		case "wss":
-			u.Scheme = "https"
-		}
-
-		addr = u.String()
-	}
-
-	return client.NewStorageMinerRPC(ctx.Context, addr, headers)
-}
-
-func GetWorkerAPI(ctx *cli.Context) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
-	addr, headers, err := GetRawAPI(ctx, repo.Worker)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return client.NewWorkerRPC(ctx.Context, addr, headers)
-}
-
-func GetGatewayAPI(ctx *cli.Context) (api.GatewayAPI, jsonrpc.ClientCloser, error) {
-	addr, headers, err := GetRawAPI(ctx, repo.FullNode)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return client.NewGatewayRPC(ctx.Context, addr, headers)
-}
-
-func DaemonContext(cctx *cli.Context) context.Context {
-	if mtCtx, ok := cctx.App.Metadata[metadataTraceContext]; ok {
-		return mtCtx.(context.Context)
-	}
-
-	return context.Background()
-}
-
-// ReqContext returns context for cli execution. Calling it for the first time
-// installs SIGTERM handler that will close returned context.
-// Not safe for concurrent execution.
-func ReqContext(cctx *cli.Context) context.Context {
-	tCtx := DaemonContext(cctx)
-
-	ctx, done := context.WithCancel(tCtx)
-	sigChan := make(chan os.Signal, 2)
-	go func() {
-		<-sigChan
-		done()
-	}()
-	signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)
-
-	return ctx
-}
 
 var CommonCommands = []*cli.Command{
 	netCmd,

cli/multisig.go

@@ -29,7 +29,7 @@ import (
 	init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init"
 	msig2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig"
 
-	"github.com/filecoin-project/lotus/api/apibstore"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/multisig"
@@ -202,7 +202,7 @@ var msigInspectCmd = &cli.Command{
 		defer closer()
 		ctx := ReqContext(cctx)
 
-		store := adt.WrapStore(ctx, cbor.NewCborStore(apibstore.NewAPIBlockstore(api)))
+		store := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(api)))
 
 		maddr, err := address.NewFromString(cctx.Args().First())
 		if err != nil {
@@ -1275,7 +1275,7 @@ var msigLockApproveCmd = &cli.Command{
 		params, actErr := actors.SerializeParams(&msig2.LockBalanceParams{
 			StartEpoch:     abi.ChainEpoch(start),
 			UnlockDuration: abi.ChainEpoch(duration),
-			Amount:         abi.NewTokenAmount(amount.Int64()),
+			Amount:         big.Int(amount),
 		})
 
 		if actErr != nil {
@@ -1367,7 +1367,7 @@ var msigLockCancelCmd = &cli.Command{
 		params, actErr := actors.SerializeParams(&msig2.LockBalanceParams{
 			StartEpoch:     abi.ChainEpoch(start),
 			UnlockDuration: abi.ChainEpoch(duration),
-			Amount:         abi.NewTokenAmount(amount.Int64()),
+			Amount:         big.Int(amount),
 		})
 
 		if actErr != nil {

cli/net.go

@@ -48,6 +48,11 @@ var NetPeers = &cli.Command{
 			Aliases: []string{"a"},
 			Usage:   "Print agent name",
 		},
+		&cli.BoolFlag{
+			Name:    "extended",
+			Aliases: []string{"x"},
+			Usage:   "Print extended peer information in json",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		api, closer, err := GetAPI(cctx)
@@ -65,6 +70,30 @@ var NetPeers = &cli.Command{
 			return strings.Compare(string(peers[i].ID), string(peers[j].ID)) > 0
 		})
 
+		if cctx.Bool("extended") {
+			// deduplicate
+			seen := make(map[peer.ID]struct{})
+
+			for _, peer := range peers {
+				_, dup := seen[peer.ID]
+				if dup {
+					continue
+				}
+				seen[peer.ID] = struct{}{}
+
+				info, err := api.NetPeerInfo(ctx, peer.ID)
+				if err != nil {
+					log.Warnf("error getting extended peer info: %s", err)
+				} else {
+					bytes, err := json.Marshal(&info)
+					if err != nil {
+						log.Warnf("error marshalling extended peer info: %s", err)
+					} else {
+						fmt.Println(string(bytes))
+					}
+				}
+			}
+		} else {
 			for _, peer := range peers {
 				var agent string
 				if cctx.Bool("agent") {
@@ -75,9 +104,9 @@ var NetPeers = &cli.Command{
 						agent = ", " + agent
 					}
 				}
-
 				fmt.Printf("%s, %s%s\n", peer.ID, peer.Addrs, agent)
 			}
+		}
 
 		return nil
 	},
@@ -89,6 +118,7 @@ var netScores = &cli.Command{
 	Flags: []cli.Flag{
 		&cli.BoolFlag{
 			Name:    "extended",
+			Aliases: []string{"x"},
 			Usage:   "print extended peer scores in json",
 		},
 	},

cli/paych_test.go

@@ -20,8 +20,8 @@ import (
 	cbor "github.com/ipfs/go-ipld-cbor"
 	"github.com/stretchr/testify/require"
 
-	"github.com/filecoin-project/lotus/api/apibstore"
 	"github.com/filecoin-project/lotus/api/test"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/events"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -400,7 +400,7 @@ func getPaychState(ctx context.Context, t *testing.T, node test.TestNode, chAddr
 	act, err := node.StateGetActor(ctx, chAddr, types.EmptyTSK)
 	require.NoError(t, err)
 
-	store := cbor.NewCborStore(apibstore.NewAPIBlockstore(node))
+	store := cbor.NewCborStore(blockstore.NewAPIBlockstore(node))
 	chState, err := paych.Load(adt.WrapStore(ctx, store), act)
 	require.NoError(t, err)
 

cli/send.go

@@ -1,22 +1,17 @@
 package cli
 
 import (
-	"bytes"
-	"context"
 	"encoding/hex"
-	"encoding/json"
+	"errors"
 	"fmt"
-	"reflect"
 
 	"github.com/urfave/cli/v2"
-	cbg "github.com/whyrusleeping/cbor-gen"
+	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 
-	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
-	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
@@ -72,15 +67,16 @@ var sendCmd = &cli.Command{
 			return ShowHelp(cctx, fmt.Errorf("'send' expects two arguments, target and amount"))
 		}
 
-		api, closer, err := GetFullNodeAPI(cctx)
+		srv, err := GetFullNodeServices(cctx)
 		if err != nil {
 			return err
 		}
-		defer closer()
+		defer srv.Close() //nolint:errcheck
 
 		ctx := ReqContext(cctx)
+		var params SendParams
 
-		toAddr, err := address.NewFromString(cctx.Args().Get(0))
+		params.To, err = address.NewFromString(cctx.Args().Get(0))
 		if err != nil {
 			return ShowHelp(cctx, fmt.Errorf("failed to parse target address: %w", err))
 		}
@@ -89,123 +85,75 @@ var sendCmd = &cli.Command{
 		if err != nil {
 			return ShowHelp(cctx, fmt.Errorf("failed to parse amount: %w", err))
 		}
+		params.Val = abi.TokenAmount(val)
 
-		var fromAddr address.Address
+		if from := cctx.String("from"); from != "" {
-		if from := cctx.String("from"); from == "" {
-			defaddr, err := api.WalletDefaultAddress(ctx)
-			if err != nil {
-				return err
-			}
-
-			fromAddr = defaddr
-		} else {
 			addr, err := address.NewFromString(from)
 			if err != nil {
 				return err
 			}
 
-			fromAddr = addr
+			params.From = addr
 		}
 
+		if cctx.IsSet("gas-premium") {
 			gp, err := types.BigFromString(cctx.String("gas-premium"))
 			if err != nil {
 				return err
 			}
+			params.GasPremium = &gp
+		}
+
+		if cctx.IsSet("gas-feecap") {
 			gfc, err := types.BigFromString(cctx.String("gas-feecap"))
 			if err != nil {
 				return err
 			}
+			params.GasFeeCap = &gfc
+		}
 
-		method := abi.MethodNum(cctx.Uint64("method"))
+		if cctx.IsSet("gas-limit") {
+			limit := cctx.Int64("gas-limit")
+			params.GasLimit = &limit
+		}
+
+		params.Method = abi.MethodNum(cctx.Uint64("method"))
 
-		var params []byte
 		if cctx.IsSet("params-json") {
-			decparams, err := decodeTypedParams(ctx, api, toAddr, method, cctx.String("params-json"))
+			decparams, err := srv.DecodeTypedParamsFromJSON(ctx, params.To, params.Method, cctx.String("params-json"))
 			if err != nil {
 				return fmt.Errorf("failed to decode json params: %w", err)
 			}
-			params = decparams
+			params.Params = decparams
 		}
 		if cctx.IsSet("params-hex") {
-			if params != nil {
+			if params.Params != nil {
 				return fmt.Errorf("can only specify one of 'params-json' and 'params-hex'")
 			}
 			decparams, err := hex.DecodeString(cctx.String("params-hex"))
 			if err != nil {
 				return fmt.Errorf("failed to decode hex params: %w", err)
 			}
-			params = decparams
+			params.Params = decparams
 		}
 
-		msg := &types.Message{
+		params.Force = cctx.Bool("force")
-			From:       fromAddr,
-			To:         toAddr,
-			Value:      types.BigInt(val),
-			GasPremium: gp,
-			GasFeeCap:  gfc,
-			GasLimit:   cctx.Int64("gas-limit"),
-			Method:     method,
-			Params:     params,
-		}
-
-		if !cctx.Bool("force") {
-			// Funds insufficient check
-			fromBalance, err := api.WalletBalance(ctx, msg.From)
-			if err != nil {
-				return err
-			}
-			totalCost := types.BigAdd(types.BigMul(msg.GasFeeCap, types.NewInt(uint64(msg.GasLimit))), msg.Value)
-
-			if fromBalance.LessThan(totalCost) {
-				fmt.Printf("WARNING: From balance %s less than total cost %s\n", types.FIL(fromBalance), types.FIL(totalCost))
-				return fmt.Errorf("--force must be specified for this action to have an effect; you have been warned")
-			}
-		}
 
 		if cctx.IsSet("nonce") {
-			msg.Nonce = cctx.Uint64("nonce")
+			n := cctx.Uint64("nonce")
-			sm, err := api.WalletSignMessage(ctx, fromAddr, msg)
+			params.Nonce = &n
-			if err != nil {
-				return err
 		}
 
-			_, err = api.MpoolPush(ctx, sm)
+		msgCid, err := srv.Send(ctx, params)
+
 		if err != nil {
-				return err
+			if errors.Is(err, ErrSendBalanceTooLow) {
+				return fmt.Errorf("--force must be specified for this action to have an effect; you have been warned: %w", err)
 			}
-			fmt.Println(sm.Cid())
+			return xerrors.Errorf("executing send: %w", err)
-		} else {
-			sm, err := api.MpoolPushMessage(ctx, msg, nil)
-			if err != nil {
-				return err
-			}
-			fmt.Println(sm.Cid())
 		}
 
+		fmt.Fprintf(cctx.App.Writer, "%s\n", msgCid)
 		return nil
 	},
 }
-
-func decodeTypedParams(ctx context.Context, fapi api.FullNode, to address.Address, method abi.MethodNum, paramstr string) ([]byte, error) {
-	act, err := fapi.StateGetActor(ctx, to, types.EmptyTSK)
-	if err != nil {
-		return nil, err
-	}
-
-	methodMeta, found := stmgr.MethodsMap[act.Code][method]
-	if !found {
-		return nil, fmt.Errorf("method %d not found on actor %s", method, act.Code)
-	}
-
-	p := reflect.New(methodMeta.Params.Elem()).Interface().(cbg.CBORMarshaler)
-
-	if err := json.Unmarshal([]byte(paramstr), p); err != nil {
-		return nil, fmt.Errorf("unmarshaling input into params type: %w", err)
-	}
-
-	buf := new(bytes.Buffer)
-	if err := p.MarshalCBOR(buf); err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}

cli/send_test.go

@@ -0,0 +1,128 @@
+package cli
+
+import (
+	"bytes"
+	"errors"
+	"testing"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	types "github.com/filecoin-project/lotus/chain/types"
+	gomock "github.com/golang/mock/gomock"
+	cid "github.com/ipfs/go-cid"
+	"github.com/stretchr/testify/assert"
+	ucli "github.com/urfave/cli/v2"
+)
+
+var arbtCid = (&types.Message{
+	From:  mustAddr(address.NewIDAddress(2)),
+	To:    mustAddr(address.NewIDAddress(1)),
+	Value: types.NewInt(1000),
+}).Cid()
+
+func mustAddr(a address.Address, err error) address.Address {
+	if err != nil {
+		panic(err)
+	}
+	return a
+}
+
+func newMockApp(t *testing.T, cmd *ucli.Command) (*ucli.App, *MockServicesAPI, *bytes.Buffer, func()) {
+	app := ucli.NewApp()
+	app.Commands = ucli.Commands{cmd}
+	app.Setup()
+
+	mockCtrl := gomock.NewController(t)
+	mockSrvcs := NewMockServicesAPI(mockCtrl)
+	app.Metadata["test-services"] = mockSrvcs
+
+	buf := &bytes.Buffer{}
+	app.Writer = buf
+
+	return app, mockSrvcs, buf, mockCtrl.Finish
+}
+
+func TestSendCLI(t *testing.T) {
+	oneFil := abi.TokenAmount(types.MustParseFIL("1"))
+
+	t.Run("simple", func(t *testing.T) {
+		app, mockSrvcs, buf, done := newMockApp(t, sendCmd)
+		defer done()
+
+		gomock.InOrder(
+			mockSrvcs.EXPECT().Send(gomock.Any(), SendParams{
+				To:  mustAddr(address.NewIDAddress(1)),
+				Val: oneFil,
+			}).Return(arbtCid, nil),
+			mockSrvcs.EXPECT().Close(),
+		)
+		err := app.Run([]string{"lotus", "send", "t01", "1"})
+		assert.NoError(t, err)
+		assert.EqualValues(t, arbtCid.String()+"\n", buf.String())
+	})
+	t.Run("ErrSendBalanceTooLow", func(t *testing.T) {
+		app, mockSrvcs, _, done := newMockApp(t, sendCmd)
+		defer done()
+
+		gomock.InOrder(
+			mockSrvcs.EXPECT().Send(gomock.Any(), SendParams{
+				To:  mustAddr(address.NewIDAddress(1)),
+				Val: oneFil,
+			}).Return(cid.Undef, ErrSendBalanceTooLow),
+			mockSrvcs.EXPECT().Close(),
+		)
+		err := app.Run([]string{"lotus", "send", "t01", "1"})
+		assert.ErrorIs(t, err, ErrSendBalanceTooLow)
+	})
+	t.Run("generic-err-is-forwarded", func(t *testing.T) {
+		app, mockSrvcs, _, done := newMockApp(t, sendCmd)
+		defer done()
+
+		errMark := errors.New("something")
+		gomock.InOrder(
+			mockSrvcs.EXPECT().Send(gomock.Any(), SendParams{
+				To:  mustAddr(address.NewIDAddress(1)),
+				Val: oneFil,
+			}).Return(cid.Undef, errMark),
+			mockSrvcs.EXPECT().Close(),
+		)
+		err := app.Run([]string{"lotus", "send", "t01", "1"})
+		assert.ErrorIs(t, err, errMark)
+	})
+
+	t.Run("from-specific", func(t *testing.T) {
+		app, mockSrvcs, buf, done := newMockApp(t, sendCmd)
+		defer done()
+
+		gomock.InOrder(
+			mockSrvcs.EXPECT().Send(gomock.Any(), SendParams{
+				To:   mustAddr(address.NewIDAddress(1)),
+				From: mustAddr(address.NewIDAddress(2)),
+				Val:  oneFil,
+			}).Return(arbtCid, nil),
+			mockSrvcs.EXPECT().Close(),
+		)
+		err := app.Run([]string{"lotus", "send", "--from=t02", "t01", "1"})
+		assert.NoError(t, err)
+		assert.EqualValues(t, arbtCid.String()+"\n", buf.String())
+	})
+
+	t.Run("nonce-specific", func(t *testing.T) {
+		app, mockSrvcs, buf, done := newMockApp(t, sendCmd)
+		defer done()
+		zero := uint64(0)
+
+		gomock.InOrder(
+			mockSrvcs.EXPECT().Send(gomock.Any(), SendParams{
+				To:    mustAddr(address.NewIDAddress(1)),
+				Nonce: &zero,
+				Val:   oneFil,
+			}).Return(arbtCid, nil),
+			mockSrvcs.EXPECT().Close(),
+		)
+		err := app.Run([]string{"lotus", "send", "--nonce=0", "t01", "1"})
+		assert.NoError(t, err)
+		assert.EqualValues(t, arbtCid.String()+"\n", buf.String())
+	})
+
+}

cli/services.go

@@ -0,0 +1,166 @@
+package cli
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"reflect"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-jsonrpc"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/stmgr"
+	types "github.com/filecoin-project/lotus/chain/types"
+	cid "github.com/ipfs/go-cid"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	"golang.org/x/xerrors"
+)
+
+//go:generate go run github.com/golang/mock/mockgen -destination=servicesmock_test.go -package=cli -self_package github.com/filecoin-project/lotus/cli . ServicesAPI
+
+type ServicesAPI interface {
+	// Sends executes a send given SendParams
+	Send(ctx context.Context, params SendParams) (cid.Cid, error)
+	// DecodeTypedParamsFromJSON takes in information needed to identify a method and converts JSON
+	// parameters to bytes of their CBOR encoding
+	DecodeTypedParamsFromJSON(ctx context.Context, to address.Address, method abi.MethodNum, paramstr string) ([]byte, error)
+
+	// Close ends the session of services and disconnects from RPC, using Services after Close is called
+	// most likely will result in an error
+	// Should not be called concurrently
+	Close() error
+}
+
+type ServicesImpl struct {
+	api    api.FullNode
+	closer jsonrpc.ClientCloser
+}
+
+func (s *ServicesImpl) Close() error {
+	if s.closer == nil {
+		return xerrors.Errorf("Services already closed")
+	}
+	s.closer()
+	s.closer = nil
+	return nil
+}
+
+func (s *ServicesImpl) DecodeTypedParamsFromJSON(ctx context.Context, to address.Address, method abi.MethodNum, paramstr string) ([]byte, error) {
+	act, err := s.api.StateGetActor(ctx, to, types.EmptyTSK)
+	if err != nil {
+		return nil, err
+	}
+
+	methodMeta, found := stmgr.MethodsMap[act.Code][method]
+	if !found {
+		return nil, fmt.Errorf("method %d not found on actor %s", method, act.Code)
+	}
+
+	p := reflect.New(methodMeta.Params.Elem()).Interface().(cbg.CBORMarshaler)
+
+	if err := json.Unmarshal([]byte(paramstr), p); err != nil {
+		return nil, fmt.Errorf("unmarshaling input into params type: %w", err)
+	}
+
+	buf := new(bytes.Buffer)
+	if err := p.MarshalCBOR(buf); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+type SendParams struct {
+	To   address.Address
+	From address.Address
+	Val  abi.TokenAmount
+
+	GasPremium *abi.TokenAmount
+	GasFeeCap  *abi.TokenAmount
+	GasLimit   *int64
+
+	Nonce  *uint64
+	Method abi.MethodNum
+	Params []byte
+
+	Force bool
+}
+
+// This is specialised Send for Send command
+// There might be room for generic Send that other commands can use to send their messages
+// We will see
+
+var ErrSendBalanceTooLow = errors.New("balance too low")
+
+func (s *ServicesImpl) Send(ctx context.Context, params SendParams) (cid.Cid, error) {
+	if params.From == address.Undef {
+		defaddr, err := s.api.WalletDefaultAddress(ctx)
+		if err != nil {
+			return cid.Undef, err
+		}
+		params.From = defaddr
+	}
+
+	msg := &types.Message{
+		From:  params.From,
+		To:    params.To,
+		Value: params.Val,
+
+		Method: params.Method,
+		Params: params.Params,
+	}
+
+	if params.GasPremium != nil {
+		msg.GasPremium = *params.GasPremium
+	} else {
+		msg.GasPremium = types.NewInt(0)
+	}
+	if params.GasFeeCap != nil {
+		msg.GasFeeCap = *params.GasFeeCap
+	} else {
+		msg.GasFeeCap = types.NewInt(0)
+	}
+	if params.GasLimit != nil {
+		msg.GasLimit = *params.GasLimit
+	} else {
+		msg.GasLimit = 0
+	}
+
+	if !params.Force {
+		// Funds insufficient check
+		fromBalance, err := s.api.WalletBalance(ctx, msg.From)
+		if err != nil {
+			return cid.Undef, err
+		}
+		totalCost := types.BigAdd(types.BigMul(msg.GasFeeCap, types.NewInt(uint64(msg.GasLimit))), msg.Value)
+
+		if fromBalance.LessThan(totalCost) {
+			return cid.Undef, xerrors.Errorf("From balance %s less than total cost %s: %w", types.FIL(fromBalance), types.FIL(totalCost), ErrSendBalanceTooLow)
+
+		}
+	}
+
+	if params.Nonce != nil {
+		msg.Nonce = *params.Nonce
+		sm, err := s.api.WalletSignMessage(ctx, params.From, msg)
+		if err != nil {
+			return cid.Undef, err
+		}
+
+		_, err = s.api.MpoolPush(ctx, sm)
+		if err != nil {
+			return cid.Undef, err
+		}
+
+		return sm.Cid(), nil
+	}
+
+	sm, err := s.api.MpoolPushMessage(ctx, msg, nil)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	return sm.Cid(), nil
+}

cli/services_send_test.go

@@ -0,0 +1,266 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/api/mocks"
+	types "github.com/filecoin-project/lotus/chain/types"
+	gomock "github.com/golang/mock/gomock"
+	cid "github.com/ipfs/go-cid"
+	"github.com/stretchr/testify/assert"
+)
+
+type markerKeyType struct{}
+
+var markerKey = markerKeyType{}
+
+type contextMatcher struct {
+	marker *int
+}
+
+// Matches returns whether x is a match.
+func (cm contextMatcher) Matches(x interface{}) bool {
+	ctx, ok := x.(context.Context)
+	if !ok {
+		return false
+	}
+	maybeMarker, ok := ctx.Value(markerKey).(*int)
+	if !ok {
+		return false
+	}
+
+	return cm.marker == maybeMarker
+}
+
+func (cm contextMatcher) String() string {
+	return fmt.Sprintf("Context with Value(%v/%T, %p)", markerKey, markerKey, cm.marker)
+}
+
+func ContextWithMarker(ctx context.Context) (context.Context, gomock.Matcher) {
+	marker := new(int)
+	outCtx := context.WithValue(ctx, markerKey, marker)
+	return outCtx, contextMatcher{marker: marker}
+
+}
+
+func setupMockSrvcs(t *testing.T) (*ServicesImpl, *mocks.MockFullNode) {
+	mockCtrl := gomock.NewController(t)
+
+	mockApi := mocks.NewMockFullNode(mockCtrl)
+
+	srvcs := &ServicesImpl{
+		api:    mockApi,
+		closer: mockCtrl.Finish,
+	}
+	return srvcs, mockApi
+}
+
+func fakeSign(msg *types.Message) *types.SignedMessage {
+	return &types.SignedMessage{
+		Message:   *msg,
+		Signature: crypto.Signature{Type: crypto.SigTypeSecp256k1, Data: make([]byte, 32)},
+	}
+}
+
+func makeMessageSigner() (*cid.Cid, interface{}) {
+	smCid := cid.Undef
+	return &smCid,
+		func(_ context.Context, msg *types.Message, _ *api.MessageSendSpec) (*types.SignedMessage, error) {
+			sm := fakeSign(msg)
+			smCid = sm.Cid()
+			return sm, nil
+		}
+}
+
+type MessageMatcher SendParams
+
+var _ gomock.Matcher = MessageMatcher{}
+
+// Matches returns whether x is a match.
+func (mm MessageMatcher) Matches(x interface{}) bool {
+	m, ok := x.(*types.Message)
+	if !ok {
+		return false
+	}
+
+	if mm.From != address.Undef && mm.From != m.From {
+		return false
+	}
+	if mm.To != address.Undef && mm.To != m.To {
+		return false
+	}
+
+	if types.BigCmp(mm.Val, m.Value) != 0 {
+		return false
+	}
+
+	if mm.Nonce != nil && *mm.Nonce != m.Nonce {
+		return false
+	}
+
+	if mm.GasPremium != nil && big.Cmp(*mm.GasPremium, m.GasPremium) != 0 {
+		return false
+	}
+	if mm.GasPremium == nil && m.GasPremium.Sign() != 0 {
+		return false
+	}
+
+	if mm.GasFeeCap != nil && big.Cmp(*mm.GasFeeCap, m.GasFeeCap) != 0 {
+		return false
+	}
+	if mm.GasFeeCap == nil && m.GasFeeCap.Sign() != 0 {
+		return false
+	}
+
+	if mm.GasLimit != nil && *mm.GasLimit != m.GasLimit {
+		return false
+	}
+
+	if mm.GasLimit == nil && m.GasLimit != 0 {
+		return false
+	}
+	// handle rest of options
+	return true
+}
+
+// String describes what the matcher matches.
+func (mm MessageMatcher) String() string {
+	return fmt.Sprintf("%#v", SendParams(mm))
+}
+
+func TestSendService(t *testing.T) {
+	addrGen := address.NewForTestGetter()
+	a1 := addrGen()
+	a2 := addrGen()
+
+	const balance = 10000
+
+	params := SendParams{
+		From: a1,
+		To:   a2,
+		Val:  types.NewInt(balance - 100),
+	}
+
+	ctx, ctxM := ContextWithMarker(context.Background())
+
+	t.Run("happy", func(t *testing.T) {
+		params := params
+		srvcs, mockApi := setupMockSrvcs(t)
+		defer srvcs.Close() //nolint:errcheck
+		msgCid, sign := makeMessageSigner()
+		gomock.InOrder(
+			mockApi.EXPECT().WalletBalance(ctxM, params.From).Return(types.NewInt(balance), nil),
+			mockApi.EXPECT().MpoolPushMessage(ctxM, MessageMatcher(params), nil).DoAndReturn(sign),
+		)
+
+		c, err := srvcs.Send(ctx, params)
+		assert.NoError(t, err)
+		assert.Equal(t, *msgCid, c)
+	})
+
+	t.Run("balance-too-low", func(t *testing.T) {
+		params := params
+		srvcs, mockApi := setupMockSrvcs(t)
+		defer srvcs.Close() //nolint:errcheck
+		gomock.InOrder(
+			mockApi.EXPECT().WalletBalance(ctxM, a1).Return(types.NewInt(balance-200), nil),
+			// no MpoolPushMessage
+		)
+
+		c, err := srvcs.Send(ctx, params)
+		assert.Equal(t, c, cid.Undef)
+		assert.ErrorIs(t, err, ErrSendBalanceTooLow)
+	})
+
+	t.Run("force", func(t *testing.T) {
+		params := params
+		params.Force = true
+		srvcs, mockApi := setupMockSrvcs(t)
+		defer srvcs.Close() //nolint:errcheck
+		msgCid, sign := makeMessageSigner()
+		gomock.InOrder(
+			mockApi.EXPECT().WalletBalance(ctxM, a1).Return(types.NewInt(balance-200), nil).AnyTimes(),
+			mockApi.EXPECT().MpoolPushMessage(ctxM, MessageMatcher(params), nil).DoAndReturn(sign),
+		)
+
+		c, err := srvcs.Send(ctx, params)
+		assert.NoError(t, err)
+		assert.Equal(t, *msgCid, c)
+	})
+
+	t.Run("default-from", func(t *testing.T) {
+		params := params
+		params.From = address.Undef
+		mm := MessageMatcher(params)
+		mm.From = a1
+
+		srvcs, mockApi := setupMockSrvcs(t)
+		defer srvcs.Close() //nolint:errcheck
+		msgCid, sign := makeMessageSigner()
+		gomock.InOrder(
+			mockApi.EXPECT().WalletDefaultAddress(ctxM).Return(a1, nil),
+			mockApi.EXPECT().WalletBalance(ctxM, a1).Return(types.NewInt(balance), nil),
+			mockApi.EXPECT().MpoolPushMessage(ctxM, mm, nil).DoAndReturn(sign),
+		)
+
+		c, err := srvcs.Send(ctx, params)
+		assert.NoError(t, err)
+		assert.Equal(t, *msgCid, c)
+	})
+
+	t.Run("set-nonce", func(t *testing.T) {
+		params := params
+		n := uint64(5)
+		params.Nonce = &n
+		mm := MessageMatcher(params)
+
+		srvcs, mockApi := setupMockSrvcs(t)
+		defer srvcs.Close() //nolint:errcheck
+		_, _ = mm, mockApi
+
+		var sm *types.SignedMessage
+		gomock.InOrder(
+			mockApi.EXPECT().WalletBalance(ctxM, a1).Return(types.NewInt(balance), nil),
+			mockApi.EXPECT().WalletSignMessage(ctxM, a1, mm).DoAndReturn(
+				func(_ context.Context, _ address.Address, msg *types.Message) (*types.SignedMessage, error) {
+					sm = fakeSign(msg)
+
+					// now we expect MpoolPush with that SignedMessage
+					mockApi.EXPECT().MpoolPush(ctxM, sm).Return(sm.Cid(), nil)
+					return sm, nil
+				}),
+		)
+
+		c, err := srvcs.Send(ctx, params)
+		assert.NoError(t, err)
+		assert.Equal(t, sm.Cid(), c)
+	})
+
+	t.Run("gas-params", func(t *testing.T) {
+		params := params
+		limit := int64(1)
+		params.GasLimit = &limit
+		gfc := big.NewInt(100)
+		params.GasFeeCap = &gfc
+		gp := big.NewInt(10)
+		params.GasPremium = &gp
+
+		srvcs, mockApi := setupMockSrvcs(t)
+		defer srvcs.Close() //nolint:errcheck
+		msgCid, sign := makeMessageSigner()
+		gomock.InOrder(
+			mockApi.EXPECT().WalletBalance(ctxM, params.From).Return(types.NewInt(balance), nil),
+			mockApi.EXPECT().MpoolPushMessage(ctxM, MessageMatcher(params), nil).DoAndReturn(sign),
+		)
+
+		c, err := srvcs.Send(ctx, params)
+		assert.NoError(t, err)
+		assert.Equal(t, *msgCid, c)
+	})
+}

cli/servicesmock_test.go

@@ -0,0 +1,81 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/filecoin-project/lotus/cli (interfaces: ServicesAPI)
+
+// Package cli is a generated GoMock package.
+package cli
+
+import (
+	context "context"
+	go_address "github.com/filecoin-project/go-address"
+	abi "github.com/filecoin-project/go-state-types/abi"
+	gomock "github.com/golang/mock/gomock"
+	go_cid "github.com/ipfs/go-cid"
+	reflect "reflect"
+)
+
+// MockServicesAPI is a mock of ServicesAPI interface
+type MockServicesAPI struct {
+	ctrl     *gomock.Controller
+	recorder *MockServicesAPIMockRecorder
+}
+
+// MockServicesAPIMockRecorder is the mock recorder for MockServicesAPI
+type MockServicesAPIMockRecorder struct {
+	mock *MockServicesAPI
+}
+
+// NewMockServicesAPI creates a new mock instance
+func NewMockServicesAPI(ctrl *gomock.Controller) *MockServicesAPI {
+	mock := &MockServicesAPI{ctrl: ctrl}
+	mock.recorder = &MockServicesAPIMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockServicesAPI) EXPECT() *MockServicesAPIMockRecorder {
+	return m.recorder
+}
+
+// Close mocks base method
+func (m *MockServicesAPI) Close() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Close")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Close indicates an expected call of Close
+func (mr *MockServicesAPIMockRecorder) Close() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockServicesAPI)(nil).Close))
+}
+
+// DecodeTypedParamsFromJSON mocks base method
+func (m *MockServicesAPI) DecodeTypedParamsFromJSON(arg0 context.Context, arg1 go_address.Address, arg2 abi.MethodNum, arg3 string) ([]byte, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DecodeTypedParamsFromJSON", arg0, arg1, arg2, arg3)
+	ret0, _ := ret[0].([]byte)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// DecodeTypedParamsFromJSON indicates an expected call of DecodeTypedParamsFromJSON
+func (mr *MockServicesAPIMockRecorder) DecodeTypedParamsFromJSON(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DecodeTypedParamsFromJSON", reflect.TypeOf((*MockServicesAPI)(nil).DecodeTypedParamsFromJSON), arg0, arg1, arg2, arg3)
+}
+
+// Send mocks base method
+func (m *MockServicesAPI) Send(arg0 context.Context, arg1 SendParams) (go_cid.Cid, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Send", arg0, arg1)
+	ret0, _ := ret[0].(go_cid.Cid)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Send indicates an expected call of Send
+func (mr *MockServicesAPIMockRecorder) Send(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockServicesAPI)(nil).Send), arg0, arg1)
+}

cli/state.go

@@ -34,7 +34,7 @@ import (
 
 	"github.com/filecoin-project/lotus/api"
 	lapi "github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/api/apibstore"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/state"
 	"github.com/filecoin-project/lotus/chain/stmgr"
@@ -1010,7 +1010,7 @@ var stateComputeStateCmd = &cli.Command{
 		}
 
 		if cctx.Bool("html") {
-			st, err := state.LoadStateTree(cbor.NewCborStore(apibstore.NewAPIBlockstore(api)), stout.Root)
+			st, err := state.LoadStateTree(cbor.NewCborStore(blockstore.NewAPIBlockstore(api)), stout.Root)
 			if err != nil {
 				return xerrors.Errorf("loading state tree: %w", err)
 			}

cli/util/api.go

@@ -0,0 +1,274 @@
+package cliutil
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+
+	"github.com/mitchellh/go-homedir"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-jsonrpc"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/api/client"
+	"github.com/filecoin-project/lotus/node/repo"
+)
+
+const (
+	metadataTraceContext = "traceContext"
+)
+
+// The flag passed on the command line with the listen address of the API
+// server (only used by the tests)
+func flagForAPI(t repo.RepoType) string {
+	switch t {
+	case repo.FullNode:
+		return "api-url"
+	case repo.StorageMiner:
+		return "miner-api-url"
+	case repo.Worker:
+		return "worker-api-url"
+	default:
+		panic(fmt.Sprintf("Unknown repo type: %v", t))
+	}
+}
+
+func flagForRepo(t repo.RepoType) string {
+	switch t {
+	case repo.FullNode:
+		return "repo"
+	case repo.StorageMiner:
+		return "miner-repo"
+	case repo.Worker:
+		return "worker-repo"
+	default:
+		panic(fmt.Sprintf("Unknown repo type: %v", t))
+	}
+}
+
+func EnvForRepo(t repo.RepoType) string {
+	switch t {
+	case repo.FullNode:
+		return "FULLNODE_API_INFO"
+	case repo.StorageMiner:
+		return "MINER_API_INFO"
+	case repo.Worker:
+		return "WORKER_API_INFO"
+	default:
+		panic(fmt.Sprintf("Unknown repo type: %v", t))
+	}
+}
+
+// TODO remove after deprecation period
+func envForRepoDeprecation(t repo.RepoType) string {
+	switch t {
+	case repo.FullNode:
+		return "FULLNODE_API_INFO"
+	case repo.StorageMiner:
+		return "STORAGE_API_INFO"
+	case repo.Worker:
+		return "WORKER_API_INFO"
+	default:
+		panic(fmt.Sprintf("Unknown repo type: %v", t))
+	}
+}
+
+func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) {
+	// Check if there was a flag passed with the listen address of the API
+	// server (only used by the tests)
+	apiFlag := flagForAPI(t)
+	if ctx.IsSet(apiFlag) {
+		strma := ctx.String(apiFlag)
+		strma = strings.TrimSpace(strma)
+
+		return APIInfo{Addr: strma}, nil
+	}
+
+	envKey := EnvForRepo(t)
+	env, ok := os.LookupEnv(envKey)
+	if !ok {
+		// TODO remove after deprecation period
+		envKey = envForRepoDeprecation(t)
+		env, ok = os.LookupEnv(envKey)
+		if ok {
+			log.Warnf("Use deprecation env(%s) value, please use env(%s) instead.", envKey, EnvForRepo(t))
+		}
+	}
+	if ok {
+		return ParseApiInfo(env), nil
+	}
+
+	repoFlag := flagForRepo(t)
+
+	p, err := homedir.Expand(ctx.String(repoFlag))
+	if err != nil {
+		return APIInfo{}, xerrors.Errorf("could not expand home dir (%s): %w", repoFlag, err)
+	}
+
+	r, err := repo.NewFS(p)
+	if err != nil {
+		return APIInfo{}, xerrors.Errorf("could not open repo at path: %s; %w", p, err)
+	}
+
+	ma, err := r.APIEndpoint()
+	if err != nil {
+		return APIInfo{}, xerrors.Errorf("could not get api endpoint: %w", err)
+	}
+
+	token, err := r.APIToken()
+	if err != nil {
+		log.Warnf("Couldn't load CLI token, capabilities may be limited: %v", err)
+	}
+
+	return APIInfo{
+		Addr:  ma.String(),
+		Token: token,
+	}, nil
+}
+
+func GetRawAPI(ctx *cli.Context, t repo.RepoType) (string, http.Header, error) {
+	ainfo, err := GetAPIInfo(ctx, t)
+	if err != nil {
+		return "", nil, xerrors.Errorf("could not get API info: %w", err)
+	}
+
+	addr, err := ainfo.DialArgs()
+	if err != nil {
+		return "", nil, xerrors.Errorf("could not get DialArgs: %w", err)
+	}
+
+	return addr, ainfo.AuthHeader(), nil
+}
+
+func GetAPI(ctx *cli.Context) (api.Common, jsonrpc.ClientCloser, error) {
+	ti, ok := ctx.App.Metadata["repoType"]
+	if !ok {
+		log.Errorf("unknown repo type, are you sure you want to use GetAPI?")
+		ti = repo.FullNode
+	}
+	t, ok := ti.(repo.RepoType)
+	if !ok {
+		log.Errorf("repoType type does not match the type of repo.RepoType")
+	}
+
+	if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
+		return tn.(api.StorageMiner), func() {}, nil
+	}
+	if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
+		return tn.(api.FullNode), func() {}, nil
+	}
+
+	addr, headers, err := GetRawAPI(ctx, t)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return client.NewCommonRPC(ctx.Context, addr, headers)
+}
+
+func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error) {
+	if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
+		return tn.(api.FullNode), func() {}, nil
+	}
+
+	addr, headers, err := GetRawAPI(ctx, repo.FullNode)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return client.NewFullNodeRPC(ctx.Context, addr, headers)
+}
+
+type GetStorageMinerOptions struct {
+	PreferHttp bool
+}
+
+type GetStorageMinerOption func(*GetStorageMinerOptions)
+
+func StorageMinerUseHttp(opts *GetStorageMinerOptions) {
+	opts.PreferHttp = true
+}
+
+func GetStorageMinerAPI(ctx *cli.Context, opts ...GetStorageMinerOption) (api.StorageMiner, jsonrpc.ClientCloser, error) {
+	var options GetStorageMinerOptions
+	for _, opt := range opts {
+		opt(&options)
+	}
+
+	if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
+		return tn.(api.StorageMiner), func() {}, nil
+	}
+
+	addr, headers, err := GetRawAPI(ctx, repo.StorageMiner)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if options.PreferHttp {
+		u, err := url.Parse(addr)
+		if err != nil {
+			return nil, nil, xerrors.Errorf("parsing miner api URL: %w", err)
+		}
+
+		switch u.Scheme {
+		case "ws":
+			u.Scheme = "http"
+		case "wss":
+			u.Scheme = "https"
+		}
+
+		addr = u.String()
+	}
+
+	return client.NewStorageMinerRPC(ctx.Context, addr, headers)
+}
+
+func GetWorkerAPI(ctx *cli.Context) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
+	addr, headers, err := GetRawAPI(ctx, repo.Worker)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return client.NewWorkerRPC(ctx.Context, addr, headers)
+}
+
+func GetGatewayAPI(ctx *cli.Context) (api.GatewayAPI, jsonrpc.ClientCloser, error) {
+	addr, headers, err := GetRawAPI(ctx, repo.FullNode)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return client.NewGatewayRPC(ctx.Context, addr, headers)
+}
+
+func DaemonContext(cctx *cli.Context) context.Context {
+	if mtCtx, ok := cctx.App.Metadata[metadataTraceContext]; ok {
+		return mtCtx.(context.Context)
+	}
+
+	return context.Background()
+}
+
+// ReqContext returns context for cli execution. Calling it for the first time
+// installs SIGTERM handler that will close returned context.
+// Not safe for concurrent execution.
+func ReqContext(cctx *cli.Context) context.Context {
+	tCtx := DaemonContext(cctx)
+
+	ctx, done := context.WithCancel(tCtx)
+	sigChan := make(chan os.Signal, 2)
+	go func() {
+		<-sigChan
+		done()
+	}()
+	signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)
+
+	return ctx
+}

cmd/lotus-bench/import.go

@@ -20,18 +20,17 @@ import (
 	"github.com/cockroachdb/pebble"
 	"github.com/cockroachdb/pebble/bloom"
 	"github.com/ipfs/go-cid"
-	metricsi "github.com/ipfs/go-metrics-interface"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/blockstore"
+	badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
 	lcli "github.com/filecoin-project/lotus/cli"
-	"github.com/filecoin-project/lotus/lib/blockstore"
-	badgerbs "github.com/filecoin-project/lotus/lib/blockstore/badger"
 	_ "github.com/filecoin-project/lotus/lib/sigs/bls"
 	_ "github.com/filecoin-project/lotus/lib/sigs/secp"
 	"github.com/filecoin-project/lotus/node/repo"
@@ -204,7 +203,7 @@ var importBenchCmd = &cli.Command{
 		case cctx.Bool("use-native-badger"):
 			log.Info("using native badger")
 			var opts badgerbs.Options
-			if opts, err = repo.BadgerBlockstoreOptions(repo.BlockstoreChain, tdir, false); err != nil {
+			if opts, err = repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, tdir, false); err != nil {
 				return err
 			}
 			opts.SyncWrites = false
@@ -229,21 +228,13 @@ var importBenchCmd = &cli.Command{
 		if ds != nil {
 			ds = measure.New("dsbench", ds)
 			defer ds.Close() //nolint:errcheck
-			bs = blockstore.NewBlockstore(ds)
+			bs = blockstore.FromDatastore(ds)
 		}
 
 		if c, ok := bs.(io.Closer); ok {
 			defer c.Close() //nolint:errcheck
 		}
 
-		ctx := metricsi.CtxScope(context.Background(), "lotus")
-		cacheOpts := blockstore.DefaultCacheOpts()
-		cacheOpts.HasBloomFilterSize = 0
-		bs, err = blockstore.CachedBlockstore(ctx, bs, cacheOpts)
-		if err != nil {
-			return err
-		}
-
 		var verifier ffiwrapper.Verifier = ffiwrapper.ProofVerifier
 		if cctx.IsSet("syscall-cache") {
 			scds, err := badger.NewDatastore(cctx.String("syscall-cache"), &badger.DefaultOptions)
@@ -267,6 +258,15 @@ var importBenchCmd = &cli.Command{
 
 		stm := stmgr.NewStateManager(cs)
 
+		var carFile *os.File
+		// open the CAR file if one is provided.
+		if path := cctx.String("car"); path != "" {
+			var err error
+			if carFile, err = os.Open(path); err != nil {
+				return xerrors.Errorf("failed to open provided CAR file: %w", err)
+			}
+		}
+
 		startTime := time.Now()
 
 		// register a gauge that reports how long since the measurable
@@ -308,18 +308,7 @@ var importBenchCmd = &cli.Command{
 			writeProfile("allocs")
 		}()
 
-		var carFile *os.File
-
-		// open the CAR file if one is provided.
-		if path := cctx.String("car"); path != "" {
-			var err error
-			if carFile, err = os.Open(path); err != nil {
-				return xerrors.Errorf("failed to open provided CAR file: %w", err)
-			}
-		}
-
 		var head *types.TipSet
-
 		// --- IMPORT ---
 		if !cctx.Bool("no-import") {
 			if cctx.Bool("global-profile") {

cmd/lotus-chainwatch/processor/miner.go

@@ -15,7 +15,7 @@ import (
 	"github.com/filecoin-project/go-state-types/big"
 
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/api/apibstore"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
 	"github.com/filecoin-project/lotus/chain/events/state"
@@ -202,7 +202,7 @@ func (p *Processor) processMiners(ctx context.Context, minerTips map[types.TipSe
 		log.Debugw("Processed Miners", "duration", time.Since(start).String())
 	}()
 
-	stor := store.ActorStore(ctx, apibstore.NewAPIBlockstore(p.node))
+	stor := store.ActorStore(ctx, blockstore.NewAPIBlockstore(p.node))
 
 	var out []minerActorInfo
 	// TODO add parallel calls if this becomes slow
@@ -649,7 +649,7 @@ func (p *Processor) getMinerStateAt(ctx context.Context, maddr address.Address,
 	if err != nil {
 		return nil, err
 	}
-	return miner.Load(store.ActorStore(ctx, apibstore.NewAPIBlockstore(p.node)), prevActor)
+	return miner.Load(store.ActorStore(ctx, blockstore.NewAPIBlockstore(p.node)), prevActor)
 }
 
 func (p *Processor) getMinerPreCommitChanges(ctx context.Context, m minerActorInfo) (*miner.PreCommitChanges, error) {

cmd/lotus-gateway/api.go

@@ -34,7 +34,7 @@ var (
 // gatewayDepsAPI defines the API methods that the GatewayAPI depends on
 // (to make it easy to mock for tests)
 type gatewayDepsAPI interface {
-	Version(context.Context) (api.Version, error)
+	Version(context.Context) (api.APIVersion, error)
 	ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error)
 	ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
 	ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error)
@@ -130,7 +130,7 @@ func (a *GatewayAPI) checkTimestamp(at time.Time) error {
 	return nil
 }
 
-func (a *GatewayAPI) Version(ctx context.Context) (api.Version, error) {
+func (a *GatewayAPI) Version(ctx context.Context) (api.APIVersion, error) {
 	return a.api.Version(ctx)
 }
 

cmd/lotus-gateway/main.go

@@ -77,7 +77,7 @@ var runCmd = &cli.Command{
 
 		// Register all metric views
 		if err := view.Register(
-			metrics.DefaultViews...,
+			metrics.ChainNodeViews...,
 		); err != nil {
 			log.Fatalf("Cannot register the view: %v", err)
 		}

cmd/lotus-seal-worker/main.go

@@ -31,6 +31,7 @@ import (
 	"github.com/filecoin-project/lotus/api/apistruct"
 	"github.com/filecoin-project/lotus/build"
 	lcli "github.com/filecoin-project/lotus/cli"
+	cliutil "github.com/filecoin-project/lotus/cli/util"
 	sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
 	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
@@ -49,7 +50,7 @@ const FlagWorkerRepo = "worker-repo"
 const FlagWorkerRepoDeprecation = "workerrepo"
 
 func main() {
-	build.RunningNodeType = build.NodeWorker
+	api.RunningNodeType = api.NodeWorker
 
 	lotuslog.SetupLogLevels()
 
@@ -183,7 +184,7 @@ var runCmd = &cli.Command{
 		var closer func()
 		var err error
 		for {
-			nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, lcli.StorageMinerUseHttp)
+			nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp)
 			if err == nil {
 				_, err = nodeApi.Version(ctx)
 				if err == nil {
@@ -210,8 +211,8 @@ var runCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
-		if v.APIVersion != build.MinerAPIVersion {
+		if v.APIVersion != api.MinerAPIVersion {
-			return xerrors.Errorf("lotus-miner API version doesn't match: expected: %s", api.Version{APIVersion: build.MinerAPIVersion})
+			return xerrors.Errorf("lotus-miner API version doesn't match: expected: %s", api.APIVersion{APIVersion: api.MinerAPIVersion})
 		}
 		log.Infof("Remote version %s", v)
 

cmd/lotus-seal-worker/rpc.go

@@ -8,7 +8,7 @@ import (
 	"github.com/mitchellh/go-homedir"
 	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/api"
 	sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
@@ -23,8 +23,8 @@ type worker struct {
 	disabled int64
 }
 
-func (w *worker) Version(context.Context) (build.Version, error) {
+func (w *worker) Version(context.Context) (api.Version, error) {
-	return build.WorkerAPIVersion, nil
+	return api.WorkerAPIVersion, nil
 }
 
 func (w *worker) StorageAddLocal(ctx context.Context, path string) error {

cmd/lotus-seal-worker/storage.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/docker/go-units"
 	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
@@ -46,6 +47,10 @@ var storageAttachCmd = &cli.Command{
 			Name:  "store",
 			Usage: "(for init) use path for long-term storage",
 		},
+		&cli.StringFlag{
+			Name:  "max-storage",
+			Usage: "(for init) limit storage space for sectors (expensive for very large paths!)",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		nodeApi, closer, err := lcli.GetWorkerAPI(cctx)
@@ -79,11 +84,20 @@ var storageAttachCmd = &cli.Command{
 				return err
 			}
 
+			var maxStor int64
+			if cctx.IsSet("max-storage") {
+				maxStor, err = units.RAMInBytes(cctx.String("max-storage"))
+				if err != nil {
+					return xerrors.Errorf("parsing max-storage: %w", err)
+				}
+			}
+
 			cfg := &stores.LocalStorageMeta{
 				ID:         stores.ID(uuid.New().String()),
 				Weight:     cctx.Uint64("weight"),
 				CanSeal:    cctx.Bool("seal"),
 				CanStore:   cctx.Bool("store"),
+				MaxStorage: uint64(maxStor),
 			}
 
 			if !(cfg.CanStore || cfg.CanSeal) {

cmd/lotus-seed/genesis.go

@@ -9,10 +9,10 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/lotus/journal"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/node/modules/testing"
 	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
@@ -37,6 +37,8 @@ var genesisCmd = &cli.Command{
 		genesisNewCmd,
 		genesisAddMinerCmd,
 		genesisAddMsigsCmd,
+		genesisSetVRKCmd,
+		genesisSetRemainderCmd,
 		genesisCarCmd,
 	},
 }
@@ -309,6 +311,200 @@ func parseMultisigCsv(csvf string) ([]GenAccountEntry, error) {
 	return entries, nil
 }
 
+var genesisSetVRKCmd = &cli.Command{
+	Name:  "set-vrk",
+	Usage: "Set the verified registry's root key",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "multisig",
+			Usage: "CSV file to parse the multisig that will be set as the root key",
+		},
+		&cli.StringFlag{
+			Name:  "account",
+			Usage: "pubkey address that will be set as the root key (must NOT be declared anywhere else, since it must be given ID 80)",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() != 1 {
+			return fmt.Errorf("must specify template file")
+		}
+
+		genf, err := homedir.Expand(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		csvf, err := homedir.Expand(cctx.Args().Get(1))
+		if err != nil {
+			return err
+		}
+
+		var template genesis.Template
+		b, err := ioutil.ReadFile(genf)
+		if err != nil {
+			return xerrors.Errorf("read genesis template: %w", err)
+		}
+
+		if err := json.Unmarshal(b, &template); err != nil {
+			return xerrors.Errorf("unmarshal genesis template: %w", err)
+		}
+
+		if cctx.IsSet("account") {
+			addr, err := address.NewFromString(cctx.String("account"))
+			if err != nil {
+				return err
+			}
+
+			am := genesis.AccountMeta{Owner: addr}
+
+			template.VerifregRootKey = genesis.Actor{
+				Type:    genesis.TAccount,
+				Balance: big.Zero(),
+				Meta:    am.ActorMeta(),
+			}
+		} else if cctx.IsSet("multisig") {
+
+			entries, err := parseMultisigCsv(csvf)
+			if err != nil {
+				return xerrors.Errorf("parsing multisig csv file: %w", err)
+			}
+
+			if len(entries) == 0 {
+				return xerrors.Errorf("no msig entries in csv file: %w", err)
+			}
+
+			e := entries[0]
+			if len(e.Addresses) != e.N {
+				return fmt.Errorf("entry had mismatch between 'N' and number of addresses")
+			}
+
+			msig := &genesis.MultisigMeta{
+				Signers:         e.Addresses,
+				Threshold:       e.M,
+				VestingDuration: monthsToBlocks(e.VestingMonths),
+				VestingStart:    0,
+			}
+
+			act := genesis.Actor{
+				Type:    genesis.TMultisig,
+				Balance: abi.TokenAmount(e.Amount),
+				Meta:    msig.ActorMeta(),
+			}
+
+			template.VerifregRootKey = act
+		} else {
+			return xerrors.Errorf("must include either --account or --multisig flag")
+		}
+
+		b, err = json.MarshalIndent(&template, "", "  ")
+		if err != nil {
+			return err
+		}
+
+		if err := ioutil.WriteFile(genf, b, 0644); err != nil {
+			return err
+		}
+		return nil
+	},
+}
+
+var genesisSetRemainderCmd = &cli.Command{
+	Name:  "set-remainder",
+	Usage: "Set the remainder actor",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "multisig",
+			Usage: "CSV file to parse the multisig that will be set as the remainder actor",
+		},
+		&cli.StringFlag{
+			Name:  "account",
+			Usage: "pubkey address that will be set as the remainder key (must NOT be declared anywhere else, since it must be given ID 90)",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() != 1 {
+			return fmt.Errorf("must specify template file")
+		}
+
+		genf, err := homedir.Expand(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		csvf, err := homedir.Expand(cctx.Args().Get(1))
+		if err != nil {
+			return err
+		}
+
+		var template genesis.Template
+		b, err := ioutil.ReadFile(genf)
+		if err != nil {
+			return xerrors.Errorf("read genesis template: %w", err)
+		}
+
+		if err := json.Unmarshal(b, &template); err != nil {
+			return xerrors.Errorf("unmarshal genesis template: %w", err)
+		}
+
+		if cctx.IsSet("account") {
+			addr, err := address.NewFromString(cctx.String("account"))
+			if err != nil {
+				return err
+			}
+
+			am := genesis.AccountMeta{Owner: addr}
+
+			template.RemainderAccount = genesis.Actor{
+				Type:    genesis.TAccount,
+				Balance: big.Zero(),
+				Meta:    am.ActorMeta(),
+			}
+		} else if cctx.IsSet("multisig") {
+
+			entries, err := parseMultisigCsv(csvf)
+			if err != nil {
+				return xerrors.Errorf("parsing multisig csv file: %w", err)
+			}
+
+			if len(entries) == 0 {
+				return xerrors.Errorf("no msig entries in csv file: %w", err)
+			}
+
+			e := entries[0]
+			if len(e.Addresses) != e.N {
+				return fmt.Errorf("entry had mismatch between 'N' and number of addresses")
+			}
+
+			msig := &genesis.MultisigMeta{
+				Signers:         e.Addresses,
+				Threshold:       e.M,
+				VestingDuration: monthsToBlocks(e.VestingMonths),
+				VestingStart:    0,
+			}
+
+			act := genesis.Actor{
+				Type:    genesis.TMultisig,
+				Balance: abi.TokenAmount(e.Amount),
+				Meta:    msig.ActorMeta(),
+			}
+
+			template.RemainderAccount = act
+		} else {
+			return xerrors.Errorf("must include either --account or --multisig flag")
+		}
+
+		b, err = json.MarshalIndent(&template, "", "  ")
+		if err != nil {
+			return err
+		}
+
+		if err := ioutil.WriteFile(genf, b, 0644); err != nil {
+			return err
+		}
+		return nil
+	},
+}
+
 var genesisCarCmd = &cli.Command{
 	Name:        "car",
 	Description: "write genesis car file",
@@ -327,7 +523,7 @@ var genesisCarCmd = &cli.Command{
 		}
 		ofile := c.String("out")
 		jrnl := journal.NilJournal()
-		bstor := blockstore.NewTemporarySync()
+		bstor := blockstore.NewMemorySync()
 		sbldr := vm.Syscalls(ffiwrapper.ProofVerifier)
 		_, err := testing.MakeGenesis(ofile, c.Args().First())(bstor, sbldr, jrnl)()
 		return err

cmd/lotus-shed/balances.go

@@ -175,7 +175,7 @@ var chainBalanceStateCmd = &cli.Command{
 
 		defer lkrepo.Close() //nolint:errcheck
 
-		bs, err := lkrepo.Blockstore(ctx, repo.BlockstoreChain)
+		bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore)
 		if err != nil {
 			return fmt.Errorf("failed to open blockstore: %w", err)
 		}
@@ -396,7 +396,7 @@ var chainPledgeCmd = &cli.Command{
 
 		defer lkrepo.Close() //nolint:errcheck
 
-		bs, err := lkrepo.Blockstore(ctx, repo.BlockstoreChain)
+		bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore)
 		if err != nil {
 			return xerrors.Errorf("failed to open blockstore: %w", err)
 		}

cmd/lotus-shed/consensus.go

@@ -36,7 +36,7 @@ type consensusItem struct {
 	targetTipset  *types.TipSet
 	headTipset    *types.TipSet
 	peerID        peer.ID
-	version       api.Version
+	version       api.APIVersion
 	api           api.FullNode
 }
 

cmd/lotus-shed/datastore.go

@@ -319,7 +319,7 @@ var datastoreRewriteCmd = &cli.Command{
 		)
 
 		// open the destination (to) store.
-		opts, err := repo.BadgerBlockstoreOptions(repo.BlockstoreChain, toPath, false)
+		opts, err := repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, toPath, false)
 		if err != nil {
 			return xerrors.Errorf("failed to get badger options: %w", err)
 		}
@@ -329,7 +329,7 @@ var datastoreRewriteCmd = &cli.Command{
 		}
 
 		// open the source (from) store.
-		opts, err = repo.BadgerBlockstoreOptions(repo.BlockstoreChain, fromPath, true)
+		opts, err = repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, fromPath, true)
 		if err != nil {
 			return xerrors.Errorf("failed to get badger options: %w", err)
 		}

cmd/lotus-shed/export.go

@@ -72,7 +72,7 @@ var exportChainCmd = &cli.Command{
 
 		defer fi.Close() //nolint:errcheck
 
-		bs, err := lr.Blockstore(ctx, repo.BlockstoreChain)
+		bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore)
 		if err != nil {
 			return fmt.Errorf("failed to open blockstore: %w", err)
 		}

cmd/lotus-shed/genesis-verify.go

@@ -17,6 +17,7 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/big"
 
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/account"
@@ -26,7 +27,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 type addrInfo struct {
@@ -50,7 +50,7 @@ var genesisVerifyCmd = &cli.Command{
 		if !cctx.Args().Present() {
 			return fmt.Errorf("must pass genesis car file")
 		}
-		bs := blockstore.NewBlockstore(datastore.NewMapDatastore())
+		bs := blockstore.FromDatastore(datastore.NewMapDatastore())
 
 		cs := store.NewChainStore(bs, bs, datastore.NewMapDatastore(), nil, nil)
 		defer cs.Close() //nolint:errcheck