segregate chain and state blockstores.

This paves the way for better object lifetime management.

Concretely, it makes it possible to:
- have different stores backing chain and state data.
- having the same datastore library, but using different parameters.
- attach different caching layers/policies to each class of data, e.g.
  sizing caches differently.
- specifying different retention policies for chain and state data.

This separation is important because:
- access patterns/frequency of chain and state data are different.
- state is derivable from chain, so one could never expunge the chain
  store, and only retain state objects reachable from the last finality
  in the state store.

node/repo/blockstore_opts.go

@@ -5,10 +5,6 @@ import badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
 // BadgerBlockstoreOptions returns the badger options to apply for the provided
 // domain.
 func BadgerBlockstoreOptions(domain BlockstoreDomain, path string, readonly bool) (badgerbs.Options, error) {
-	if domain != BlockstoreChain {
-		return badgerbs.Options{}, ErrInvalidBlockstoreDomain
-	}
-
 	opts := badgerbs.DefaultOptions(path)
 
 	// Due to legacy usage of blockstore.Blockstore, over a datastore, all

node/repo/fsrepo.go

@@ -13,7 +13,7 @@ import (
 	"sync"
 
 	"github.com/BurntSushi/toml"
-	"github.com/filecoin-project/lotus/blockstore"
+
 	"github.com/ipfs/go-datastore"
 	fslock "github.com/ipfs/go-fs-lock"
 	logging "github.com/ipfs/go-log/v2"
@@ -22,7 +22,7 @@ import (
 	"github.com/multiformats/go-multiaddr"
 	"golang.org/x/xerrors"
 
-	lblockstore "github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/blockstore"
 	badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
 	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
@@ -264,11 +264,18 @@ type fsLockedRepo struct {
 	bs     blockstore.Blockstore
 	bsErr  error
 	bsOnce sync.Once
+	ssPath string
+	ssErr  error
+	ssOnce sync.Once
 
 	storageLk sync.Mutex
 	configLk  sync.Mutex
 }
 
+func (fsr *fsLockedRepo) Readonly() bool {
+	return fsr.readonly
+}
+
 func (fsr *fsLockedRepo) Path() string {
 	return fsr.path
 }
@@ -301,7 +308,7 @@ func (fsr *fsLockedRepo) Close() error {
 
 // Blockstore returns a blockstore for the provided data domain.
 func (fsr *fsLockedRepo) Blockstore(ctx context.Context, domain BlockstoreDomain) (blockstore.Blockstore, error) {
-	if domain != BlockstoreChain {
+	if domain != UniversalBlockstore {
 		return nil, ErrInvalidBlockstoreDomain
 	}
 
@@ -325,12 +332,27 @@ func (fsr *fsLockedRepo) Blockstore(ctx context.Context, domain BlockstoreDomain
 			fsr.bsErr = err
 			return
 		}
-		fsr.bs = lblockstore.WrapIDStore(bs)
+		fsr.bs = blockstore.WrapIDStore(bs)
 	})
 
 	return fsr.bs, fsr.bsErr
 }
 
+func (fsr *fsLockedRepo) SplitstorePath() (string, error) {
+	fsr.ssOnce.Do(func() {
+		path := fsr.join(filepath.Join(fsDatastore, "splitstore"))
+
+		if err := os.MkdirAll(path, 0755); err != nil {
+			fsr.ssErr = err
+			return
+		}
+
+		fsr.ssPath = path
+	})
+
+	return fsr.ssPath, fsr.ssErr
+}
+
 // join joins path elements with fsr.path
 func (fsr *fsLockedRepo) join(paths ...string) string {
 	return filepath.Join(append([]string{fsr.path}, paths...)...)

node/repo/importmgr/mgr.go

@@ -16,7 +16,7 @@ type Mgr struct {
 	mds *multistore.MultiStore
 	ds  datastore.Batching
 
-	Blockstore blockstore.Blockstore
+	Blockstore blockstore.BasicBlockstore
 }
 
 type Label string

node/repo/interface.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"errors"
 
-	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/ipfs/go-datastore"
 	"github.com/multiformats/go-multiaddr"
 
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 
@@ -18,11 +18,11 @@ import (
 type BlockstoreDomain string
 
 const (
-	// BlockstoreChain represents the blockstore domain for chain data.
+	// UniversalBlockstore represents the blockstore domain for all data.
 	// Right now, this includes chain objects (tipsets, blocks, messages), as
 	// well as state. In the future, they may get segregated into different
 	// domains.
-	BlockstoreChain = BlockstoreDomain("chain")
+	UniversalBlockstore = BlockstoreDomain("universal")
 )
 
 var (
@@ -63,6 +63,9 @@ type LockedRepo interface {
 	// the lifecycle.
 	Blockstore(ctx context.Context, domain BlockstoreDomain) (blockstore.Blockstore, error)
 
+	// SplitstorePath returns the path for the SplitStore
+	SplitstorePath() (string, error)
+
 	// Returns config in this repo
 	Config() (interface{}, error)
 	SetConfig(func(interface{})) error
@@ -84,4 +87,7 @@ type LockedRepo interface {
 
 	// Path returns absolute path of the repo
 	Path() string
+
+	// Readonly returns true if the repo is readonly
+	Readonly() bool
 }

node/repo/memrepo.go

@@ -201,6 +201,10 @@ func (mem *MemRepo) Lock(t RepoType) (LockedRepo, error) {
 	}, nil
 }
 
+func (lmem *lockedMemRepo) Readonly() bool {
+	return false
+}
+
 func (lmem *lockedMemRepo) checkToken() error {
 	lmem.RLock()
 	defer lmem.RUnlock()
@@ -246,12 +250,16 @@ func (lmem *lockedMemRepo) Datastore(_ context.Context, ns string) (datastore.Ba
 }
 
 func (lmem *lockedMemRepo) Blockstore(ctx context.Context, domain BlockstoreDomain) (blockstore.Blockstore, error) {
-	if domain != BlockstoreChain {
+	if domain != UniversalBlockstore {
 		return nil, ErrInvalidBlockstoreDomain
 	}
 	return lmem.mem.blockstore, nil
 }
 
+func (lmem *lockedMemRepo) SplitstorePath() (string, error) {
+	return ioutil.TempDir("", "splitstore.*")
+}
+
 func (lmem *lockedMemRepo) ListDatastores(ns string) ([]int64, error) {
 	return nil, nil
 }

node/repo/retrievalstoremgr/retrievalstoremgr.go

@@ -73,13 +73,13 @@ func (mrs *multiStoreRetrievalStore) DAGService() ipldformat.DAGService {
 
 // BlockstoreRetrievalStoreManager manages a single blockstore as if it were multiple stores
 type BlockstoreRetrievalStoreManager struct {
-	bs blockstore.Blockstore
+	bs blockstore.BasicBlockstore
 }
 
 var _ RetrievalStoreManager = &BlockstoreRetrievalStoreManager{}
 
 // NewBlockstoreRetrievalStoreManager returns a new blockstore based RetrievalStoreManager
-func NewBlockstoreRetrievalStoreManager(bs blockstore.Blockstore) RetrievalStoreManager {
+func NewBlockstoreRetrievalStoreManager(bs blockstore.BasicBlockstore) RetrievalStoreManager {
 	return &BlockstoreRetrievalStoreManager{
 		bs: bs,
 	}