splitstore constructor

chain/store/splitstore/liveset.go

@@ -1,11 +1,16 @@
 package splitstore
 
 import (
+	"fmt"
+	"os"
+
 	"github.com/bmatsuo/lmdb-go/lmdb"
 
 	cid "github.com/ipfs/go-cid"
 )
 
+var LiveSetMapSize int64 = 1 << 34 // 16G; TODO this may be a little too big, we should figure out how to gradually grow the map.
+
 type LiveSet interface {
 	Mark(cid.Cid) error
 	Has(cid.Cid) (bool, error)
@@ -19,6 +24,39 @@ type liveSet struct {
 
 var markBytes = []byte{}
 
+func NewLiveSetEnv(path string) (*lmdb.Env, error) {
+	env, err := lmdb.NewEnv()
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize LDMB env: %w", err)
+	}
+	if err = env.SetMapSize(LiveSetMapSize); err != nil {
+		return nil, fmt.Errorf("failed to set LMDB map size: %w", err)
+	}
+	if err = env.SetMaxDBs(2); err != nil {
+		return nil, fmt.Errorf("failed to set LMDB max dbs: %w", err)
+	}
+	if err = env.SetMaxReaders(1); err != nil {
+		return nil, fmt.Errorf("failed to set LMDB max readers: %w", err)
+	}
+
+	if st, err := os.Stat(path); os.IsNotExist(err) {
+		if err := os.MkdirAll(path, 0777); err != nil {
+			return nil, fmt.Errorf("failed to create LMDB data directory at %s: %w", path, err)
+		}
+	} else if err != nil {
+		return nil, fmt.Errorf("failed to stat LMDB data dir: %w", err)
+	} else if !st.IsDir() {
+		return nil, fmt.Errorf("LMDB path is not a directory %s", path)
+	}
+	err = env.Open(path, lmdb.NoSync|lmdb.WriteMap|lmdb.MapAsync|lmdb.NoReadahead, 0777)
+	if err != nil {
+		env.Close() //nolint:errcheck
+		return nil, fmt.Errorf("error opening LMDB database: %w", err)
+	}
+
+	return env, nil
+}
+
 func NewLiveSet(env *lmdb.Env, name string) (LiveSet, error) {
 	var db lmdb.DBI
 	err := env.Update(func(txn *lmdb.Txn) (err error) {

chain/store/splitstore/splitstore.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
+	"path/filepath"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -15,6 +16,7 @@ import (
 	dstore "github.com/ipfs/go-datastore"
 	logging "github.com/ipfs/go-log/v2"
 
+	"github.com/filecoin-project/go-bs-lmdb"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/store"
@@ -36,12 +38,10 @@ type SplitStore struct {
 	mx    sync.Mutex
 	curTs *types.TipSet
 
-	cs *store.ChainStore
+	cs    *store.ChainStore
-	ds dstore.Datastore
+	ds    dstore.Datastore
-
+	hot   bstore.Blockstore
-	hot  bstore.Blockstore
+	cold  bstore.Blockstore
-	cold bstore.Blockstore
-
 	snoop TrackingStore
 
 	env *lmdb.Env
@@ -49,6 +49,43 @@ type SplitStore struct {
 
 var _ bstore.Blockstore = (*SplitStore)(nil)
 
+// NewSplitStore creates a new SplitStore instance, given a path for the hotstore dbs and a cold
+// blockstore. The SplitStore must be attached to the ChainStore with Start in order to trigger
+// compaction.
+func NewSplitStore(path string, ds dstore.Datastore, cold bstore.Blockstore) (*SplitStore, error) {
+	// the hot store
+	hot, err := lmdbbs.Open(filepath.Join(path, "hot.db"))
+	if err != nil {
+		return nil, err
+	}
+
+	// the tracking store
+	snoop, err := NewTrackingStore(filepath.Join(path, "snoop.db"))
+	if err != nil {
+		hot.Close() //nolint:errcheck
+		return nil, err
+	}
+
+	// the liveset env
+	env, err := NewLiveSetEnv(filepath.Join(path, "sweep.db"))
+	if err != nil {
+		hot.Close()   //nolint:errcheck
+		snoop.Close() //nolint:errcheck
+		return nil, err
+	}
+
+	// and now we can make a SplitStore
+	ss := &SplitStore{
+		ds:    ds,
+		hot:   hot,
+		cold:  cold,
+		snoop: snoop,
+		env:   env,
+	}
+
+	return ss, nil
+}
+
 // Blockstore interface
 func (s *SplitStore) DeleteBlock(cid cid.Cid) error {
 	// afaict we don't seem to be using this method, so it's not implemented