From d2e2322fd242e73e11a883117d639dd490e03377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Sun, 1 Nov 2020 13:01:26 +0000 Subject: [PATCH] make the lotus node use the new native badger blockstore. --- node/modules/chain.go | 4 ++-- node/repo/fsrepo.go | 48 ++++++++++++++++++++++++++++++++++++++++++ node/repo/fsrepo_ds.go | 12 ----------- node/repo/interface.go | 19 +++++++++++++++++ node/repo/memrepo.go | 27 ++++++++++++++++-------- 5 files changed, 87 insertions(+), 23 deletions(-) diff --git a/node/modules/chain.go b/node/modules/chain.go index d1414b307..aec332217 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -77,12 +77,12 @@ func MessagePool(lc fx.Lifecycle, sm *stmgr.StateManager, ps *pubsub.PubSub, ds } func ChainBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (dtypes.ChainBlockstore, error) { - blocks, err := r.Datastore("/chain") + bs, err := r.Blockstore(repo.BlockstoreChain) if err != nil { return nil, err } - bs := blockstore.NewBlockstore(blocks) + // TODO potentially replace this cached blockstore by a CBOR cache. cbs, err := blockstore.CachedBlockstore(helpers.LifecycleCtx(mctx, lc), bs, blockstore.DefaultCacheOpts()) if err != nil { return nil, err diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index c1b6b5233..2883e57d6 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -14,6 +14,7 @@ import ( "github.com/BurntSushi/toml" "github.com/ipfs/go-datastore" fslock "github.com/ipfs/go-fs-lock" + blockstore "github.com/ipfs/go-ipfs-blockstore" logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-base32" @@ -22,6 +23,7 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" "github.com/filecoin-project/lotus/extern/sector-storage/stores" + badgerbs "github.com/filecoin-project/lotus/lib/blockstore/badger" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/config" @@ -284,6 +286,52 @@ func (fsr *fsLockedRepo) Close() error { return err } +func (fsr *fsLockedRepo) Blockstore(domain BlockstoreDomain) (blockstore.Blockstore, error) { + if domain != BlockstoreChain { + return nil, ErrInvalidBlockstoreDomain + } + + path := fsr.join(filepath.Join(fsDatastore, "chain")) + opts := badgerbs.DefaultOptions(path) + + // Due to legacy usage of blockstore.Blockstore, over a datastore, all + // blocks are prefixed with this namespace. In the future, this can go away, + // in order to shorten keys, but it'll require a migration. + opts.Prefix = "/blocks/" + + // Blockstore values are immutable; therefore we do not expect any + // conflicts to emerge. + opts.DetectConflicts = false + + // This is to optimize the database on close so it can be opened + // read-only and efficiently queried. We don't do that and hanging on + // stop isn't nice. + opts.CompactL0OnClose = false + + // The alternative is "crash on start and tell the user to fix it". This + // will truncate corrupt and unsynced data, which we don't guarantee to + // persist anyways. + opts.Truncate = true + + // We mmap the index into memory, and access values from disk. + // Ideally the table loading mode would be settable by LSM level. + opts.ValueLogLoadingMode = badgerbs.FileIO + opts.TableLoadingMode = badgerbs.MemoryMap + + // Embed only values < 128 bytes in the LSM; larger values in value logs. + opts.ValueThreshold = 128 + + // Reduce this from 64MiB to 16MiB. That means badger will hold on to + // 20MiB by default instead of 80MiB. This does not appear to have a + // significant performance hit. + opts.MaxTableSize = 16 << 20 + + // NOTE: The chain blockstore doesn't require any GC (blocks are never + // deleted). This will change if we move to a tiered blockstore. + + return badgerbs.Open(opts) +} + // join joins path elements with fsr.path func (fsr *fsLockedRepo) join(paths ...string) string { return filepath.Join(append([]string{fsr.path}, paths...)...) diff --git a/node/repo/fsrepo_ds.go b/node/repo/fsrepo_ds.go index aa91d2514..307aa3a76 100644 --- a/node/repo/fsrepo_ds.go +++ b/node/repo/fsrepo_ds.go @@ -17,7 +17,6 @@ import ( type dsCtor func(path string, readonly bool) (datastore.Batching, error) var fsDatastores = map[string]dsCtor{ - "chain": chainBadgerDs, "metadata": levelDs, // Those need to be fast for large writes... but also need a really good GC :c @@ -26,17 +25,6 @@ var fsDatastores = map[string]dsCtor{ "client": badgerDs, // client specific } -func chainBadgerDs(path string, readonly bool) (datastore.Batching, error) { - opts := badger.DefaultOptions - opts.GcInterval = 0 // disable GC for chain datastore - opts.ReadOnly = readonly - - opts.Options = dgbadger.DefaultOptions("").WithTruncate(true). - WithValueThreshold(1 << 10) - - return badger.NewDatastore(path, &opts) -} - func badgerDs(path string, readonly bool) (datastore.Batching, error) { opts := badger.DefaultOptions opts.ReadOnly = readonly diff --git a/node/repo/interface.go b/node/repo/interface.go index c25bcb534..8dab9260f 100644 --- a/node/repo/interface.go +++ b/node/repo/interface.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/ipfs/go-datastore" + blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/multiformats/go-multiaddr" "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" @@ -12,11 +13,26 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +// BlockstoreDomain represents the domain of a blockstore. +type BlockstoreDomain string + +const ( + // BlockstoreChain represents the blockstore domain for chain data. + // Right now, this includes chain objects (tipsets, blocks, messages), as + // well as state. In the future, they may get segretated into different + // domains. + BlockstoreChain = BlockstoreDomain("chain") +) + var ( ErrNoAPIEndpoint = errors.New("API not running (no endpoint)") ErrNoAPIToken = errors.New("API token not set") ErrRepoAlreadyLocked = errors.New("repo is already locked (lotus daemon already running)") ErrClosedRepo = errors.New("repo is no longer open") + + // ErrInvalidBlockstoreDomain is returned by LockedRepo#Blockstore() when + // an unrecognized domain is requested. + ErrInvalidBlockstoreDomain = errors.New("invalid blockstore domain") ) type Repo interface { @@ -37,6 +53,9 @@ type LockedRepo interface { // Returns datastore defined in this repo. Datastore(namespace string) (datastore.Batching, error) + // Blockstore returns an IPLD blockstore for the requested domain. + Blockstore(domain BlockstoreDomain) (blockstore.Blockstore, error) + // Returns config in this repo Config() (interface{}, error) SetConfig(func(interface{})) error diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index 34e3637eb..4ef5e1203 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -11,13 +11,14 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" dssync "github.com/ipfs/go-datastore/sync" + "github.com/ipfs/go-ipfs-blockstore" "github.com/multiformats/go-multiaddr" "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" - "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" "github.com/filecoin-project/lotus/extern/sector-storage/stores" + lblockstore "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/node/config" ) @@ -31,8 +32,9 @@ type MemRepo struct { repoLock chan struct{} token *byte - datastore datastore.Datastore - keystore map[string]types.KeyInfo + datastore datastore.Datastore + keystore map[string]types.KeyInfo + blockstore blockstore.Blockstore // given a repo type, produce the default config configF func(t RepoType) interface{} @@ -158,11 +160,11 @@ func NewMemory(opts *MemRepoOptions) *MemRepo { } return &MemRepo{ - repoLock: make(chan struct{}, 1), - - datastore: opts.Ds, - configF: opts.ConfigF, - keystore: opts.KeyStore, + repoLock: make(chan struct{}, 1), + blockstore: lblockstore.NewTemporarySync(), + datastore: opts.Ds, + configF: opts.ConfigF, + keystore: opts.KeyStore, } } @@ -243,6 +245,13 @@ func (lmem *lockedMemRepo) Datastore(ns string) (datastore.Batching, error) { return namespace.Wrap(lmem.mem.datastore, datastore.NewKey(ns)), nil } +func (lmem *lockedMemRepo) Blockstore(domain BlockstoreDomain) (blockstore.Blockstore, error) { + if domain != BlockstoreChain { + return nil, ErrInvalidBlockstoreDomain + } + return lmem.mem.blockstore, nil +} + func (lmem *lockedMemRepo) ListDatastores(ns string) ([]int64, error) { return nil, nil }