cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #4992 from filecoin-project/feat/splitstore

Browse Source 
hot/cold blockstore segregation (aka. splitstore)
This commit is contained in:
Łukasz Magiera
2021-03-08 22:01:06 +01:00
committed by GitHub
parent b13046293f 51ed4c7b48
commit 6591af960f
27 changed files with 2499 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/builder.go
+28 -3
View File
@@ -586,14 +586,39 @@ func Repo(r repo.Repo) Option {
return err
}
var cfg *config.Chainstore
switch settings.nodeType {
case repo.FullNode:
cfgp, ok := c.(*config.FullNode)
if !ok {
return xerrors.Errorf("invalid config from repo, got: %T", c)
}
cfg = &cfgp.Chainstore
default:
cfg = &config.Chainstore{}
}
return Options(
Override(new(repo.LockedRepo), modules.LockedRepo(lr)), // module handles closing
Override(new(dtypes.MetadataDS), modules.Datastore),
Override(new(dtypes.UniversalBlockstore), modules.UniversalBlockstore),
Override(new(dtypes.ChainBlockstore), modules.ChainBlockstore),
Override(new(dtypes.StateBlockstore), modules.StateBlockstore),
Override(new(dtypes.ExposedBlockstore), From(new(dtypes.UniversalBlockstore))),
If(cfg.EnableSplitstore,
If(cfg.Splitstore.HotStoreType == "badger",
Override(new(dtypes.HotBlockstore), modules.BadgerHotBlockstore)),
Override(new(dtypes.SplitBlockstore), modules.SplitBlockstore(cfg)),
Override(new(dtypes.ChainBlockstore), modules.ChainSplitBlockstore),
Override(new(dtypes.StateBlockstore), modules.StateSplitBlockstore),
Override(new(dtypes.BaseBlockstore), From(new(dtypes.SplitBlockstore))),
Override(new(dtypes.ExposedBlockstore), From(new(dtypes.SplitBlockstore))),
),
If(!cfg.EnableSplitstore,
Override(new(dtypes.ChainBlockstore), modules.ChainFlatBlockstore),
Override(new(dtypes.StateBlockstore), modules.StateFlatBlockstore),
Override(new(dtypes.BaseBlockstore), From(new(dtypes.UniversalBlockstore))),
Override(new(dtypes.ExposedBlockstore), From(new(dtypes.UniversalBlockstore))),
),
If(os.Getenv("LOTUS_ENABLE_CHAINSTORE_FALLBACK") == "1",
Override(new(dtypes.ChainBlockstore), modules.FallbackChainBlockstore),
node/config/def.go
+25 -4
View File
@@ -20,10 +20,11 @@ type Common struct {
// FullNode is a full node config
type FullNode struct {
Common
Client Client
Metrics Metrics
Wallet Wallet
Fees FeeConfig
Client Client
Metrics Metrics
Wallet Wallet
Fees FeeConfig
Chainstore Chainstore
}
// // Common
@@ -120,6 +121,20 @@ type Pubsub struct {
RemoteTracer string
}
type Chainstore struct {
EnableSplitstore bool
Splitstore Splitstore
}
type Splitstore struct {
HotStoreType string
TrackingStoreType string
MarkSetType string
EnableFullCompaction bool
EnableGC bool // EXPERIMENTAL
Archival bool
}
// // Full Node
type Metrics struct {
@@ -185,6 +200,12 @@ func DefaultFullNode() *FullNode {
Client: Client{
SimultaneousTransfers: DefaultSimultaneousTransfers,
},
Chainstore: Chainstore{
EnableSplitstore: false,
Splitstore: Splitstore{
HotStoreType: "badger",
},
},
}
}
node/modules/blockstore.go
+74 -9
View File
@@ -3,19 +3,25 @@ package modules
import (
"context"
"io"
"os"
"path/filepath"
bstore "github.com/ipfs/go-ipfs-blockstore"
"go.uber.org/fx"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/blockstore"
badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
"github.com/filecoin-project/lotus/blockstore/splitstore"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/modules/helpers"
"github.com/filecoin-project/lotus/node/repo"
)
// UniversalBlockstore returns a single universal blockstore that stores both
// chain data and state data.
// chain data and state data. It can be backed by a blockstore directly
// (e.g. Badger), or by a Splitstore.
func UniversalBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (dtypes.UniversalBlockstore, error) {
bs, err := r.Blockstore(helpers.LifecycleCtx(mctx, lc), repo.UniversalBlockstore)
if err != nil {
@@ -31,17 +37,76 @@ func UniversalBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.Locked
return bs, err
}
// StateBlockstore is a hook to overlay caches for state objects, or in the
// future, to segregate the universal blockstore into different physical state
// and chain stores.
func StateBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtypes.UniversalBlockstore) (dtypes.StateBlockstore, error) {
func BadgerHotBlockstore(lc fx.Lifecycle, r repo.LockedRepo) (dtypes.HotBlockstore, error) {
path, err := r.SplitstorePath()
if err != nil {
return nil, err
}
path = filepath.Join(path, "hot.badger")
if err := os.MkdirAll(path, 0755); err != nil {
return nil, err
}
opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, r.Readonly())
if err != nil {
return nil, err
}
bs, err := badgerbs.Open(opts)
if err != nil {
return nil, err
}
lc.Append(fx.Hook{
OnStop: func(_ context.Context) error {
return bs.Close()
}})
return bs, nil
}
// ChainBlockstore is a hook to overlay caches for state objects, or in the
// future, to segregate the universal blockstore into different physical state
// and chain stores.
func ChainBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtypes.UniversalBlockstore) (dtypes.ChainBlockstore, error) {
func SplitBlockstore(cfg *config.Chainstore) func(lc fx.Lifecycle, r repo.LockedRepo, ds dtypes.MetadataDS, cold dtypes.UniversalBlockstore, hot dtypes.HotBlockstore) (dtypes.SplitBlockstore, error) {
return func(lc fx.Lifecycle, r repo.LockedRepo, ds dtypes.MetadataDS, cold dtypes.UniversalBlockstore, hot dtypes.HotBlockstore) (dtypes.SplitBlockstore, error) {
path, err := r.SplitstorePath()
if err != nil {
return nil, err
}
cfg := &splitstore.Config{
TrackingStoreType: cfg.Splitstore.TrackingStoreType,
MarkSetType: cfg.Splitstore.MarkSetType,
EnableFullCompaction: cfg.Splitstore.EnableFullCompaction,
EnableGC: cfg.Splitstore.EnableGC,
Archival: cfg.Splitstore.Archival,
}
ss, err := splitstore.Open(path, ds, hot, cold, cfg)
if err != nil {
return nil, err
}
lc.Append(fx.Hook{
OnStop: func(context.Context) error {
return ss.Close()
},
})
return ss, err
}
}
func StateFlatBlockstore(_ fx.Lifecycle, _ helpers.MetricsCtx, bs dtypes.UniversalBlockstore) (dtypes.StateBlockstore, error) {
return bs, nil
}
func StateSplitBlockstore(_ fx.Lifecycle, _ helpers.MetricsCtx, bs dtypes.SplitBlockstore) (dtypes.StateBlockstore, error) {
return bs, nil
}
func ChainFlatBlockstore(_ fx.Lifecycle, _ helpers.MetricsCtx, bs dtypes.UniversalBlockstore) (dtypes.ChainBlockstore, error) {
return bs, nil
}
func ChainSplitBlockstore(_ fx.Lifecycle, _ helpers.MetricsCtx, bs dtypes.SplitBlockstore) (dtypes.ChainBlockstore, error) {
return bs, nil
}
node/modules/chain.go
+14 -1
View File
@@ -14,6 +14,7 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/blockstore/splitstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain"
"github.com/filecoin-project/lotus/chain/beacon"
@@ -72,14 +73,26 @@ func MessagePool(lc fx.Lifecycle, sm *stmgr.StateManager, ps *pubsub.PubSub, ds
return mp, nil
}
func ChainStore(lc fx.Lifecycle, cbs dtypes.ChainBlockstore, sbs dtypes.StateBlockstore, ds dtypes.MetadataDS, syscalls vm.SyscallBuilder, j journal.Journal) *store.ChainStore {
func ChainStore(lc fx.Lifecycle, cbs dtypes.ChainBlockstore, sbs dtypes.StateBlockstore, ds dtypes.MetadataDS, basebs dtypes.BaseBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) *store.ChainStore {
chain := store.NewChainStore(cbs, sbs, ds, syscalls, j)
if err := chain.Load(); err != nil {
log.Warnf("loading chain state from disk: %s", err)
}
var startHook func(context.Context) error
if ss, ok := basebs.(*splitstore.SplitStore); ok {
startHook = func(_ context.Context) error {
err := ss.Start(chain)
if err != nil {
err = xerrors.Errorf("error starting splitstore: %w", err)
}
return err
}
}
lc.Append(fx.Hook{
OnStart: startHook,
OnStop: func(_ context.Context) error {
return chain.Close()
},
node/modules/dtypes/storage.go
+9
View File
@@ -27,6 +27,15 @@ type (
// UniversalBlockstore is the cold blockstore.
UniversalBlockstore blockstore.Blockstore
// HotBlockstore is the Hot blockstore abstraction for the splitstore
HotBlockstore blockstore.Blockstore
// SplitBlockstore is the hot/cold blockstore that sits on top of the ColdBlockstore.
SplitBlockstore blockstore.Blockstore
// BaseBlockstore is something, coz DI
BaseBlockstore blockstore.Blockstore
// ChainBlockstore is a blockstore to store chain data (tipsets, blocks,
// messages). It is physically backed by the BareMonolithBlockstore, but it
// has a cache on top that is specially tuned for chain data access
node/repo/interface.go
+1
View File
@@ -23,6 +23,7 @@ const (
// well as state. In the future, they may get segregated into different
// domains.
UniversalBlockstore = BlockstoreDomain("universal")
HotBlockstore = BlockstoreDomain("hot")
)
var (