Merge pull request #5755 from filecoin-project/feat/kvlog

Metadata datastore log

node/builder.go

@@ -508,6 +508,7 @@ func ConfigCommon(cfg *config.Common) Option {
 		Override(AddrsFactoryKey, lp2p.AddrsFactory(
 			cfg.Libp2p.AnnounceAddresses,
 			cfg.Libp2p.NoAnnounceAddresses)),
+		Override(new(dtypes.MetadataDS), modules.Datastore(cfg.Backup.DisableMetadataLog)),
 	)
 }
 
@@ -601,7 +602,6 @@ func Repo(r repo.Repo) Option {
 		return Options(
 			Override(new(repo.LockedRepo), modules.LockedRepo(lr)), // module handles closing
 
-			Override(new(dtypes.MetadataDS), modules.Datastore),
 			Override(new(dtypes.UniversalBlockstore), modules.UniversalBlockstore),
 
 			If(cfg.EnableSplitstore,

node/config/def.go

@@ -13,6 +13,7 @@ import (
 // Common is common config between full node and miner
 type Common struct {
 	API    API
+	Backup Backup
 	Libp2p Libp2p
 	Pubsub Pubsub
 }
@@ -29,6 +30,10 @@ type FullNode struct {
 
 // // Common
 
+type Backup struct {
+	DisableMetadataLog bool
+}
+
 // StorageMiner is a miner config
 type StorageMiner struct {
 	Common

node/modules/storage.go

@@ -2,8 +2,10 @@ package modules
 
 import (
 	"context"
+	"path/filepath"
 
 	"go.uber.org/fx"
+	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/lib/backupds"
@@ -28,12 +30,30 @@ func KeyStore(lr repo.LockedRepo) (types.KeyStore, error) {
 	return lr.KeyStore()
 }
 
-func Datastore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (dtypes.MetadataDS, error) {
-	ctx := helpers.LifecycleCtx(mctx, lc)
-	mds, err := r.Datastore(ctx, "/metadata")
-	if err != nil {
-		return nil, err
-	}
+func Datastore(disableLog bool) func(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (dtypes.MetadataDS, error) {
+	return func(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (dtypes.MetadataDS, error) {
+		ctx := helpers.LifecycleCtx(mctx, lc)
+		mds, err := r.Datastore(ctx, "/metadata")
+		if err != nil {
+			return nil, err
+		}
 
-	return backupds.Wrap(mds), nil
+		var logdir string
+		if !disableLog {
+			logdir = filepath.Join(r.Path(), "kvlog/metadata")
+		}
+
+		bds, err := backupds.Wrap(mds, logdir)
+		if err != nil {
+			return nil, xerrors.Errorf("opening backupds: %w", err)
+		}
+
+		lc.Append(fx.Hook{
+			OnStop: func(_ context.Context) error {
+				return bds.CloseLog()
+			},
+		})
+
+		return bds, nil
+	}
 }