lotus/cmd/lotus-shed/splitstore.go
2023-02-15 11:09:24 -05:00

406 lines
9.3 KiB
Go

package main
import (
"bufio"
"context"
"fmt"
"io"
"os"
"path/filepath"
"runtime"
"github.com/dgraph-io/badger/v2"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/query"
"github.com/urfave/cli/v2"
"go.uber.org/multierr"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
"golang.org/x/xerrors"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/repo"
)
var splitstoreCmd = &cli.Command{
Name: "splitstore",
Description: "splitstore utilities",
Subcommands: []*cli.Command{
splitstoreRollbackCmd,
splitstoreClearCmd,
splitstoreCheckCmd,
splitstoreInfoCmd,
},
}
var splitstoreRollbackCmd = &cli.Command{
Name: "rollback",
Description: "rollbacks a splitstore installation",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "repo",
Value: "~/.lotus",
},
&cli.BoolFlag{
Name: "gc-coldstore",
Usage: "compact and garbage collect the coldstore after copying the hotstore",
},
&cli.BoolFlag{
Name: "rewrite-config",
Usage: "rewrite the lotus configuration to disable splitstore",
},
},
Action: func(cctx *cli.Context) error {
r, err := repo.NewFS(cctx.String("repo"))
if err != nil {
return xerrors.Errorf("error opening fs repo: %w", err)
}
exists, err := r.Exists()
if err != nil {
return err
}
if !exists {
return xerrors.Errorf("lotus repo doesn't exist")
}
lr, err := r.Lock(repo.FullNode)
if err != nil {
return xerrors.Errorf("error locking repo: %w", err)
}
defer lr.Close() //nolint:errcheck
cfg, err := lr.Config()
if err != nil {
return xerrors.Errorf("error getting config: %w", err)
}
fncfg, ok := cfg.(*config.FullNode)
if !ok {
return xerrors.Errorf("wrong config type: %T", cfg)
}
if !fncfg.Chainstore.EnableSplitstore {
return xerrors.Errorf("splitstore is not enabled")
}
fmt.Println("copying hotstore to coldstore...")
err = copyHotstoreToColdstore(lr, cctx.Bool("gc-coldstore"))
if err != nil {
return xerrors.Errorf("error copying hotstore to coldstore: %w", err)
}
fmt.Println("clearing splitstore directory...")
err = clearSplitstoreDir(lr)
if err != nil {
return xerrors.Errorf("error clearing splitstore directory: %w", err)
}
fmt.Println("deleting splitstore directory...")
err = deleteSplitstoreDir(lr)
if err != nil {
log.Warnf("error deleting splitstore directory: %s", err)
}
fmt.Println("deleting splitstore keys from metadata datastore...")
err = deleteSplitstoreKeys(lr)
if err != nil {
return xerrors.Errorf("error deleting splitstore keys: %w", err)
}
if cctx.Bool("rewrite-config") {
fmt.Println("disabling splitstore in config...")
err = lr.SetConfig(func(cfg interface{}) {
cfg.(*config.FullNode).Chainstore.EnableSplitstore = false
})
if err != nil {
return xerrors.Errorf("error disabling splitstore in config: %w", err)
}
}
fmt.Println("splitstore has been rolled back.")
return nil
},
}
var splitstoreClearCmd = &cli.Command{
Name: "clear",
Description: "clears a splitstore installation for restart from snapshot",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "repo",
Value: "~/.lotus",
},
&cli.BoolFlag{
Name: "keys-only",
Usage: "only delete splitstore keys",
},
},
Action: func(cctx *cli.Context) error {
r, err := repo.NewFS(cctx.String("repo"))
if err != nil {
return xerrors.Errorf("error opening fs repo: %w", err)
}
exists, err := r.Exists()
if err != nil {
return err
}
if !exists {
return xerrors.Errorf("lotus repo doesn't exist")
}
lr, err := r.Lock(repo.FullNode)
if err != nil {
return xerrors.Errorf("error locking repo: %w", err)
}
defer lr.Close() //nolint:errcheck
cfg, err := lr.Config()
if err != nil {
return xerrors.Errorf("error getting config: %w", err)
}
fncfg, ok := cfg.(*config.FullNode)
if !ok {
return xerrors.Errorf("wrong config type: %T", cfg)
}
if !fncfg.Chainstore.EnableSplitstore {
return xerrors.Errorf("splitstore is not enabled")
}
if !cctx.Bool("keys-only") {
fmt.Println("clearing splitstore directory...")
err = clearSplitstoreDir(lr)
if err != nil {
return xerrors.Errorf("error clearing splitstore directory: %w", err)
}
}
fmt.Println("deleting splitstore keys from metadata datastore...")
err = deleteSplitstoreKeys(lr)
if err != nil {
return xerrors.Errorf("error deleting splitstore keys: %w", err)
}
return nil
},
}
func copyHotstoreToColdstore(lr repo.LockedRepo, gcColdstore bool) error {
repoPath := lr.Path()
dataPath := filepath.Join(repoPath, "datastore")
coldPath := filepath.Join(dataPath, "chain")
hotPath := filepath.Join(dataPath, "splitstore", "hot.badger")
blog := &badgerLogger{
SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(),
skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(),
}
coldOpts, err := repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, coldPath, false)
if err != nil {
return xerrors.Errorf("error getting coldstore badger options: %w", err)
}
coldOpts.SyncWrites = false
coldOpts.Logger = blog
hotOpts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, hotPath, true)
if err != nil {
return xerrors.Errorf("error getting hotstore badger options: %w", err)
}
hotOpts.Logger = blog
cold, err := badger.Open(coldOpts.Options)
if err != nil {
return xerrors.Errorf("error opening coldstore: %w", err)
}
defer cold.Close() //nolint
hot, err := badger.Open(hotOpts.Options)
if err != nil {
return xerrors.Errorf("error opening hotstore: %w", err)
}
defer hot.Close() //nolint
rd, wr := io.Pipe()
g := new(errgroup.Group)
g.Go(func() error {
bwr := bufio.NewWriterSize(wr, 64<<20)
_, err := hot.Backup(bwr, 0)
if err != nil {
_ = wr.CloseWithError(err)
return err
}
err = bwr.Flush()
if err != nil {
_ = wr.CloseWithError(err)
return err
}
return wr.Close()
})
g.Go(func() error {
err := cold.Load(rd, 1024)
if err != nil {
return err
}
return cold.Sync()
})
err = g.Wait()
if err != nil {
return err
}
// compact + gc the coldstore if so requested
if gcColdstore {
fmt.Println("compacting coldstore...")
nworkers := runtime.NumCPU()
if nworkers < 2 {
nworkers = 2
}
err = cold.Flatten(nworkers)
if err != nil {
return xerrors.Errorf("error compacting coldstore: %w", err)
}
fmt.Println("garbage collecting coldstore...")
for err == nil {
err = cold.RunValueLogGC(0.0625)
}
if err != badger.ErrNoRewrite {
return xerrors.Errorf("error garbage collecting coldstore: %w", err)
}
}
return nil
}
func deleteSplitstoreDir(lr repo.LockedRepo) error {
path, err := lr.SplitstorePath()
if err != nil {
return xerrors.Errorf("error getting splitstore path: %w", err)
}
return os.RemoveAll(path)
}
func clearSplitstoreDir(lr repo.LockedRepo) error {
path, err := lr.SplitstorePath()
if err != nil {
return xerrors.Errorf("error getting splitstore path: %w", err)
}
entries, err := os.ReadDir(path)
if err != nil {
return xerrors.Errorf("error reading splitstore directory %s: %W", path, err)
}
var result error
for _, e := range entries {
target := filepath.Join(path, e.Name())
err = os.RemoveAll(target)
if err != nil {
log.Errorf("error removing %s: %s", target, err)
result = multierr.Append(result, err)
}
}
return result
}
func deleteSplitstoreKeys(lr repo.LockedRepo) error {
ds, err := lr.Datastore(context.TODO(), "/metadata")
if err != nil {
return xerrors.Errorf("error opening datastore: %w", err)
}
if closer, ok := ds.(io.Closer); ok {
defer closer.Close() //nolint
}
var keys []datastore.Key
res, err := ds.Query(context.Background(), query.Query{Prefix: "/splitstore"})
if err != nil {
return xerrors.Errorf("error querying datastore for splitstore keys: %w", err)
}
for r := range res.Next() {
if r.Error != nil {
return xerrors.Errorf("datastore query error: %w", r.Error)
}
keys = append(keys, datastore.NewKey(r.Key))
}
for _, k := range keys {
fmt.Printf("deleting %s from datastore...\n", k)
err = ds.Delete(context.Background(), k)
if err != nil {
return xerrors.Errorf("error deleting key %s from datastore: %w", k, err)
}
}
return nil
}
// badger logging through go-log
type badgerLogger struct {
*zap.SugaredLogger
skip2 *zap.SugaredLogger
}
func (b *badgerLogger) Warningf(format string, args ...interface{}) {}
func (b *badgerLogger) Infof(format string, args ...interface{}) {}
func (b *badgerLogger) Debugf(format string, args ...interface{}) {}
var splitstoreCheckCmd = &cli.Command{
Name: "check",
Description: "runs a healthcheck on a splitstore installation",
Action: func(cctx *cli.Context) error {
api, closer, err := lcli.GetFullNodeAPIV1(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
return api.ChainCheckBlockstore(ctx)
},
}
var splitstoreInfoCmd = &cli.Command{
Name: "info",
Description: "prints some basic splitstore information",
Action: func(cctx *cli.Context) error {
api, closer, err := lcli.GetFullNodeAPIV1(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
info, err := api.ChainBlockstoreInfo(ctx)
if err != nil {
return err
}
for k, v := range info {
fmt.Print(k)
fmt.Print(": ")
fmt.Println(v)
}
return nil
},
}