From 0e3dd3cb3b0f9e1a8f66f85bb2e1924577558081 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 10 Sep 2020 17:28:25 -0700 Subject: [PATCH] Add faster and slimmer option to chain export via lotus-shed --- chain/store/store.go | 12 ++-- chain/store/store_test.go | 2 +- cmd/lotus-shed/export.go | 123 ++++++++++++++++++++++++++++++++++++++ cmd/lotus-shed/main.go | 1 + node/impl/full/chain.go | 2 +- 5 files changed, 134 insertions(+), 6 deletions(-) create mode 100644 cmd/lotus-shed/export.go diff --git a/chain/store/store.go b/chain/store/store.go index c85f547a1..d5ecf95ef 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -1159,7 +1159,7 @@ func recurseLinks(bs bstore.Blockstore, walked *cid.Set, root cid.Cid, in []cid. return in, rerr } -func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, w io.Writer) error { +func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, skipOldMsgs bool, w io.Writer) error { if ts == nil { ts = cs.GetHeaviestTipSet() } @@ -1197,9 +1197,13 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo return xerrors.Errorf("unmarshaling block header (cid=%s): %w", blk, err) } - cids, err := recurseLinks(cs.bs, walked, b.Messages, []cid.Cid{b.Messages}) - if err != nil { - return xerrors.Errorf("recursing messages failed: %w", err) + var cids []cid.Cid + if !skipOldMsgs || b.Height > ts.Height()-inclRecentRoots { + mcids, err := recurseLinks(cs.bs, walked, b.Messages, []cid.Cid{b.Messages}) + if err != nil { + return xerrors.Errorf("recursing messages failed: %w", err) + } + cids = mcids } if b.Height > 0 { diff --git a/chain/store/store_test.go b/chain/store/store_test.go index 1e3673f44..e56bab4c9 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -96,7 +96,7 @@ func TestChainExportImport(t *testing.T) { } buf := new(bytes.Buffer) - if err := cg.ChainStore().Export(context.TODO(), last, 0, buf); err != nil { + if err := cg.ChainStore().Export(context.TODO(), last, 0, false, buf); err != nil { t.Fatal(err) } diff --git a/cmd/lotus-shed/export.go b/cmd/lotus-shed/export.go new file mode 100644 index 000000000..6fa8d63f6 --- /dev/null +++ b/cmd/lotus-shed/export.go @@ -0,0 +1,123 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/filecoin-project/lotus/node/repo" +) + +var exportChainCmd = &cli.Command{ + Name: "export", + Description: "Export chain from repo (requires node to be offline)", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "repo", + Value: "~/.lotus", + }, + &cli.StringFlag{ + Name: "tipset", + Usage: "tipset to export from", + }, + &cli.Int64Flag{ + Name: "recent-stateroots", + }, + &cli.BoolFlag{ + Name: "full-state", + }, + &cli.BoolFlag{ + Name: "skip-old-msgs", + }, + }, + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return lcli.ShowHelp(cctx, fmt.Errorf("must specifiy file name to write export to")) + } + + ctx := context.TODO() + + r, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return xerrors.Errorf("opening fs repo: %w", err) + } + + exists, err := r.Exists() + if err != nil { + return err + } + if !exists { + return xerrors.Errorf("lotus repo doesn't exist") + } + + lr, err := r.Lock(repo.FullNode) + if err != nil { + return err + } + defer lr.Close() //nolint:errcheck + + fi, err := os.Create(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("opening the output file: %w", err) + } + + defer fi.Close() + + ds, err := lr.Datastore("/chain") + if err != nil { + return err + } + + mds, err := lr.Datastore("/metadata") + if err != nil { + return err + } + + bs := blockstore.NewBlockstore(ds) + + cs := store.NewChainStore(bs, mds, nil) + if err := cs.Load(); err != nil { + return err + } + + nroots := abi.ChainEpoch(cctx.Int64("recent-stateroots")) + fullstate := cctx.Bool("full-state") + skipoldmsgs := cctx.Bool("skip-old-msgs") + + var ts *types.TipSet + if tss := cctx.String("tipset"); tss != "" { + cids, err := lcli.ParseTipSetString(tss) + if err != nil { + return xerrors.Errorf("failed to parse tipset (%q): %w", tss, err) + } + + tsk := types.NewTipSetKey(cids...) + + selts, err := cs.LoadTipSet(tsk) + if err != nil { + return xerrors.Errorf("loading tipset: %w", err) + } + ts = selts + } else { + ts = cs.GetHeaviestTipSet() + } + + if fullstate { + nroots = ts.Height() + 1 + } + + if err := cs.Export(ctx, ts, nroots, skipoldmsgs, fi); err != nil { + return xerrors.Errorf("export failed: %w", err) + } + + return nil + }, +} diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 4944b67aa..cff3059b6 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -34,6 +34,7 @@ func main() { genesisVerifyCmd, mathCmd, mpoolStatsCmd, + exportChainCmd, } app := &cli.App{ diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index c5dd5c9a9..13b9dd00b 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -508,7 +508,7 @@ func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk t bw := bufio.NewWriterSize(w, 1<<20) defer bw.Flush() //nolint:errcheck // it is a write to a pipe - if err := a.Chain.Export(ctx, ts, nroots, bw); err != nil { + if err := a.Chain.Export(ctx, ts, nroots, false, bw); err != nil { log.Errorf("chain export call failed: %s", err) return }