Add faster and slimmer option to chain export via lotus-shed

chain/store/store.go

@@ -1159,7 +1159,7 @@ func recurseLinks(bs bstore.Blockstore, walked *cid.Set, root cid.Cid, in []cid.
 	return in, rerr
 }
 
-func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, w io.Writer) error {
+func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, skipOldMsgs bool, w io.Writer) error {
 	if ts == nil {
 		ts = cs.GetHeaviestTipSet()
 	}
@@ -1197,9 +1197,13 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo
 			return xerrors.Errorf("unmarshaling block header (cid=%s): %w", blk, err)
 		}
 
-		cids, err := recurseLinks(cs.bs, walked, b.Messages, []cid.Cid{b.Messages})
-		if err != nil {
-			return xerrors.Errorf("recursing messages failed: %w", err)
+		var cids []cid.Cid
+		if !skipOldMsgs || b.Height > ts.Height()-inclRecentRoots {
+			mcids, err := recurseLinks(cs.bs, walked, b.Messages, []cid.Cid{b.Messages})
+			if err != nil {
+				return xerrors.Errorf("recursing messages failed: %w", err)
+			}
+			cids = mcids
 		}
 
 		if b.Height > 0 {

chain/store/store_test.go

@@ -96,7 +96,7 @@ func TestChainExportImport(t *testing.T) {
 	}
 
 	buf := new(bytes.Buffer)
-	if err := cg.ChainStore().Export(context.TODO(), last, 0, buf); err != nil {
+	if err := cg.ChainStore().Export(context.TODO(), last, 0, false, buf); err != nil {
 		t.Fatal(err)
 	}
 

cmd/lotus-shed/export.go

@@ -0,0 +1,123 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/types"
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/lib/blockstore"
+	"github.com/filecoin-project/lotus/node/repo"
+)
+
+var exportChainCmd = &cli.Command{
+	Name:        "export",
+	Description: "Export chain from repo (requires node to be offline)",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "repo",
+			Value: "~/.lotus",
+		},
+		&cli.StringFlag{
+			Name:  "tipset",
+			Usage: "tipset to export from",
+		},
+		&cli.Int64Flag{
+			Name: "recent-stateroots",
+		},
+		&cli.BoolFlag{
+			Name: "full-state",
+		},
+		&cli.BoolFlag{
+			Name: "skip-old-msgs",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Args().Present() {
+			return lcli.ShowHelp(cctx, fmt.Errorf("must specifiy file name to write export to"))
+		}
+
+		ctx := context.TODO()
+
+		r, err := repo.NewFS(cctx.String("repo"))
+		if err != nil {
+			return xerrors.Errorf("opening fs repo: %w", err)
+		}
+
+		exists, err := r.Exists()
+		if err != nil {
+			return err
+		}
+		if !exists {
+			return xerrors.Errorf("lotus repo doesn't exist")
+		}
+
+		lr, err := r.Lock(repo.FullNode)
+		if err != nil {
+			return err
+		}
+		defer lr.Close() //nolint:errcheck
+
+		fi, err := os.Create(cctx.Args().First())
+		if err != nil {
+			return xerrors.Errorf("opening the output file: %w", err)
+		}
+
+		defer fi.Close()
+
+		ds, err := lr.Datastore("/chain")
+		if err != nil {
+			return err
+		}
+
+		mds, err := lr.Datastore("/metadata")
+		if err != nil {
+			return err
+		}
+
+		bs := blockstore.NewBlockstore(ds)
+
+		cs := store.NewChainStore(bs, mds, nil)
+		if err := cs.Load(); err != nil {
+			return err
+		}
+
+		nroots := abi.ChainEpoch(cctx.Int64("recent-stateroots"))
+		fullstate := cctx.Bool("full-state")
+		skipoldmsgs := cctx.Bool("skip-old-msgs")
+
+		var ts *types.TipSet
+		if tss := cctx.String("tipset"); tss != "" {
+			cids, err := lcli.ParseTipSetString(tss)
+			if err != nil {
+				return xerrors.Errorf("failed to parse tipset (%q): %w", tss, err)
+			}
+
+			tsk := types.NewTipSetKey(cids...)
+
+			selts, err := cs.LoadTipSet(tsk)
+			if err != nil {
+				return xerrors.Errorf("loading tipset: %w", err)
+			}
+			ts = selts
+		} else {
+			ts = cs.GetHeaviestTipSet()
+		}
+
+		if fullstate {
+			nroots = ts.Height() + 1
+		}
+
+		if err := cs.Export(ctx, ts, nroots, skipoldmsgs, fi); err != nil {
+			return xerrors.Errorf("export failed: %w", err)
+		}
+
+		return nil
+	},
+}

cmd/lotus-shed/main.go

@@ -34,6 +34,7 @@ func main() {
 		genesisVerifyCmd,
 		mathCmd,
 		mpoolStatsCmd,
+		exportChainCmd,
 	}
 
 	app := &cli.App{

node/impl/full/chain.go

@@ -508,7 +508,7 @@ func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk t
 		bw := bufio.NewWriterSize(w, 1<<20)
 		defer bw.Flush() //nolint:errcheck // it is a write to a pipe
 
-		if err := a.Chain.Export(ctx, ts, nroots, bw); err != nil {
+		if err := a.Chain.Export(ctx, ts, nroots, false, bw); err != nil {
 			log.Errorf("chain export call failed: %s", err)
 			return
 		}