cerc-io/lotus
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

allow exporting a number of recent chain state trees

Browse Source
This commit is contained in:
whyrusleeping 2020-09-01 16:38:24 -07:00 committed by Łukasz Magiera
parent edc2a28f1d
commit 09ebd1a557
6 changed files with 28 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api/api_full.go
View File

@ -109,7 +109,10 @@ type FullNode interface {
ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*HeadChange, error) ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*HeadChange, error)
// ChainExport returns a stream of bytes with CAR dump of chain data. // ChainExport returns a stream of bytes with CAR dump of chain data.
ChainExport(context.Context, types.TipSetKey) (<-chan []byte, error) // The exported chain data includes the header chain from the given tipset
// back to genesis, the entire genesis state, and the most recent 'nroots'
// state trees.
ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk types.TipSetKey) (<-chan []byte, error)
// MethodGroup: Beacon // MethodGroup: Beacon
// The Beacon method group contains methods for interacting with the random beacon (DRAND) // The Beacon method group contains methods for interacting with the random beacon (DRAND)

6
api/apistruct/struct.go
View File

@ -86,7 +86,7 @@ type FullNodeStruct struct {
ChainGetNode func(ctx context.Context, p string) (*api.IpldObject, error) `perm:"read"` ChainGetNode func(ctx context.Context, p string) (*api.IpldObject, error) `perm:"read"`
ChainGetMessage func(context.Context, cid.Cid) (*types.Message, error) `perm:"read"` ChainGetMessage func(context.Context, cid.Cid) (*types.Message, error) `perm:"read"`
ChainGetPath func(context.Context, types.TipSetKey, types.TipSetKey) ([]*api.HeadChange, error) `perm:"read"` ChainGetPath func(context.Context, types.TipSetKey, types.TipSetKey) ([]*api.HeadChange, error) `perm:"read"`
ChainExport func(context.Context, types.TipSetKey) (<-chan []byte, error) `perm:"read"` ChainExport func(context.Context, abi.ChainEpoch, types.TipSetKey) (<-chan []byte, error) `perm:"read"`
BeaconGetEntry func(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) `perm:"read"` BeaconGetEntry func(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) `perm:"read"`
@ -654,8 +654,8 @@ func (c *FullNodeStruct) ChainGetPath(ctx context.Context, from types.TipSetKey,
return c.Internal.ChainGetPath(ctx, from, to) return c.Internal.ChainGetPath(ctx, from, to)
} }
func (c *FullNodeStruct) ChainExport(ctx context.Context, tsk types.TipSetKey) (<-chan []byte, error) { func (c *FullNodeStruct) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk types.TipSetKey) (<-chan []byte, error) {
return c.Internal.ChainExport(ctx, tsk) return c.Internal.ChainExport(ctx, nroots, tsk)
} }
func (c *FullNodeStruct) BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) { func (c *FullNodeStruct) BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) {

17
chain/store/store.go
View File

@ -1114,7 +1114,7 @@ func (cs *ChainStore) GetTipsetByHeight(ctx context.Context, h abi.ChainEpoch, t
return cs.LoadTipSet(lbts.Parents()) return cs.LoadTipSet(lbts.Parents())
} }
func recurseLinks(bs bstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid, error) { func recurseLinks(bs bstore.Blockstore, seen *cid.Set, root cid.Cid, in []cid.Cid) ([]cid.Cid, error) {
if root.Prefix().Codec != cid.DagCBOR { if root.Prefix().Codec != cid.DagCBOR {
return in, nil return in, nil
} }
@ -1131,9 +1131,14 @@ func recurseLinks(bs bstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid,
return return
} }
// traversed this already...
if !seen.Visit(c) {
return
}
in = append(in, c) in = append(in, c)
var err error var err error
in, err = recurseLinks(bs, c, in) in, err = recurseLinks(bs, seen, c, in)
if err != nil { if err != nil {
rerr = err rerr = err
} }
@ -1145,7 +1150,7 @@ func recurseLinks(bs bstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid,
return in, rerr return in, rerr
} }
func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, w io.Writer) error { func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, w io.Writer) error {
if ts == nil { if ts == nil {
ts = cs.GetHeaviestTipSet() ts = cs.GetHeaviestTipSet()
} }
@ -1182,7 +1187,7 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, w io.Writer)
return xerrors.Errorf("unmarshaling block header (cid=%s): %w", blk, err) return xerrors.Errorf("unmarshaling block header (cid=%s): %w", blk, err)
} }
cids, err := recurseLinks(cs.bs, b.Messages, []cid.Cid{b.Messages}) cids, err := recurseLinks(cs.bs, seen, b.Messages, []cid.Cid{b.Messages})
if err != nil { if err != nil {
return xerrors.Errorf("recursing messages failed: %w", err) return xerrors.Errorf("recursing messages failed: %w", err)
} }
@ -1198,8 +1203,8 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, w io.Writer)
out := cids out := cids
if b.Height == 0 { if b.Height == 0 || b.Height > ts.Height()-inclRecentRoots {
cids, err := recurseLinks(cs.bs, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot}) cids, err := recurseLinks(cs.bs, seen, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot})
if err != nil { if err != nil {
return xerrors.Errorf("recursing genesis state failed: %w", err) return xerrors.Errorf("recursing genesis state failed: %w", err)
} }

2
chain/store/store_test.go
View File

@ -96,7 +96,7 @@ func TestChainExportImport(t *testing.T) {
} }
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
if err := cg.ChainStore().Export(context.TODO(), last, buf); err != nil { if err := cg.ChainStore().Export(context.TODO(), last, 0, buf); err != nil {
t.Fatal(err) t.Fatal(err)
} }

8
cli/chain.go
View File

@ -859,6 +859,10 @@ var chainExportCmd = &cli.Command{
&cli.StringFlag{ &cli.StringFlag{
Name: "tipset", Name: "tipset",
}, },
&cli.Int64Flag{
Name: "recent-stateroots",
Usage: "specify the number of recent state roots to include in the export",
},
}, },
Action: func(cctx *cli.Context) error { Action: func(cctx *cli.Context) error {
api, closer, err := GetFullNodeAPI(cctx) api, closer, err := GetFullNodeAPI(cctx)
@ -872,6 +876,8 @@ var chainExportCmd = &cli.Command{
return fmt.Errorf("must specify filename to export chain to") return fmt.Errorf("must specify filename to export chain to")
} }
rsrs := cctx.Int64("recent-stateroots")
fi, err := os.Create(cctx.Args().First()) fi, err := os.Create(cctx.Args().First())
if err != nil { if err != nil {
return err return err
@ -888,7 +894,7 @@ var chainExportCmd = &cli.Command{
return err return err
} }
stream, err := api.ChainExport(ctx, ts.Key()) stream, err := api.ChainExport(ctx, abi.ChainEpoch(rsrs), ts.Key())
if err != nil { if err != nil {
return err return err
} }

4
node/impl/full/chain.go
View File

@ -494,7 +494,7 @@ func (a *ChainAPI) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Mess
return cm.VMMessage(), nil return cm.VMMessage(), nil
} }
func (a *ChainAPI) ChainExport(ctx context.Context, tsk types.TipSetKey) (<-chan []byte, error) { func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk types.TipSetKey) (<-chan []byte, error) {
ts, err := a.Chain.GetTipSetFromKey(tsk) ts, err := a.Chain.GetTipSetFromKey(tsk)
if err != nil { if err != nil {
return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
@ -503,7 +503,7 @@ func (a *ChainAPI) ChainExport(ctx context.Context, tsk types.TipSetKey) (<-chan
out := make(chan []byte) out := make(chan []byte)
go func() { go func() {
defer w.Close() //nolint:errcheck // it is a pipe defer w.Close() //nolint:errcheck // it is a pipe
if err := a.Chain.Export(ctx, ts, w); err != nil { if err := a.Chain.Export(ctx, ts, nroots, w); err != nil {
log.Errorf("chain export call failed: %s", err) log.Errorf("chain export call failed: %s", err)
return return
} }