cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #1346 from filecoin-project/feat/stat-cmd

Browse Source 
Add stat-obj to API and a few extras
This commit is contained in:
Whyrusleeping 2020-03-06 18:55:48 -08:00 committed by GitHub
commit 0b5991850c
8 changed files with 227 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
api/api_full.go
View File

@ -43,6 +43,7 @@ type FullNode interface {
ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
ChainHasObj(context.Context, cid.Cid) (bool, error)
ChainStatObj(context.Context, cid.Cid, cid.Cid) (ObjStat, error)
ChainSetHead(context.Context, types.TipSetKey) error
ChainGetGenesis(context.Context) (*types.TipSet, error)
ChainTipSetWeight(context.Context, types.TipSetKey) (types.BigInt, error)

5
api/apistruct/struct.go
View File

@ -60,6 +60,7 @@ type FullNodeStruct struct {
ChainGetTipSetByHeight func(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) `perm:"read"`
ChainReadObj func(context.Context, cid.Cid) ([]byte, error) `perm:"read"`
ChainHasObj func(context.Context, cid.Cid) (bool, error) `perm:"read"`
ChainStatObj func(context.Context, cid.Cid, cid.Cid) (api.ObjStat, error) `perm:"read"`
ChainSetHead func(context.Context, types.TipSetKey) error `perm:"admin"`
ChainGetGenesis func(context.Context) (*types.TipSet, error) `perm:"read"`
ChainTipSetWeight func(context.Context, types.TipSetKey) (types.BigInt, error) `perm:"read"`
@ -380,6 +381,10 @@ func (c *FullNodeStruct) ChainHasObj(ctx context.Context, o cid.Cid) (bool, erro
return c.Internal.ChainHasObj(ctx, o)
}
func (c *FullNodeStruct) ChainStatObj(ctx context.Context, obj, base cid.Cid) (api.ObjStat, error) {
return c.Internal.ChainStatObj(ctx, obj, base)
}
func (c *FullNodeStruct) ChainSetHead(ctx context.Context, tsk types.TipSetKey) error {
return c.Internal.ChainSetHead(ctx, tsk)
}

5
api/types.go
View File

@ -27,3 +27,8 @@ func (m *MultiaddrSlice) UnmarshalJSON(raw []byte) (err error) {
}
var _ json.Unmarshaler = new(MultiaddrSlice)
type ObjStat struct {
Size uint64
Links uint64
}

48
cli/chain.go
View File

@ -11,6 +11,7 @@ import (
"strings"
"time"
"github.com/docker/go-units"
cborutil "github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin"
@ -31,6 +32,7 @@ var chainCmd = &cli.Command{
chainHeadCmd,
chainGetBlock,
chainReadObjCmd,
chainStatObjCmd,
chainGetMsgCmd,
chainSetHeadCmd,
chainListCmd,
@ -182,6 +184,50 @@ var chainReadObjCmd = &cli.Command{
},
}
var chainStatObjCmd = &cli.Command{
Name: "stat-obj",
Usage: "Collect size and ipld link counts for objs",
ArgsUsage: "[cid]",
Description: `Collect object size and ipld link count for an object.
When a base is provided it will be walked first, and all links visisted
will be ignored when the passed in object is walked.
`,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "base",
Usage: "ignore links found in this obj",
},
},
Action: func(cctx *cli.Context) error {
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := ReqContext(cctx)
obj, err := cid.Decode(cctx.Args().First())
if err != nil {
return fmt.Errorf("failed to parse cid input: %s", err)
}
base := cid.Undef
if cctx.IsSet("base") {
base, err = cid.Decode(cctx.String("base"))
}
stats, err := api.ChainStatObj(ctx, obj, base)
if err != nil {
return err
}
fmt.Printf("Links: %d\n", stats.Links)
fmt.Printf("Size: %s (%d)\n", units.BytesSize(float64(stats.Size)), stats.Size)
return nil
},
}
var chainGetMsgCmd = &cli.Command{
Name: "getmessage",
Usage: "Get and print a message by its cid",
@ -548,7 +594,7 @@ var chainExportCmd = &cli.Command{
}
defer fi.Close()
ts, err := loadTipSet(ctx, cctx, api)
ts, err := LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}

32
cli/state.go
View File

@ -82,7 +82,7 @@ var stateMinerInfo = &cli.Command{
return err
}
ts, err := loadTipSet(ctx, cctx, api)
ts, err := LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}
@ -133,7 +133,7 @@ func parseTipSetString(cctx *cli.Context) ([]cid.Cid, error) {
return cids, nil
}
func loadTipSet(ctx context.Context, cctx *cli.Context, api api.FullNode) (*types.TipSet, error) {
func LoadTipSet(ctx context.Context, cctx *cli.Context, api api.FullNode) (*types.TipSet, error) {
cids, err := parseTipSetString(cctx)
if err != nil {
return nil, err
@ -173,7 +173,7 @@ var statePowerCmd = &cli.Command{
}
}
ts, err := loadTipSet(ctx, cctx, api)
ts, err := LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}
@ -218,7 +218,7 @@ var stateSectorsCmd = &cli.Command{
return err
}
ts, err := loadTipSet(ctx, cctx, api)
ts, err := LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}
@ -258,7 +258,7 @@ var stateProvingSetCmd = &cli.Command{
return err
}
ts, err := loadTipSet(ctx, cctx, api)
ts, err := LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}
@ -368,7 +368,7 @@ var statePledgeCollateralCmd = &cli.Command{
ctx := ReqContext(cctx)
ts, err := loadTipSet(ctx, cctx, api)
ts, err := LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}
@ -405,7 +405,7 @@ var stateGetDealSetCmd = &cli.Command{
return xerrors.Errorf("parsing deal ID: %w", err)
}
ts, err := loadTipSet(ctx, cctx, api)
ts, err := LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}
@ -437,7 +437,7 @@ var stateListMinersCmd = &cli.Command{
ctx := ReqContext(cctx)
ts, err := loadTipSet(ctx, cctx, api)
ts, err := LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}
@ -467,7 +467,7 @@ var stateListActorsCmd = &cli.Command{
ctx := ReqContext(cctx)
ts, err := loadTipSet(ctx, cctx, api)
ts, err := LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}
@ -507,7 +507,7 @@ var stateGetActorCmd = &cli.Command{
return err
}
ts, err := loadTipSet(ctx, cctx, api)
ts, err := LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}
@ -549,7 +549,7 @@ var stateLookupIDCmd = &cli.Command{
return err
}
ts, err := loadTipSet(ctx, cctx, api)
ts, err := LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}
@ -587,7 +587,7 @@ var stateSectorSizeCmd = &cli.Command{
return err
}
ts, err := loadTipSet(ctx, cctx, api)
ts, err := LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}
@ -624,7 +624,7 @@ var stateReadStateCmd = &cli.Command{
return err
}
ts, err := loadTipSet(ctx, cctx, api)
ts, err := LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}
@ -698,7 +698,7 @@ var stateListMessagesCmd = &cli.Command{
toh := cctx.Uint64("toheight")
ts, err := loadTipSet(ctx, cctx, api)
ts, err := LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}
@ -751,7 +751,7 @@ var stateComputeStateCmd = &cli.Command{
ctx := ReqContext(cctx)
ts, err := loadTipSet(ctx, cctx, api)
ts, err := LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}
@ -880,7 +880,7 @@ var stateCallCmd = &cli.Command{
return fmt.Errorf("given 'from' address %q was invalid: %w", cctx.String("from"), err)
}
ts, err := loadTipSet(ctx, cctx, api)
ts, err := LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}

9
cmd/lotus-shed/main.go
View File

@ -21,6 +21,7 @@ func main() {
peerkeyCmd,
noncefix,
bigIntParseCmd,
staterootStatsCmd,
}
app := &cli.App{
@ -28,6 +29,14 @@ func main() {
Usage: "A place for all the lotus tools",
Version: build.BuildVersion,
Commands: local,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "repo",
EnvVars: []string{"LOTUS_PATH"},
Hidden: true,
Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME
},
},
}
if err := app.Run(os.Args); err != nil {

94
cmd/lotus-shed/stateroot-stats.go Normal file
View File

@ -0,0 +1,94 @@
package main
import (
"fmt"
"gopkg.in/urfave/cli.v2"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
)
var staterootStatsCmd = &cli.Command{
Name: "stateroot-stats",
Description: "Walk down the chain and collect stats-obj changes between tipsets",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "tipset",
Usage: "specify tipset to start from",
},
&cli.IntFlag{
Name: "count",
Usage: "number of tipsets to count back",
Value: 30,
},
&cli.BoolFlag{
Name: "diff",
Usage: "compare tipset with previous",
Value: false,
},
},
Action: func(cctx *cli.Context) error {
api, closer, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
ts, err := lcli.LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}
if ts == nil {
ts, err = api.ChainHead(ctx)
if err != nil {
return err
}
}
fn := func(ts *types.TipSet) (cid.Cid, []cid.Cid) {
blk := ts.Blocks()[0]
strt := blk.ParentStateRoot
cids := blk.Parents
return strt, cids
}
count := cctx.Int("count")
diff := cctx.Bool("diff")
fmt.Printf("Height\tSize\tLinks\tObj\tBase\n")
for i := 0; i < count; i++ {
if ts.Height() == 0 {
return nil
}
strt, cids := fn(ts)
k := types.NewTipSetKey(cids...)
ts, err = api.ChainGetTipSet(ctx, k)
if err != nil {
return err
}
pstrt, _ := fn(ts)
if !diff {
pstrt = cid.Undef
}
stats, err := api.ChainStatObj(ctx, strt, pstrt)
if err != nil {
return err
}
fmt.Printf("%d\t%d\t%d\t%s\t%s\n", ts.Height(), stats.Size, stats.Links, strt, pstrt)
}
return nil
},
}

50
node/impl/full/chain.go
View File

@ -6,8 +6,10 @@ import (
"io"
"strconv"
"strings"
"sync"
"github.com/filecoin-project/go-amt-ipld/v2"
commcid "github.com/filecoin-project/go-fil-commcid"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/ipfs/go-blockservice"
@ -185,6 +187,54 @@ func (a *ChainAPI) ChainHasObj(ctx context.Context, obj cid.Cid) (bool, error) {
return a.Chain.Blockstore().Has(obj)
}
func (a *ChainAPI) ChainStatObj(ctx context.Context, obj cid.Cid, base cid.Cid) (api.ObjStat, error) {
bs := a.Chain.Blockstore()
bsvc := blockservice.New(bs, offline.Exchange(bs))
dag := merkledag.NewDAGService(bsvc)
seen := cid.NewSet()
var statslk sync.Mutex
var stats api.ObjStat
var collect = true
walker := func(ctx context.Context, c cid.Cid) ([]*ipld.Link, error) {
if c.Prefix().MhType == uint64(commcid.FC_SEALED_V1) || c.Prefix().MhType == uint64(commcid.FC_UNSEALED_V1) {
return []*ipld.Link{}, nil
}
nd, err := dag.Get(ctx, c)
if err != nil {
return nil, err
}
if collect {
s := uint64(len(nd.RawData()))
statslk.Lock()
stats.Size = stats.Size + s
stats.Links = stats.Links + 1
statslk.Unlock()
}
return nd.Links(), nil
}
if base != cid.Undef {
collect = false
if err := merkledag.Walk(ctx, walker, base, seen.Visit, merkledag.Concurrent()); err != nil {
return api.ObjStat{}, err
}
collect = true
}
if err := merkledag.Walk(ctx, walker, obj, seen.Visit, merkledag.Concurrent()); err != nil {
return api.ObjStat{}, err
}
return stats, nil
}
func (a *ChainAPI) ChainSetHead(ctx context.Context, tsk types.TipSetKey) error {
ts, err := a.Chain.GetTipSetFromKey(tsk)
if err != nil {