Merge pull request #1346 from filecoin-project/feat/stat-cmd

Add stat-obj to API and a few extras

api/api_full.go

@@ -43,6 +43,7 @@ type FullNode interface {
 	ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)
 	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
+	ChainStatObj(context.Context, cid.Cid, cid.Cid) (ObjStat, error)
 	ChainSetHead(context.Context, types.TipSetKey) error
 	ChainGetGenesis(context.Context) (*types.TipSet, error)
 	ChainTipSetWeight(context.Context, types.TipSetKey) (types.BigInt, error)

api/apistruct/struct.go

@@ -60,6 +60,7 @@ type FullNodeStruct struct {
 		ChainGetTipSetByHeight func(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)                                      `perm:"read"`
 		ChainReadObj           func(context.Context, cid.Cid) ([]byte, error)                                                                     `perm:"read"`
 		ChainHasObj            func(context.Context, cid.Cid) (bool, error)                                                                       `perm:"read"`
+		ChainStatObj           func(context.Context, cid.Cid, cid.Cid) (api.ObjStat, error)                                                       `perm:"read"`
 		ChainSetHead           func(context.Context, types.TipSetKey) error                                                                       `perm:"admin"`
 		ChainGetGenesis        func(context.Context) (*types.TipSet, error)                                                                       `perm:"read"`
 		ChainTipSetWeight      func(context.Context, types.TipSetKey) (types.BigInt, error)                                                       `perm:"read"`
@@ -380,6 +381,10 @@ func (c *FullNodeStruct) ChainHasObj(ctx context.Context, o cid.Cid) (bool, erro
 	return c.Internal.ChainHasObj(ctx, o)
 }
 
+func (c *FullNodeStruct) ChainStatObj(ctx context.Context, obj, base cid.Cid) (api.ObjStat, error) {
+	return c.Internal.ChainStatObj(ctx, obj, base)
+}
+
 func (c *FullNodeStruct) ChainSetHead(ctx context.Context, tsk types.TipSetKey) error {
 	return c.Internal.ChainSetHead(ctx, tsk)
 }

api/types.go

@@ -27,3 +27,8 @@ func (m *MultiaddrSlice) UnmarshalJSON(raw []byte) (err error) {
 }
 
 var _ json.Unmarshaler = new(MultiaddrSlice)
+
+type ObjStat struct {
+	Size  uint64
+	Links uint64
+}

cli/chain.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/docker/go-units"
 	cborutil "github.com/filecoin-project/go-cbor-util"
 	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
@@ -31,6 +32,7 @@ var chainCmd = &cli.Command{
 		chainHeadCmd,
 		chainGetBlock,
 		chainReadObjCmd,
+		chainStatObjCmd,
 		chainGetMsgCmd,
 		chainSetHeadCmd,
 		chainListCmd,
@@ -182,6 +184,50 @@ var chainReadObjCmd = &cli.Command{
 	},
 }
 
+var chainStatObjCmd = &cli.Command{
+	Name:      "stat-obj",
+	Usage:     "Collect size and ipld link counts for objs",
+	ArgsUsage: "[cid]",
+	Description: `Collect object size and ipld link count for an object.
+
+   When a base is provided it will be walked first, and all links visisted
+   will be ignored when the passed in object is walked.
+`,
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "base",
+			Usage: "ignore links found in this obj",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		obj, err := cid.Decode(cctx.Args().First())
+		if err != nil {
+			return fmt.Errorf("failed to parse cid input: %s", err)
+		}
+
+		base := cid.Undef
+		if cctx.IsSet("base") {
+			base, err = cid.Decode(cctx.String("base"))
+		}
+
+		stats, err := api.ChainStatObj(ctx, obj, base)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Links: %d\n", stats.Links)
+		fmt.Printf("Size: %s (%d)\n", units.BytesSize(float64(stats.Size)), stats.Size)
+		return nil
+	},
+}
+
 var chainGetMsgCmd = &cli.Command{
 	Name:      "getmessage",
 	Usage:     "Get and print a message by its cid",
@@ -548,7 +594,7 @@ var chainExportCmd = &cli.Command{
 		}
 		defer fi.Close()
 
-		ts, err := loadTipSet(ctx, cctx, api)
+		ts, err := LoadTipSet(ctx, cctx, api)
 		if err != nil {
 			return err
 		}

cli/state.go

@@ -82,7 +82,7 @@ var stateMinerInfo = &cli.Command{
 			return err
 		}
 
-		ts, err := loadTipSet(ctx, cctx, api)
+		ts, err := LoadTipSet(ctx, cctx, api)
 		if err != nil {
 			return err
 		}
@@ -133,7 +133,7 @@ func parseTipSetString(cctx *cli.Context) ([]cid.Cid, error) {
 	return cids, nil
 }
 
-func loadTipSet(ctx context.Context, cctx *cli.Context, api api.FullNode) (*types.TipSet, error) {
+func LoadTipSet(ctx context.Context, cctx *cli.Context, api api.FullNode) (*types.TipSet, error) {
 	cids, err := parseTipSetString(cctx)
 	if err != nil {
 		return nil, err
@@ -173,7 +173,7 @@ var statePowerCmd = &cli.Command{
 			}
 		}
 
-		ts, err := loadTipSet(ctx, cctx, api)
+		ts, err := LoadTipSet(ctx, cctx, api)
 		if err != nil {
 			return err
 		}
@@ -218,7 +218,7 @@ var stateSectorsCmd = &cli.Command{
 			return err
 		}
 
-		ts, err := loadTipSet(ctx, cctx, api)
+		ts, err := LoadTipSet(ctx, cctx, api)
 		if err != nil {
 			return err
 		}
@@ -258,7 +258,7 @@ var stateProvingSetCmd = &cli.Command{
 			return err
 		}
 
-		ts, err := loadTipSet(ctx, cctx, api)
+		ts, err := LoadTipSet(ctx, cctx, api)
 		if err != nil {
 			return err
 		}
@@ -368,7 +368,7 @@ var statePledgeCollateralCmd = &cli.Command{
 
 		ctx := ReqContext(cctx)
 
-		ts, err := loadTipSet(ctx, cctx, api)
+		ts, err := LoadTipSet(ctx, cctx, api)
 		if err != nil {
 			return err
 		}
@@ -405,7 +405,7 @@ var stateGetDealSetCmd = &cli.Command{
 			return xerrors.Errorf("parsing deal ID: %w", err)
 		}
 
-		ts, err := loadTipSet(ctx, cctx, api)
+		ts, err := LoadTipSet(ctx, cctx, api)
 		if err != nil {
 			return err
 		}
@@ -437,7 +437,7 @@ var stateListMinersCmd = &cli.Command{
 
 		ctx := ReqContext(cctx)
 
-		ts, err := loadTipSet(ctx, cctx, api)
+		ts, err := LoadTipSet(ctx, cctx, api)
 		if err != nil {
 			return err
 		}
@@ -467,7 +467,7 @@ var stateListActorsCmd = &cli.Command{
 
 		ctx := ReqContext(cctx)
 
-		ts, err := loadTipSet(ctx, cctx, api)
+		ts, err := LoadTipSet(ctx, cctx, api)
 		if err != nil {
 			return err
 		}
@@ -507,7 +507,7 @@ var stateGetActorCmd = &cli.Command{
 			return err
 		}
 
-		ts, err := loadTipSet(ctx, cctx, api)
+		ts, err := LoadTipSet(ctx, cctx, api)
 		if err != nil {
 			return err
 		}
@@ -549,7 +549,7 @@ var stateLookupIDCmd = &cli.Command{
 			return err
 		}
 
-		ts, err := loadTipSet(ctx, cctx, api)
+		ts, err := LoadTipSet(ctx, cctx, api)
 		if err != nil {
 			return err
 		}
@@ -587,7 +587,7 @@ var stateSectorSizeCmd = &cli.Command{
 			return err
 		}
 
-		ts, err := loadTipSet(ctx, cctx, api)
+		ts, err := LoadTipSet(ctx, cctx, api)
 		if err != nil {
 			return err
 		}
@@ -624,7 +624,7 @@ var stateReadStateCmd = &cli.Command{
 			return err
 		}
 
-		ts, err := loadTipSet(ctx, cctx, api)
+		ts, err := LoadTipSet(ctx, cctx, api)
 		if err != nil {
 			return err
 		}
@@ -698,7 +698,7 @@ var stateListMessagesCmd = &cli.Command{
 
 		toh := cctx.Uint64("toheight")
 
-		ts, err := loadTipSet(ctx, cctx, api)
+		ts, err := LoadTipSet(ctx, cctx, api)
 		if err != nil {
 			return err
 		}
@@ -751,7 +751,7 @@ var stateComputeStateCmd = &cli.Command{
 
 		ctx := ReqContext(cctx)
 
-		ts, err := loadTipSet(ctx, cctx, api)
+		ts, err := LoadTipSet(ctx, cctx, api)
 		if err != nil {
 			return err
 		}
@@ -880,7 +880,7 @@ var stateCallCmd = &cli.Command{
 			return fmt.Errorf("given 'from' address %q was invalid: %w", cctx.String("from"), err)
 		}
 
-		ts, err := loadTipSet(ctx, cctx, api)
+		ts, err := LoadTipSet(ctx, cctx, api)
 		if err != nil {
 			return err
 		}

cmd/lotus-shed/main.go

@@ -21,6 +21,7 @@ func main() {
 		peerkeyCmd,
 		noncefix,
 		bigIntParseCmd,
+		staterootStatsCmd,
 	}
 
 	app := &cli.App{
@@ -28,6 +29,14 @@ func main() {
 		Usage:    "A place for all the lotus tools",
 		Version:  build.BuildVersion,
 		Commands: local,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "repo",
+				EnvVars: []string{"LOTUS_PATH"},
+				Hidden:  true,
+				Value:   "~/.lotus", // TODO: Consider XDG_DATA_HOME
+			},
+		},
 	}
 
 	if err := app.Run(os.Args); err != nil {

cmd/lotus-shed/stateroot-stats.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"fmt"
+
+	"gopkg.in/urfave/cli.v2"
+
+	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	lcli "github.com/filecoin-project/lotus/cli"
+)
+
+var staterootStatsCmd = &cli.Command{
+	Name:        "stateroot-stats",
+	Description: "Walk down the chain and collect stats-obj changes between tipsets",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "tipset",
+			Usage: "specify tipset to start from",
+		},
+		&cli.IntFlag{
+			Name:  "count",
+			Usage: "number of tipsets to count back",
+			Value: 30,
+		},
+		&cli.BoolFlag{
+			Name:  "diff",
+			Usage: "compare tipset with previous",
+			Value: false,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		ts, err := lcli.LoadTipSet(ctx, cctx, api)
+		if err != nil {
+			return err
+		}
+
+		if ts == nil {
+			ts, err = api.ChainHead(ctx)
+			if err != nil {
+				return err
+			}
+		}
+
+		fn := func(ts *types.TipSet) (cid.Cid, []cid.Cid) {
+			blk := ts.Blocks()[0]
+			strt := blk.ParentStateRoot
+			cids := blk.Parents
+
+			return strt, cids
+		}
+
+		count := cctx.Int("count")
+		diff := cctx.Bool("diff")
+
+		fmt.Printf("Height\tSize\tLinks\tObj\tBase\n")
+		for i := 0; i < count; i++ {
+			if ts.Height() == 0 {
+				return nil
+			}
+			strt, cids := fn(ts)
+
+			k := types.NewTipSetKey(cids...)
+			ts, err = api.ChainGetTipSet(ctx, k)
+			if err != nil {
+				return err
+			}
+
+			pstrt, _ := fn(ts)
+
+			if !diff {
+				pstrt = cid.Undef
+			}
+
+			stats, err := api.ChainStatObj(ctx, strt, pstrt)
+			if err != nil {
+				return err
+			}
+
+			fmt.Printf("%d\t%d\t%d\t%s\t%s\n", ts.Height(), stats.Size, stats.Links, strt, pstrt)
+		}
+
+		return nil
+	},
+}

node/impl/full/chain.go

@@ -6,8 +6,10 @@ import (
 	"io"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/filecoin-project/go-amt-ipld/v2"
+	commcid "github.com/filecoin-project/go-fil-commcid"
 	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-actors/actors/crypto"
 	"github.com/ipfs/go-blockservice"
@@ -185,6 +187,54 @@ func (a *ChainAPI) ChainHasObj(ctx context.Context, obj cid.Cid) (bool, error) {
 	return a.Chain.Blockstore().Has(obj)
 }
 
+func (a *ChainAPI) ChainStatObj(ctx context.Context, obj cid.Cid, base cid.Cid) (api.ObjStat, error) {
+	bs := a.Chain.Blockstore()
+	bsvc := blockservice.New(bs, offline.Exchange(bs))
+
+	dag := merkledag.NewDAGService(bsvc)
+
+	seen := cid.NewSet()
+
+	var statslk sync.Mutex
+	var stats api.ObjStat
+	var collect = true
+
+	walker := func(ctx context.Context, c cid.Cid) ([]*ipld.Link, error) {
+		if c.Prefix().MhType == uint64(commcid.FC_SEALED_V1) || c.Prefix().MhType == uint64(commcid.FC_UNSEALED_V1) {
+			return []*ipld.Link{}, nil
+		}
+
+		nd, err := dag.Get(ctx, c)
+		if err != nil {
+			return nil, err
+		}
+
+		if collect {
+			s := uint64(len(nd.RawData()))
+			statslk.Lock()
+			stats.Size = stats.Size + s
+			stats.Links = stats.Links + 1
+			statslk.Unlock()
+		}
+
+		return nd.Links(), nil
+	}
+
+	if base != cid.Undef {
+		collect = false
+		if err := merkledag.Walk(ctx, walker, base, seen.Visit, merkledag.Concurrent()); err != nil {
+			return api.ObjStat{}, err
+		}
+		collect = true
+	}
+
+	if err := merkledag.Walk(ctx, walker, obj, seen.Visit, merkledag.Concurrent()); err != nil {
+		return api.ObjStat{}, err
+	}
+
+	return stats, nil
+}
+
 func (a *ChainAPI) ChainSetHead(ctx context.Context, tsk types.TipSetKey) error {
 	ts, err := a.Chain.GetTipSetFromKey(tsk)
 	if err != nil {