CLI tools for the DAGStore inverted index (#7361)

* cli commands for dagstore

* address comments from Dirk

* rename inverted index to piece index

api/api_storage.go

@@ -212,7 +212,13 @@ type StorageMiner interface {
 
 	// IndexerAnnounceDeal informs indexer nodes that a new deal was received,
 	// so they can download its index
-	IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error
+	IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin
+
+	// DagstorePieceIndexSize returns the size of the piece index.
+	DagstorePieceIndexSize(ctx context.Context) (int64, error) //perm:admin
+
+	// DagstoreLookupPieces returns information about shards that contain the given CID.
+	DagstoreLookupPieces(ctx context.Context, cid cid.Cid) ([]DagstoreShardInfo, error) //perm:admin
 
 	// RuntimeSubsystems returns the subsystems that are enabled
 	// in this instance.

api/proxy_gen.go

@@ -623,6 +623,10 @@ type StorageMinerStruct struct {
 
 		DagstoreListShards func(p0 context.Context) ([]DagstoreShardInfo, error) `perm:"read"`
 
+		DagstoreLookupPieces func(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) `perm:"admin"`
+
+		DagstorePieceIndexSize func(p0 context.Context) (int64, error) `perm:"admin"`
+
 		DagstoreRecoverShard func(p0 context.Context, p1 string) error `perm:"write"`
 
 		DealsConsiderOfflineRetrievalDeals func(p0 context.Context) (bool, error) `perm:"admin"`
@@ -657,7 +661,7 @@ type StorageMinerStruct struct {
 
 		DealsSetPieceCidBlocklist func(p0 context.Context, p1 []cid.Cid) error `perm:"admin"`
 
-		IndexerAnnounceDeal func(p0 context.Context, p1 cid.Cid) error ``
+		IndexerAnnounceDeal func(p0 context.Context, p1 cid.Cid) error `perm:"admin"`
 
 		MarketCancelDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"`
 
@@ -3694,6 +3698,28 @@ func (s *StorageMinerStub) DagstoreListShards(p0 context.Context) ([]DagstoreSha
 	return *new([]DagstoreShardInfo), ErrNotSupported
 }
 
+func (s *StorageMinerStruct) DagstoreLookupPieces(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) {
+	if s.Internal.DagstoreLookupPieces == nil {
+		return *new([]DagstoreShardInfo), ErrNotSupported
+	}
+	return s.Internal.DagstoreLookupPieces(p0, p1)
+}
+
+func (s *StorageMinerStub) DagstoreLookupPieces(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) {
+	return *new([]DagstoreShardInfo), ErrNotSupported
+}
+
+func (s *StorageMinerStruct) DagstorePieceIndexSize(p0 context.Context) (int64, error) {
+	if s.Internal.DagstorePieceIndexSize == nil {
+		return 0, ErrNotSupported
+	}
+	return s.Internal.DagstorePieceIndexSize(p0)
+}
+
+func (s *StorageMinerStub) DagstorePieceIndexSize(p0 context.Context) (int64, error) {
+	return 0, ErrNotSupported
+}
+
 func (s *StorageMinerStruct) DagstoreRecoverShard(p0 context.Context, p1 string) error {
 	if s.Internal.DagstoreRecoverShard == nil {
 		return ErrNotSupported

cmd/lotus-miner/dagstore.go

@@ -4,7 +4,9 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/dustin/go-humanize"
 	"github.com/fatih/color"
+	"github.com/ipfs/go-cid"
 	"github.com/urfave/cli/v2"
 
 	"github.com/filecoin-project/lotus/api"
@@ -21,6 +23,8 @@ var dagstoreCmd = &cli.Command{
 		dagstoreRecoverShardCmd,
 		dagstoreInitializeAllCmd,
 		dagstoreGcCmd,
+		dagstorePieceIndexSizeCmd,
+		dagstoreLookupPiecesCmd,
 	},
 }
 
@@ -52,38 +56,7 @@ var dagstoreListShardsCmd = &cli.Command{
 			return err
 		}
 
-		if len(shards) == 0 {
-			return nil
-		}
-
-		tw := tablewriter.New(
-			tablewriter.Col("Key"),
-			tablewriter.Col("State"),
-			tablewriter.Col("Error"),
-		)
-
-		colors := map[string]color.Attribute{
-			"ShardStateAvailable": color.FgGreen,
-			"ShardStateServing":   color.FgBlue,
-			"ShardStateErrored":   color.FgRed,
-			"ShardStateNew":       color.FgYellow,
-		}
-
-		for _, s := range shards {
-			m := map[string]interface{}{
-				"Key": s.Key,
-				"State": func() string {
-					if c, ok := colors[s.State]; ok {
-						return color.New(c).Sprint(s.State)
-					}
-					return s.State
-				}(),
-				"Error": s.Error,
-			}
-			tw.Write(m)
-		}
-
-		return tw.Flush(os.Stdout)
+		return printTableShards(shards)
 	},
 }
 
@@ -265,3 +238,114 @@ var dagstoreGcCmd = &cli.Command{
 		return nil
 	},
 }
+
+func printTableShards(shards []api.DagstoreShardInfo) error {
+	if len(shards) == 0 {
+		return nil
+	}
+
+	tw := tablewriter.New(
+		tablewriter.Col("Key"),
+		tablewriter.Col("State"),
+		tablewriter.Col("Error"),
+	)
+
+	colors := map[string]color.Attribute{
+		"ShardStateAvailable": color.FgGreen,
+		"ShardStateServing":   color.FgBlue,
+		"ShardStateErrored":   color.FgRed,
+		"ShardStateNew":       color.FgYellow,
+	}
+
+	for _, s := range shards {
+		m := map[string]interface{}{
+			"Key": s.Key,
+			"State": func() string {
+				if c, ok := colors[s.State]; ok {
+					return color.New(c).Sprint(s.State)
+				}
+				return s.State
+			}(),
+			"Error": s.Error,
+		}
+		tw.Write(m)
+	}
+	return tw.Flush(os.Stdout)
+}
+
+var dagstorePieceIndexSizeCmd = &cli.Command{
+	Name:  "piece-index-size",
+	Usage: "Inspect the dagstore piece index size",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "color",
+			Usage:       "use color in display output",
+			DefaultText: "depends on output being a TTY",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.IsSet("color") {
+			color.NoColor = !cctx.Bool("color")
+		}
+
+		marketsApi, closer, err := lcli.GetMarketsAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		size, err := marketsApi.DagstorePieceIndexSize(ctx)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(humanize.Bytes(uint64(size)))
+
+		return nil
+	},
+}
+
+var dagstoreLookupPiecesCmd = &cli.Command{
+	Name:      "lookup-pieces",
+	Usage:     "Lookup pieces that a given CID belongs to",
+	ArgsUsage: "<cid>",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "color",
+			Usage:       "use color in display output",
+			DefaultText: "depends on output being a TTY",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.IsSet("color") {
+			color.NoColor = !cctx.Bool("color")
+		}
+
+		if cctx.NArg() != 1 {
+			return fmt.Errorf("must provide a CID")
+		}
+
+		cidStr := cctx.Args().First()
+		cid, err := cid.Parse(cidStr)
+		if err != nil {
+			return fmt.Errorf("invalid CID: %w", err)
+		}
+
+		marketsApi, closer, err := lcli.GetMarketsAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		shards, err := marketsApi.DagstoreLookupPieces(ctx, cid)
+		if err != nil {
+			return err
+		}
+
+		return printTableShards(shards)
+	},
+}

documentation/en/api-v0-methods-miner.md

@@ -23,6 +23,8 @@
   * [DagstoreInitializeAll](#DagstoreInitializeAll)
   * [DagstoreInitializeShard](#DagstoreInitializeShard)
   * [DagstoreListShards](#DagstoreListShards)
+  * [DagstoreLookupPieces](#DagstoreLookupPieces)
+  * [DagstorePieceIndexSize](#DagstorePieceIndexSize)
   * [DagstoreRecoverShard](#DagstoreRecoverShard)
 * [Deals](#Deals)
   * [DealsConsiderOfflineRetrievalDeals](#DealsConsiderOfflineRetrievalDeals)
@@ -444,6 +446,33 @@ Inputs: `null`
 
 Response: `null`
 
+### DagstoreLookupPieces
+DagstoreLookupPieces returns information about shards that contain the given CID.
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  {
+    "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+  }
+]
+```
+
+Response: `null`
+
+### DagstorePieceIndexSize
+DagstorePieceIndexSize returns the size of the piece index.
+
+
+Perms: admin
+
+Inputs: `null`
+
+Response: `9`
+
 ### DagstoreRecoverShard
 DagstoreRecoverShard attempts to recover a failed shard.
 
@@ -673,7 +702,7 @@ IndexerAnnounceDeal informs indexer nodes that a new deal was received,
 so they can download its index
 
 
-Perms: 
+Perms: admin
 
 Inputs:
 ```json

documentation/en/cli-lotus-miner.md

@@ -1036,6 +1036,8 @@ COMMANDS:
    recover-shard     Attempt to recover a shard in errored state
    initialize-all    Initialize all uninitialized shards, streaming results as they're produced; only shards for unsealed pieces are initialized by default
    gc                Garbage collect the dagstore
+   piece-index-size  Inspect the dagstore piece index size
+   lookup-pieces     Lookup pieces that a given CID belongs to
    help, h           Shows a list of commands or help for one command
 
 OPTIONS:
@@ -1115,6 +1117,34 @@ OPTIONS:
    
 ```
 
+### lotus-miner dagstore piece-index-size
+```
+NAME:
+   lotus-miner dagstore piece-index-size - Inspect the dagstore piece index size
+
+USAGE:
+   lotus-miner dagstore piece-index-size [command options] [arguments...]
+
+OPTIONS:
+   --color     use color in display output (default: depends on output being a TTY)
+   --help, -h  show help (default: false)
+   
+```
+
+### lotus-miner dagstore lookup-pieces
+```
+NAME:
+   lotus-miner dagstore lookup-pieces - Lookup pieces that a given CID belongs to
+
+USAGE:
+   lotus-miner dagstore lookup-pieces [command options] <cid>
+
+OPTIONS:
+   --color     use color in display output (default: depends on output being a TTY)
+   --help, -h  show help (default: false)
+   
+```
+
 ## lotus-miner index
 ```
 NAME:

node/impl/storminer.go

@@ -836,6 +836,57 @@ func (sm *StorageMinerAPI) IndexerAnnounceDeal(ctx context.Context, proposalCid
 	return sm.StorageProvider.AnnounceDealToIndexer(ctx, proposalCid)
 }
 
+func (sm *StorageMinerAPI) DagstorePieceIndexSize(ctx context.Context) (int64, error) {
+	if sm.DAGStore == nil {
+		return 0, fmt.Errorf("dagstore not available on this node")
+	}
+
+	res, err := sm.DAGStore.TopLevelIndex.Size()
+	if err != nil {
+		return 0, fmt.Errorf("failed to get dagstore piece index size: %w", err)
+	}
+
+	return res, nil
+}
+
+func (sm *StorageMinerAPI) DagstoreLookupPieces(ctx context.Context, cid cid.Cid) ([]api.DagstoreShardInfo, error) {
+	if sm.DAGStore == nil {
+		return nil, fmt.Errorf("dagstore not available on this node")
+	}
+
+	keys, err := sm.DAGStore.TopLevelIndex.GetShardsForMultihash(cid.Hash())
+	if err != nil {
+		return nil, err
+	}
+
+	var ret []api.DagstoreShardInfo
+
+	for _, k := range keys {
+		shard, err := sm.DAGStore.GetShardInfo(k)
+		if err != nil {
+			return nil, err
+		}
+
+		ret = append(ret, api.DagstoreShardInfo{
+			Key:   k.String(),
+			State: shard.ShardState.String(),
+			Error: func() string {
+				if shard.Error == nil {
+					return ""
+				}
+				return shard.Error.Error()
+			}(),
+		})
+	}
+
+	// order by key.
+	sort.SliceStable(ret, func(i, j int) bool {
+		return ret[i].Key < ret[j].Key
+	})
+
+	return ret, nil
+}
+
 func (sm *StorageMinerAPI) DealsList(ctx context.Context) ([]api.MarketDeal, error) {
 	return sm.listDeals(ctx)
 }