WIP: retrieval pieces inspection command

api/api_storage.go

@@ -8,6 +8,7 @@ import (
 	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-fil-markets/piecestore"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/sector-storage/fsutil"
@@ -88,6 +89,11 @@ type StorageMiner interface {
 	DealsSetConsiderOfflineRetrievalDeals(context.Context, bool) error
 
 	StorageAddLocal(ctx context.Context, path string) error
+
+	PiecesListPieces(ctx context.Context) ([]cid.Cid, error)
+	PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error)
+	PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error)
+	PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error)
 }
 
 type SealRes struct {

api/apistruct/struct.go

@@ -10,6 +10,7 @@ import (
 	"github.com/libp2p/go-libp2p-core/peer"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-fil-markets/piecestore"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-jsonrpc/auth"
 	"github.com/filecoin-project/go-multistore"
@@ -263,6 +264,11 @@ type StorageMinerStruct struct {
 		DealsSetPieceCidBlocklist             func(context.Context, []cid.Cid) error                            `perm:"admin"`
 
 		StorageAddLocal func(ctx context.Context, path string) error `perm:"admin"`
+
+		PiecesListPieces   func(ctx context.Context) ([]cid.Cid, error)                               `perm:"read"`
+		PiecesListCidInfos func(ctx context.Context) ([]cid.Cid, error)                               `perm:"read"`
+		PiecesGetPieceInfo func(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) `perm:"read"`
+		PiecesGetCIDInfo   func(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) `perm:"read"`
 	}
 }
 
@@ -1039,6 +1045,22 @@ func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) e
 	return c.Internal.StorageAddLocal(ctx, path)
 }
 
+func (c *StorageMinerStruct) PiecesListPieces(ctx context.Context) ([]cid.Cid, error) {
+	return c.Internal.PiecesListPieces(ctx)
+}
+
+func (c *StorageMinerStruct) PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error) {
+	return c.Internal.PiecesListCidInfos(ctx)
+}
+
+func (c *StorageMinerStruct) PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) {
+	return c.Internal.PiecesGetPieceInfo(ctx, pieceCid)
+}
+
+func (c *StorageMinerStruct) PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) {
+	return c.Internal.PiecesGetCIDInfo(ctx, payloadCid)
+}
+
 // WorkerStruct
 
 func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) {

cli/client.go

@@ -69,6 +69,7 @@ var clientCmd = &cli.Command{
 		WithCategory("retrieval", clientRetrieveCmd),
 		WithCategory("util", clientCommPCmd),
 		WithCategory("util", clientCarGenCmd),
+		WithCategory("util", clientInfoCmd),
 	},
 }
 
@@ -844,3 +845,50 @@ var clientGetDealCmd = &cli.Command{
 		return nil
 	},
 }
+
+var clientInfoCmd = &cli.Command{
+	Name:  "info",
+	Usage: "Print storage market client information",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "client",
+			Usage: "specify storage client address",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		var addr address.Address
+		if clientFlag := cctx.String("client"); clientFlag != "" {
+			ca, err := address.NewFromString("client")
+			if err != nil {
+				return err
+			}
+
+			addr = ca
+		} else {
+			def, err := api.WalletDefaultAddress(ctx)
+			if err != nil {
+				return err
+			}
+			addr = def
+		}
+
+		balance, err := api.StateMarketBalance(ctx, addr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Client Market Info:\n")
+
+		fmt.Printf("Locked Funds:\t%s\n", types.FIL(balance.Locked))
+		fmt.Printf("Escrowed Funds:\t%s\n", types.FIL(balance.Escrow))
+
+		return nil
+	},
+}

cmd/lotus-shed/mpool.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 
+	"github.com/filecoin-project/lotus/build"
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/miner"
 	"github.com/urfave/cli/v2"
@@ -43,8 +44,14 @@ var minerSelectMsgsCmd = &cli.Command{
 			return err
 		}
 
+		var totalGas int64
+		for _, f := range filtered {
+			totalGas += f.Message.GasLimit
+		}
+
 		fmt.Println("mempool input messages: ", len(msgs))
 		fmt.Println("filtered messages: ", len(filtered))
+		fmt.Printf("total gas limit of selected messages: %d / %d (%0.2f%%)\n", totalGas, build.BlockGasLimit, 100*float64(totalGas)/float64(build.BlockGasLimit))
 		return nil
 	},
 }

cmd/lotus-storage-miner/main.go

@@ -41,6 +41,7 @@ func main() {
 		lcli.WithCategory("storage", provingCmd),
 		lcli.WithCategory("storage", storageCmd),
 		lcli.WithCategory("storage", sealingCmd),
+		lcli.WithCategory("retrieval", piecesCmd),
 	}
 	jaeger := tracing.SetupJaegerTracing("lotus")
 	defer func() {

cmd/lotus-storage-miner/pieces.go

@@ -0,0 +1,135 @@
+package main
+
+import (
+	"fmt"
+
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/ipfs/go-cid"
+	"github.com/urfave/cli/v2"
+)
+
+var piecesCmd = &cli.Command{
+	Name:        "pieces",
+	Usage:       "interact with the piecestore",
+	Description: "The piecestore is a database that tracks and manages data that is made available to the retrieval market",
+	Subcommands: []*cli.Command{
+		piecesListPiecesCmd,
+		piecesListCidInfosCmd,
+		piecesInfoCmd,
+		piecesCidInfoCmd,
+	},
+}
+
+var piecesListPiecesCmd = &cli.Command{
+	Name:  "list-pieces",
+	Usage: "list registered pieces",
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		pieceCids, err := nodeApi.PiecesListPieces(ctx)
+		if err != nil {
+			return err
+		}
+
+		for _, pc := range pieceCids {
+			fmt.Println(pc)
+		}
+		return nil
+	},
+}
+
+var piecesListCidInfosCmd = &cli.Command{
+	Name:  "list-cids",
+	Usage: "list registered payload CIDs",
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		cids, err := nodeApi.PiecesListCidInfos(ctx)
+		if err != nil {
+			return err
+		}
+
+		for _, c := range cids {
+			fmt.Println(c)
+		}
+		return nil
+	},
+}
+
+var piecesInfoCmd = &cli.Command{
+	Name:  "piece-info",
+	Usage: "get registered information for a given piece CID",
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Args().Present() {
+			return lcli.ShowHelp(cctx, fmt.Errorf("must specify piece cid"))
+		}
+
+		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		c, err := cid.Decode(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		pi, err := nodeApi.PiecesGetPieceInfo(ctx, c)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("Piece: ", pi.PieceCID)
+		fmt.Println("Deals:\nDealID\tSectorID\tLength\tOffset")
+		for _, d := range pi.Deals {
+			fmt.Printf("%d\t%d\t%d\t%d\n", d.DealID, d.SectorID, d.Length, d.Offset)
+		}
+		return nil
+	},
+}
+
+var piecesCidInfoCmd = &cli.Command{
+	Name:  "cid-info",
+	Usage: "get registered information for a given payload CID",
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Args().Present() {
+			return lcli.ShowHelp(cctx, fmt.Errorf("must specify payload cid"))
+		}
+
+		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		c, err := cid.Decode(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		ci, err := nodeApi.PiecesGetCIDInfo(ctx, c)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("Info for: ", ci.CID)
+		fmt.Printf("PieceCid\tOffset\tSize\n")
+		for _, loc := range ci.PieceBlockLocations {
+			fmt.Printf("%s\t%d\t%d\n", loc.PieceCID, loc.RelOffset, loc.BlockSize)
+		}
+		return nil
+	},
+}

markets/storageadapter/provider.go

@@ -203,6 +203,7 @@ func (n *ProviderNodeAdapter) GetBalance(ctx context.Context, addr address.Addre
 	return utils.ToSharedBalance(bal), nil
 }
 
+// TODO: why doesnt this method take in a sector ID?
 func (n *ProviderNodeAdapter) LocatePieceForDealWithinSector(ctx context.Context, dealID abi.DealID, encodedTs shared.TipSetToken) (sectorID abi.SectorNumber, offset abi.PaddedPieceSize, length abi.PaddedPieceSize, err error) {
 	refs, err := n.secb.GetRefs(dealID)
 	if err != nil {

node/impl/storminer.go

@@ -3,16 +3,18 @@ package impl
 import (
 	"context"
 	"encoding/json"
-	"github.com/filecoin-project/sector-storage/fsutil"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
-	"github.com/ipfs/go-cid"
-	"golang.org/x/xerrors"
 	"net/http"
 	"os"
 	"strconv"
 	"time"
 
+	"github.com/filecoin-project/sector-storage/fsutil"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/ipfs/go-cid"
+	"golang.org/x/xerrors"
+
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-fil-markets/piecestore"
 	storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-jsonrpc/auth"
 	sectorstorage "github.com/filecoin-project/sector-storage"
@@ -38,6 +40,7 @@ type StorageMinerAPI struct {
 	ProofsConfig *ffiwrapper.Config
 	SectorBlocks *sectorblocks.SectorBlocks
 
+	PieceStore      piecestore.PieceStore
 	StorageProvider storagemarket.StorageProvider
 	Miner           *storage.Miner
 	BlockMiner      *miner.Miner
@@ -368,4 +371,31 @@ func (sm *StorageMinerAPI) StorageAddLocal(ctx context.Context, path string) err
 	return sm.StorageMgr.AddLocalStorage(ctx, path)
 }
 
+func (sm *StorageMinerAPI) PiecesListPieces(ctx context.Context) ([]cid.Cid, error) {
+	panic("nyi")
+	//return sm.PieceStore.ListPieceInfoKeys(ctx)
+}
+
+func (sm *StorageMinerAPI) PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error) {
+	panic("nyi")
+	//return sm.PieceStore.ListCidInfoKeys(ctx)
+}
+
+func (sm *StorageMinerAPI) PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) {
+	pi, err := sm.PieceStore.GetPieceInfo(pieceCid)
+	if err != nil {
+		return nil, err
+	}
+	return &pi, nil
+}
+
+func (sm *StorageMinerAPI) PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) {
+	ci, err := sm.PieceStore.GetCIDInfo(payloadCid)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ci, nil
+}
+
 var _ api.StorageMiner = &StorageMinerAPI{}