cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

WIP: retrieval pieces inspection command

Browse Source
This commit is contained in:
whyrusleeping 2020-07-27 10:59:30 -07:00 committed by Aayush Rajasekaran
parent 34bfafc7a7
commit 7fa4cd33f0
8 changed files with 254 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
api/api_storage.go
View File

@ -8,6 +8,7 @@ import (
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/piecestore"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/sector-storage/fsutil"
@ -88,6 +89,11 @@ type StorageMiner interface {
DealsSetConsiderOfflineRetrievalDeals(context.Context, bool) error
StorageAddLocal(ctx context.Context, path string) error
PiecesListPieces(ctx context.Context) ([]cid.Cid, error)
PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error)
PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error)
PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error)
}
type SealRes struct {

22
api/apistruct/struct.go
View File

@ -10,6 +10,7 @@ import (
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/piecestore"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/go-multistore"
@ -263,6 +264,11 @@ type StorageMinerStruct struct {
DealsSetPieceCidBlocklist func(context.Context, []cid.Cid) error `perm:"admin"`
StorageAddLocal func(ctx context.Context, path string) error `perm:"admin"`
PiecesListPieces func(ctx context.Context) ([]cid.Cid, error) `perm:"read"`
PiecesListCidInfos func(ctx context.Context) ([]cid.Cid, error) `perm:"read"`
PiecesGetPieceInfo func(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) `perm:"read"`
PiecesGetCIDInfo func(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) `perm:"read"`
}
}
@ -1039,6 +1045,22 @@ func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) e
return c.Internal.StorageAddLocal(ctx, path)
}
func (c *StorageMinerStruct) PiecesListPieces(ctx context.Context) ([]cid.Cid, error) {
return c.Internal.PiecesListPieces(ctx)
}
func (c *StorageMinerStruct) PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error) {
return c.Internal.PiecesListCidInfos(ctx)
}
func (c *StorageMinerStruct) PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) {
return c.Internal.PiecesGetPieceInfo(ctx, pieceCid)
}
func (c *StorageMinerStruct) PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) {
return c.Internal.PiecesGetCIDInfo(ctx, payloadCid)
}
// WorkerStruct
func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) {

48
cli/client.go
View File

@ -69,6 +69,7 @@ var clientCmd = &cli.Command{
WithCategory("retrieval", clientRetrieveCmd),
WithCategory("util", clientCommPCmd),
WithCategory("util", clientCarGenCmd),
WithCategory("util", clientInfoCmd),
},
}
@ -844,3 +845,50 @@ var clientGetDealCmd = &cli.Command{
return nil
},
}
var clientInfoCmd = &cli.Command{
Name: "info",
Usage: "Print storage market client information",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "client",
Usage: "specify storage client address",
},
},
Action: func(cctx *cli.Context) error {
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := ReqContext(cctx)
var addr address.Address
if clientFlag := cctx.String("client"); clientFlag != "" {
ca, err := address.NewFromString("client")
if err != nil {
return err
}
addr = ca
} else {
def, err := api.WalletDefaultAddress(ctx)
if err != nil {
return err
}
addr = def
}
balance, err := api.StateMarketBalance(ctx, addr, types.EmptyTSK)
if err != nil {
return err
}
fmt.Printf("Client Market Info:\n")
fmt.Printf("Locked Funds:\t%s\n", types.FIL(balance.Locked))
fmt.Printf("Escrowed Funds:\t%s\n", types.FIL(balance.Escrow))
return nil
},
}

7
cmd/lotus-shed/mpool.go
View File

@ -3,6 +3,7 @@ package main
import (
"fmt"
"github.com/filecoin-project/lotus/build"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/miner"
"github.com/urfave/cli/v2"
@ -43,8 +44,14 @@ var minerSelectMsgsCmd = &cli.Command{
return err
}
var totalGas int64
for _, f := range filtered {
totalGas += f.Message.GasLimit
}
fmt.Println("mempool input messages: ", len(msgs))
fmt.Println("filtered messages: ", len(filtered))
fmt.Printf("total gas limit of selected messages: %d / %d (%0.2f%%)\n", totalGas, build.BlockGasLimit, 100*float64(totalGas)/float64(build.BlockGasLimit))
return nil
},
}

1
cmd/lotus-storage-miner/main.go
View File

@ -41,6 +41,7 @@ func main() {
lcli.WithCategory("storage", provingCmd),
lcli.WithCategory("storage", storageCmd),
lcli.WithCategory("storage", sealingCmd),
lcli.WithCategory("retrieval", piecesCmd),
}
jaeger := tracing.SetupJaegerTracing("lotus")
defer func() {

135
cmd/lotus-storage-miner/pieces.go Normal file
View File

@ -0,0 +1,135 @@
package main
import (
"fmt"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
)
var piecesCmd = &cli.Command{
Name: "pieces",
Usage: "interact with the piecestore",
Description: "The piecestore is a database that tracks and manages data that is made available to the retrieval market",
Subcommands: []*cli.Command{
piecesListPiecesCmd,
piecesListCidInfosCmd,
piecesInfoCmd,
piecesCidInfoCmd,
},
}
var piecesListPiecesCmd = &cli.Command{
Name: "list-pieces",
Usage: "list registered pieces",
Action: func(cctx *cli.Context) error {
nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
pieceCids, err := nodeApi.PiecesListPieces(ctx)
if err != nil {
return err
}
for _, pc := range pieceCids {
fmt.Println(pc)
}
return nil
},
}
var piecesListCidInfosCmd = &cli.Command{
Name: "list-cids",
Usage: "list registered payload CIDs",
Action: func(cctx *cli.Context) error {
nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
cids, err := nodeApi.PiecesListCidInfos(ctx)
if err != nil {
return err
}
for _, c := range cids {
fmt.Println(c)
}
return nil
},
}
var piecesInfoCmd = &cli.Command{
Name: "piece-info",
Usage: "get registered information for a given piece CID",
Action: func(cctx *cli.Context) error {
if !cctx.Args().Present() {
return lcli.ShowHelp(cctx, fmt.Errorf("must specify piece cid"))
}
nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
c, err := cid.Decode(cctx.Args().First())
if err != nil {
return err
}
pi, err := nodeApi.PiecesGetPieceInfo(ctx, c)
if err != nil {
return err
}
fmt.Println("Piece: ", pi.PieceCID)
fmt.Println("Deals:\nDealID\tSectorID\tLength\tOffset")
for _, d := range pi.Deals {
fmt.Printf("%d\t%d\t%d\t%d\n", d.DealID, d.SectorID, d.Length, d.Offset)
}
return nil
},
}
var piecesCidInfoCmd = &cli.Command{
Name: "cid-info",
Usage: "get registered information for a given payload CID",
Action: func(cctx *cli.Context) error {
if !cctx.Args().Present() {
return lcli.ShowHelp(cctx, fmt.Errorf("must specify payload cid"))
}
nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
c, err := cid.Decode(cctx.Args().First())
if err != nil {
return err
}
ci, err := nodeApi.PiecesGetCIDInfo(ctx, c)
if err != nil {
return err
}
fmt.Println("Info for: ", ci.CID)
fmt.Printf("PieceCid\tOffset\tSize\n")
for _, loc := range ci.PieceBlockLocations {
fmt.Printf("%s\t%d\t%d\n", loc.PieceCID, loc.RelOffset, loc.BlockSize)
}
return nil
},
}

1
markets/storageadapter/provider.go
View File

@ -203,6 +203,7 @@ func (n *ProviderNodeAdapter) GetBalance(ctx context.Context, addr address.Addre
return utils.ToSharedBalance(bal), nil
}
// TODO: why doesnt this method take in a sector ID?
func (n *ProviderNodeAdapter) LocatePieceForDealWithinSector(ctx context.Context, dealID abi.DealID, encodedTs shared.TipSetToken) (sectorID abi.SectorNumber, offset abi.PaddedPieceSize, length abi.PaddedPieceSize, err error) {
refs, err := n.secb.GetRefs(dealID)
if err != nil {

38
node/impl/storminer.go
View File

@ -3,16 +3,18 @@ package impl
import (
"context"
"encoding/json"
"github.com/filecoin-project/sector-storage/fsutil"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
"net/http"
"os"
"strconv"
"time"
"github.com/filecoin-project/sector-storage/fsutil"
"github.com/filecoin-project/specs-actors/actors/abi/big"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/piecestore"
storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-jsonrpc/auth"
sectorstorage "github.com/filecoin-project/sector-storage"
@ -38,6 +40,7 @@ type StorageMinerAPI struct {
ProofsConfig *ffiwrapper.Config
SectorBlocks *sectorblocks.SectorBlocks
PieceStore piecestore.PieceStore
StorageProvider storagemarket.StorageProvider
Miner *storage.Miner
BlockMiner *miner.Miner
@ -368,4 +371,31 @@ func (sm *StorageMinerAPI) StorageAddLocal(ctx context.Context, path string) err
return sm.StorageMgr.AddLocalStorage(ctx, path)
}
func (sm *StorageMinerAPI) PiecesListPieces(ctx context.Context) ([]cid.Cid, error) {
panic("nyi")
//return sm.PieceStore.ListPieceInfoKeys(ctx)
}
func (sm *StorageMinerAPI) PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error) {
panic("nyi")
//return sm.PieceStore.ListCidInfoKeys(ctx)
}
func (sm *StorageMinerAPI) PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) {
pi, err := sm.PieceStore.GetPieceInfo(pieceCid)
if err != nil {
return nil, err
}
return &pi, nil
}
func (sm *StorageMinerAPI) PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) {
ci, err := sm.PieceStore.GetCIDInfo(payloadCid)
if err != nil {
return nil, err
}
return &ci, nil
}
var _ api.StorageMiner = &StorageMinerAPI{}