cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #2040 from jsign/jsign/minerretrieval

Browse Source 
Miner QueryOffer API & non local-discovery retrieval CLI support
This commit is contained in:
Aayush Rajasekaran 2020-06-16 16:00:40 -04:00 committed by GitHub
commit 4d0df5d599
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 77 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
api/api_full.go
View File

@ -118,6 +118,7 @@ type FullNode interface {
ClientListDeals(ctx context.Context) ([]DealInfo, error) ClientListDeals(ctx context.Context) ([]DealInfo, error)
ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error)
ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error)
ClientMinerQueryOffer(ctx context.Context, root cid.Cid, miner address.Address) (QueryOffer, error)
ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *FileRef) error ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *FileRef) error
ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error)
ClientCalcCommP(ctx context.Context, inpath string, miner address.Address) (*CommPRet, error) ClientCalcCommP(ctx context.Context, inpath string, miner address.Address) (*CommPRet, error)

27
api/apistruct/struct.go
View File

@ -106,17 +106,18 @@ type FullNodeStruct struct {
WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"`
WalletDelete func(context.Context, address.Address) error `perm:"write"` WalletDelete func(context.Context, address.Address) error `perm:"write"`
ClientImport func(ctx context.Context, ref api.FileRef) (cid.Cid, error) `perm:"admin"` ClientImport func(ctx context.Context, ref api.FileRef) (cid.Cid, error) `perm:"admin"`
ClientListImports func(ctx context.Context) ([]api.Import, error) `perm:"write"` ClientListImports func(ctx context.Context) ([]api.Import, error) `perm:"write"`
ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"`
ClientFindData func(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error) `perm:"read"` ClientFindData func(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error) `perm:"read"`
ClientStartDeal func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) `perm:"admin"` ClientMinerQueryOffer func(ctx context.Context, root cid.Cid, miner address.Address) (api.QueryOffer, error) `perm:"read"`
ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error) `perm:"read"` ClientStartDeal func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) `perm:"admin"`
ClientListDeals func(ctx context.Context) ([]api.DealInfo, error) `perm:"write"` ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error) `perm:"read"`
ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error `perm:"admin"` ClientListDeals func(ctx context.Context) ([]api.DealInfo, error) `perm:"write"`
ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) `perm:"read"` ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error `perm:"admin"`
ClientCalcCommP func(ctx context.Context, inpath string, miner address.Address) (*api.CommPRet, error) `perm:"read"` ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) `perm:"read"`
ClientGenCar func(ctx context.Context, ref api.FileRef, outpath string) error `perm:"write"` ClientCalcCommP func(ctx context.Context, inpath string, miner address.Address) (*api.CommPRet, error) `perm:"read"`
ClientGenCar func(ctx context.Context, ref api.FileRef, outpath string) error `perm:"write"`
StateNetworkName func(context.Context) (dtypes.NetworkName, error) `perm:"read"` StateNetworkName func(context.Context) (dtypes.NetworkName, error) `perm:"read"`
StateMinerSectors func(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"` StateMinerSectors func(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"`
@ -318,6 +319,10 @@ func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]ap
return c.Internal.ClientFindData(ctx, root) return c.Internal.ClientFindData(ctx, root)
} }
func (c *FullNodeStruct) ClientMinerQueryOffer(ctx context.Context, root cid.Cid, miner address.Address) (api.QueryOffer, error) {
return c.Internal.ClientMinerQueryOffer(ctx, root, miner)
}
func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) { func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) {
return c.Internal.ClientStartDeal(ctx, params) return c.Internal.ClientStartDeal(ctx, params)
} }

42
cli/client.go
View File

@ -19,6 +19,7 @@ import (
"github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/specs-actors/actors/builtin/market"
"github.com/filecoin-project/lotus/api"
lapi "github.com/filecoin-project/lotus/api" lapi "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types"
) )
@ -391,6 +392,10 @@ var clientRetrieveCmd = &cli.Command{
Name: "car", Name: "car",
Usage: "export to a car file instead of a regular file", Usage: "export to a car file instead of a regular file",
}, },
&cli.StringFlag{
Name: "miner",
Usage: "miner address for retrieval, if not present it'll use local discovery",
},
}, },
Action: func(cctx *cli.Context) error { Action: func(cctx *cli.Context) error {
if cctx.NArg() != 2 { if cctx.NArg() != 2 {
@ -398,7 +403,7 @@ var clientRetrieveCmd = &cli.Command{
return nil return nil
} }
api, closer, err := GetFullNodeAPI(cctx) fapi, closer, err := GetFullNodeAPI(cctx)
if err != nil { if err != nil {
return err return err
} }
@ -409,7 +414,7 @@ var clientRetrieveCmd = &cli.Command{
if cctx.String("address") != "" { if cctx.String("address") != "" {
payer, err = address.NewFromString(cctx.String("address")) payer, err = address.NewFromString(cctx.String("address"))
} else { } else {
payer, err = api.WalletDefaultAddress(ctx) payer, err = fapi.WalletDefaultAddress(ctx)
} }
if err != nil { if err != nil {
return err return err
@ -432,23 +437,36 @@ var clientRetrieveCmd = &cli.Command{
return nil return nil
}*/ // TODO: fix }*/ // TODO: fix
offers, err := api.ClientFindData(ctx, file) var offer api.QueryOffer
if err != nil { minerStrAddr := cctx.String("miner")
return err if minerStrAddr == "" { // Local discovery
} offers, err := fapi.ClientFindData(ctx, file)
if err != nil {
return err
}
// TODO: parse offer strings from `client find`, make this smarter // TODO: parse offer strings from `client find`, make this smarter
if len(offers) < 1 {
if len(offers) < 1 { fmt.Println("Failed to find file")
fmt.Println("Failed to find file") return nil
return nil }
offer = offers[0]
} else { // Directed retrieval
minerAddr, err := address.NewFromString(minerStrAddr)
if err != nil {
return err
}
offer, err = fapi.ClientMinerQueryOffer(ctx, file, minerAddr)
if err != nil {
return err
}
} }
ref := &lapi.FileRef{ ref := &lapi.FileRef{
Path: cctx.Args().Get(1), Path: cctx.Args().Get(1),
IsCAR: cctx.Bool("car"), IsCAR: cctx.Bool("car"),
} }
if err := api.ClientRetrieve(ctx, offers[0].Order(payer), ref); err != nil { if err := fapi.ClientRetrieve(ctx, offer.Order(payer), ref); err != nil {
return xerrors.Errorf("Retrieval Failed: %w", err) return xerrors.Errorf("Retrieval Failed: %w", err)
} }

44
node/impl/client/client.go
View File

@ -201,25 +201,41 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffe
out := make([]api.QueryOffer, len(peers)) out := make([]api.QueryOffer, len(peers))
for k, p := range peers { for k, p := range peers {
queryResponse, err := a.Retrieval.Query(ctx, p, root, retrievalmarket.QueryParams{}) out[k] = a.makeRetrievalQuery(ctx, p, root, retrievalmarket.QueryParams{})
if err != nil {
out[k] = api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID}
} else {
out[k] = api.QueryOffer{
Root: root,
Size: queryResponse.Size,
MinPrice: queryResponse.PieceRetrievalPrice(),
PaymentInterval: queryResponse.MaxPaymentInterval,
PaymentIntervalIncrease: queryResponse.MaxPaymentIntervalIncrease,
Miner: queryResponse.PaymentAddress, // TODO: check
MinerPeerID: p.ID,
}
}
} }
return out, nil return out, nil
} }
func (a *API) ClientMinerQueryOffer(ctx context.Context, payload cid.Cid, miner address.Address) (api.QueryOffer, error) {
mi, err := a.StateMinerInfo(ctx, miner, types.EmptyTSK)
if err != nil {
return api.QueryOffer{}, err
}
rp := retrievalmarket.RetrievalPeer{
Address: miner,
ID: mi.PeerId,
}
return a.makeRetrievalQuery(ctx, rp, payload, retrievalmarket.QueryParams{}), nil
}
func (a *API) makeRetrievalQuery(ctx context.Context, rp retrievalmarket.RetrievalPeer, payload cid.Cid, qp retrievalmarket.QueryParams) api.QueryOffer {
queryResponse, err := a.Retrieval.Query(ctx, rp, payload, qp)
if err != nil {
return api.QueryOffer{Err: err.Error(), Miner: rp.Address, MinerPeerID: rp.ID}
}
return api.QueryOffer{
Root: payload,
Size: queryResponse.Size,
MinPrice: queryResponse.PieceRetrievalPrice(),
PaymentInterval: queryResponse.MaxPaymentInterval,
PaymentIntervalIncrease: queryResponse.MaxPaymentIntervalIncrease,
Miner: queryResponse.PaymentAddress, // TODO: check
MinerPeerID: rp.ID,
}
}
func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (cid.Cid, error) { func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (cid.Cid, error) {
bufferedDS := ipld.NewBufferedDAG(ctx, a.LocalDAG) bufferedDS := ipld.NewBufferedDAG(ctx, a.LocalDAG)