cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #5917 from filecoin-project/feat/local-retrieval

Browse Source 
Local retrieval support
This commit is contained in:
Łukasz Magiera 2021-04-02 13:07:00 +02:00 committed by GitHub
commit eede19fb0b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 183 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
api/api_full.go
View File

@ -884,7 +884,7 @@ func (o *QueryOffer) Order(client address.Address) RetrievalOrder {
Client: client,
Miner: o.Miner,
MinerPeer: o.MinerPeer,
MinerPeer: &o.MinerPeer,
}
}
@ -903,6 +903,8 @@ type RetrievalOrder struct {
Root cid.Cid
Piece *cid.Cid
Size uint64
LocalStore *multistore.StoreID // if specified, get data from local store
// TODO: support offset
Total types.BigInt
UnsealPrice types.BigInt
@ -910,7 +912,7 @@ type RetrievalOrder struct {
PaymentIntervalIncrease uint64
Client address.Address
Miner address.Address
MinerPeer retrievalmarket.RetrievalPeer
MinerPeer *retrievalmarket.RetrievalPeer
}
type InvocResult struct {

BIN
build/openrpc/full.json.gz
View File

Binary file not shown.

55
cli/client.go
View File

@ -1001,7 +1001,7 @@ var clientFindCmd = &cli.Command{
},
}
const DefaultMaxRetrievePrice = 1
const DefaultMaxRetrievePrice = "0.01"
var clientRetrieveCmd = &cli.Command{
Name: "retrieve",
@ -1022,12 +1022,15 @@ var clientRetrieveCmd = &cli.Command{
},
&cli.StringFlag{
Name: "maxPrice",
Usage: fmt.Sprintf("maximum price the client is willing to consider (default: %d FIL)", DefaultMaxRetrievePrice),
Usage: fmt.Sprintf("maximum price the client is willing to consider (default: %s FIL)", DefaultMaxRetrievePrice),
},
&cli.StringFlag{
Name: "pieceCid",
Usage: "require data to be retrieved from a specific Piece CID",
},
&cli.BoolFlag{
Name: "allow-local",
},
},
Action: func(cctx *cli.Context) error {
if cctx.NArg() != 2 {
@ -1057,18 +1060,6 @@ var clientRetrieveCmd = &cli.Command{
return err
}
// Check if we already have this data locally
/*has, err := api.ClientHasLocal(ctx, file)
if err != nil {
return err
}
if has {
fmt.Println("Success: Already in local storage")
return nil
}*/ // TODO: fix
var pieceCid *cid.Cid
if cctx.String("pieceCid") != "" {
parsed, err := cid.Parse(cctx.String("pieceCid"))
@ -1078,6 +1069,28 @@ var clientRetrieveCmd = &cli.Command{
pieceCid = &parsed
}
var order *lapi.RetrievalOrder
if cctx.Bool("allow-local") {
imports, err := fapi.ClientListImports(ctx)
if err != nil {
return err
}
for _, i := range imports {
if i.Root != nil && i.Root.Equals(file) {
order = &lapi.RetrievalOrder{
Root: file,
LocalStore: &i.Key,
Total: big.Zero(),
UnsealPrice: big.Zero(),
}
break
}
}
}
if order == nil {
var offer api.QueryOffer
minerStrAddr := cctx.String("miner")
if minerStrAddr == "" { // Local discovery
@ -1121,26 +1134,28 @@ var clientRetrieveCmd = &cli.Command{
return fmt.Errorf("The received offer errored: %s", offer.Err)
}
maxPrice := types.FromFil(DefaultMaxRetrievePrice)
maxPrice := types.MustParseFIL(DefaultMaxRetrievePrice)
if cctx.String("maxPrice") != "" {
maxPriceFil, err := types.ParseFIL(cctx.String("maxPrice"))
maxPrice, err = types.ParseFIL(cctx.String("maxPrice"))
if err != nil {
return xerrors.Errorf("parsing maxPrice: %w", err)
}
maxPrice = types.BigInt(maxPriceFil)
}
if offer.MinPrice.GreaterThan(maxPrice) {
if offer.MinPrice.GreaterThan(big.Int(maxPrice)) {
return xerrors.Errorf("failed to find offer satisfying maxPrice: %s", maxPrice)
}
o := offer.Order(payer)
order = &o
}
ref := &lapi.FileRef{
Path: cctx.Args().Get(1),
IsCAR: cctx.Bool("car"),
}
updates, err := fapi.ClientRetrieveWithEvents(ctx, offer.Order(payer), ref)
updates, err := fapi.ClientRetrieveWithEvents(ctx, *order, ref)
if err != nil {
return xerrors.Errorf("error setting up retrieval: %w", err)
}

2
documentation/en/api-methods.md
View File

@ -1383,6 +1383,7 @@ Inputs:
},
"Piece": null,
"Size": 42,
"LocalStore": 12,
"Total": "0",
"UnsealPrice": "0",
"PaymentInterval": 42,
@ -1436,6 +1437,7 @@ Inputs:
},
"Piece": null,
"Size": 42,
"LocalStore": 12,
"Total": "0",
"UnsealPrice": "0",
"PaymentInterval": 42,

39
node/impl/client/client.go
View File

@ -57,6 +57,7 @@ import (
"github.com/filecoin-project/lotus/node/impl/paych"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/filecoin-project/lotus/node/repo/retrievalstoremgr"
)
var DefaultHashFunction = uint64(mh.BLAKE2B_MIN + 31)
@ -77,6 +78,7 @@ type API struct {
Chain *store.ChainStore
Imports dtypes.ClientImportMgr
Mds dtypes.ClientMultiDstore
CombinedBstore dtypes.ClientBlockstore // TODO: try to remove
RetrievalStoreMgr dtypes.ClientRetrievalStoreManager
@ -582,14 +584,17 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
}
}
if order.MinerPeer.ID == "" {
var store retrievalstoremgr.RetrievalStore
if order.LocalStore == nil {
if order.MinerPeer == nil || order.MinerPeer.ID == "" {
mi, err := a.StateMinerInfo(ctx, order.Miner, types.EmptyTSK)
if err != nil {
finish(err)
return
}
order.MinerPeer = retrievalmarket.RetrievalPeer{
order.MinerPeer = &retrievalmarket.RetrievalPeer{
ID: *mi.PeerId,
Address: order.Miner,
}
@ -616,7 +621,7 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
return
}
store, err := a.RetrievalStoreMgr.NewStore()
store, err = a.RetrievalStoreMgr.NewStore()
if err != nil {
finish(xerrors.Errorf("Error setting up new store: %w", err))
return
@ -645,7 +650,7 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
order.Root,
params,
order.Total,
order.MinerPeer,
*order.MinerPeer,
order.Client,
order.Miner,
store.StoreID())
@ -663,6 +668,19 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
finish(xerrors.Errorf("Retrieve: %w", err))
return
}
} else {
// local retrieval
st, err := ((*multistore.MultiStore)(a.Mds)).Get(*order.LocalStore)
if err != nil {
finish(xerrors.Errorf("Retrieve: %w", err))
return
}
store = &multiStoreRetrievalStore{
storeID: *order.LocalStore,
store: st,
}
}
// If ref is nil, it only fetches the data into the configured blockstore.
if ref == nil {
@ -701,6 +719,19 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
return
}
type multiStoreRetrievalStore struct {
storeID multistore.StoreID
store *multistore.Store
}
func (mrs *multiStoreRetrievalStore) StoreID() *multistore.StoreID {
return &mrs.storeID
}
func (mrs *multiStoreRetrievalStore) DAGService() ipld.DAGService {
return mrs.store.DAG
}
func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error) {
mi, err := a.StateMinerInfo(ctx, miner, types.EmptyTSK)
if err != nil {