cleanup: Lotus client: remove markets and deal-making from Lotus Client (#11999)

* remove client CLI

* remove markets CLI from miner

* remove markets from all CLI

* remove client API

* update go mod

* changes as per review
This commit is contained in:
Aarsh Shah
2024-06-05 09:56:25 +04:00
committed by GitHub
parent c9c070727a
commit 469960ce0e
67 changed files with 491 additions and 18068 deletions
-1
View File
@@ -10,7 +10,6 @@ var Commands = []*cli.Command{
lcli.WithCategory("basic", lcli.SendCmd),
lcli.WithCategory("basic", lcli.WalletCmd),
lcli.WithCategory("basic", lcli.InfoCmd),
lcli.WithCategory("basic", lcli.ClientCmd),
lcli.WithCategory("basic", lcli.MultisigCmd),
lcli.WithCategory("basic", lcli.FilplusCmd),
lcli.WithCategory("basic", lcli.PaychCmd),
-2468
View File
File diff suppressed because it is too large Load Diff
-550
View File
@@ -1,550 +0,0 @@
package cli
import (
"bytes"
"context"
"fmt"
"io"
"os"
"sort"
"strings"
"time"
"github.com/ipfs/boxo/blockservice"
offline "github.com/ipfs/boxo/exchange/offline"
"github.com/ipfs/boxo/ipld/merkledag"
"github.com/ipfs/go-cid"
carv2 "github.com/ipld/go-car/v2"
"github.com/ipld/go-car/v2/blockstore"
"github.com/ipld/go-ipld-prime"
"github.com/ipld/go-ipld-prime/codec/dagjson"
basicnode "github.com/ipld/go-ipld-prime/node/basic"
"github.com/ipld/go-ipld-prime/traversal"
"github.com/ipld/go-ipld-prime/traversal/selector"
"github.com/ipld/go-ipld-prime/traversal/selector/builder"
selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
textselector "github.com/ipld/go-ipld-selector-text-lite"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-state-types/big"
lapi "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/types"
cliutil "github.com/filecoin-project/lotus/cli/util"
"github.com/filecoin-project/lotus/markets/utils"
"github.com/filecoin-project/lotus/node/repo"
)
const DefaultMaxRetrievePrice = "0"
func retrieve(ctx context.Context, cctx *cli.Context, fapi lapi.FullNode, sel *lapi.Selector, printf func(string, ...interface{})) (*lapi.ExportRef, error) {
var payer address.Address
var err error
if cctx.String("from") != "" {
payer, err = address.NewFromString(cctx.String("from"))
} else {
payer, err = fapi.WalletDefaultAddress(ctx)
}
if err != nil {
return nil, err
}
file, err := cid.Parse(cctx.Args().Get(0))
if err != nil {
return nil, err
}
var pieceCid *cid.Cid
if cctx.String("pieceCid") != "" {
parsed, err := cid.Parse(cctx.String("pieceCid"))
if err != nil {
return nil, err
}
pieceCid = &parsed
}
var eref *lapi.ExportRef
if cctx.Bool("allow-local") {
imports, err := fapi.ClientListImports(ctx)
if err != nil {
return nil, err
}
for _, i := range imports {
if i.Root != nil && i.Root.Equals(file) {
eref = &lapi.ExportRef{
Root: file,
FromLocalCAR: i.CARPath,
}
break
}
}
}
// no local found, so make a retrieval
if eref == nil {
var offer lapi.QueryOffer
minerStrAddr := cctx.String("provider")
if minerStrAddr == "" { // Local discovery
offers, err := fapi.ClientFindData(ctx, file, pieceCid)
var cleaned []lapi.QueryOffer
// filter out offers that errored
for _, o := range offers {
if o.Err == "" {
cleaned = append(cleaned, o)
}
}
offers = cleaned
// sort by price low to high
sort.Slice(offers, func(i, j int) bool {
return offers[i].MinPrice.LessThan(offers[j].MinPrice)
})
if err != nil {
return nil, err
}
// TODO: parse offer strings from `client find`, make this smarter
if len(offers) < 1 {
fmt.Println("Failed to find file")
return nil, nil
}
offer = offers[0]
} else { // Directed retrieval
minerAddr, err := address.NewFromString(minerStrAddr)
if err != nil {
return nil, err
}
offer, err = fapi.ClientMinerQueryOffer(ctx, minerAddr, file, pieceCid)
if err != nil {
return nil, err
}
}
if offer.Err != "" {
return nil, fmt.Errorf("offer error: %s", offer.Err)
}
maxPrice := types.MustParseFIL(DefaultMaxRetrievePrice)
if cctx.String("maxPrice") != "" {
maxPrice, err = types.ParseFIL(cctx.String("maxPrice"))
if err != nil {
return nil, xerrors.Errorf("parsing maxPrice: %w", err)
}
}
if offer.MinPrice.GreaterThan(big.Int(maxPrice)) {
return nil, xerrors.Errorf("failed to find offer satisfying maxPrice: %s. Try increasing maxPrice", maxPrice)
}
o := offer.Order(payer)
o.DataSelector = sel
subscribeEvents, err := fapi.ClientGetRetrievalUpdates(ctx)
if err != nil {
return nil, xerrors.Errorf("error setting up retrieval updates: %w", err)
}
retrievalRes, err := fapi.ClientRetrieve(ctx, o)
if err != nil {
return nil, xerrors.Errorf("error setting up retrieval: %w", err)
}
start := time.Now()
readEvents:
for {
var evt lapi.RetrievalInfo
select {
case <-ctx.Done():
return nil, xerrors.New("Retrieval Timed Out")
case evt = <-subscribeEvents:
if evt.ID != retrievalRes.DealID {
// we can't check the deal ID ahead of time because:
// 1. We need to subscribe before retrieving.
// 2. We won't know the deal ID until after retrieving.
continue
}
}
event := "New"
if evt.Event != nil {
event = retrievalmarket.ClientEvents[*evt.Event]
}
printf("Recv %s, Paid %s, %s (%s), %s [%d|%d]\n",
types.SizeStr(types.NewInt(evt.BytesReceived)),
types.FIL(evt.TotalPaid),
strings.TrimPrefix(event, "ClientEvent"),
strings.TrimPrefix(retrievalmarket.DealStatuses[evt.Status], "DealStatus"),
time.Now().Sub(start).Truncate(time.Millisecond),
evt.ID,
types.NewInt(evt.BytesReceived),
)
switch evt.Status {
case retrievalmarket.DealStatusCompleted:
break readEvents
case retrievalmarket.DealStatusRejected:
return nil, xerrors.Errorf("Retrieval Proposal Rejected: %s", evt.Message)
case retrievalmarket.DealStatusCancelled:
return nil, xerrors.Errorf("Retrieval Proposal Cancelled: %s", evt.Message)
case
retrievalmarket.DealStatusDealNotFound,
retrievalmarket.DealStatusErrored:
return nil, xerrors.Errorf("Retrieval Error: %s", evt.Message)
}
}
eref = &lapi.ExportRef{
Root: file,
DealID: retrievalRes.DealID,
}
}
return eref, nil
}
var retrFlagsCommon = []cli.Flag{
&cli.StringFlag{
Name: "from",
Usage: "address to send transactions from",
},
&cli.StringFlag{
Name: "provider",
Usage: "provider to use for retrieval, if not present it'll use local discovery",
Aliases: []string{"miner"},
},
&cli.StringFlag{
Name: "maxPrice",
Usage: fmt.Sprintf("maximum price the client is willing to consider (default: %s FIL)", DefaultMaxRetrievePrice),
},
&cli.StringFlag{
Name: "pieceCid",
Usage: "require data to be retrieved from a specific Piece CID",
},
&cli.BoolFlag{
Name: "allow-local",
// todo: default to true?
},
}
var clientRetrieveCmd = &cli.Command{
Name: "retrieve",
Usage: "Retrieve data from network",
ArgsUsage: "[dataCid outputPath]",
Description: `Retrieve data from the Filecoin network.
The retrieve command will attempt to find a provider make a retrieval deal with
them. In case a provider can't be found, it can be specified with the --provider
flag.
By default the data will be interpreted as DAG-PB UnixFSv1 File. Alternatively
a CAR file containing the raw IPLD graph can be exported by setting the --car
flag.
Partial Retrieval:
The --data-selector flag can be used to specify a sub-graph to fetch. The
selector can be specified as either IPLD datamodel text-path selector, or IPLD
json selector.
In case of unixfs retrieval, the selector must point at a single root node, and
match the entire graph under that node.
In case of CAR retrieval, the selector must have one common "sub-root" node.
Examples:
- Retrieve a file by CID
$ lotus client retrieve Qm... my-file.txt
- Retrieve a file by CID from f0123
$ lotus client retrieve --provider f0123 Qm... my-file.txt
- Retrieve a first file from a specified directory
$ lotus client retrieve --data-selector /Links/0/Hash Qm... my-file.txt
`,
Flags: append([]cli.Flag{
&cli.BoolFlag{
Name: "car",
Usage: "Export to a car file instead of a regular file",
},
&cli.StringFlag{
Name: "data-selector",
Aliases: []string{"datamodel-path-selector"},
Usage: "IPLD datamodel text-path selector, or IPLD json selector",
},
&cli.BoolFlag{
Name: "car-export-merkle-proof",
Usage: "(requires --data-selector and --car) Export data-selector merkle proof",
},
}, retrFlagsCommon...),
Action: func(cctx *cli.Context) error {
if cctx.NArg() != 2 {
return IncorrectNumArgs(cctx)
}
if cctx.Bool("car-export-merkle-proof") {
if !cctx.Bool("car") || !cctx.IsSet("data-selector") {
return ShowHelp(cctx, fmt.Errorf("--car-export-merkle-proof requires --car and --data-selector"))
}
}
fapi, closer, err := GetFullNodeAPIV1(cctx)
if err != nil {
return err
}
defer closer()
ctx := ReqContext(cctx)
afmt := NewAppFmt(cctx.App)
var s *lapi.Selector
if sel := lapi.Selector(cctx.String("data-selector")); sel != "" {
s = &sel
}
eref, err := retrieve(ctx, cctx, fapi, s, afmt.Printf)
if err != nil {
return err
}
if eref == nil {
return xerrors.Errorf("failed to find providers")
}
if s != nil {
eref.DAGs = append(eref.DAGs, lapi.DagSpec{DataSelector: s, ExportMerkleProof: cctx.Bool("car-export-merkle-proof")})
}
err = fapi.ClientExport(ctx, *eref, lapi.FileRef{
Path: cctx.Args().Get(1),
IsCAR: cctx.Bool("car"),
})
if err != nil {
return err
}
afmt.Println("Success")
return nil
},
}
var clientRetrieveCatCmd = &cli.Command{
Name: "cat",
Usage: "Show data from network",
ArgsUsage: "[dataCid]",
Flags: append([]cli.Flag{
&cli.BoolFlag{
Name: "ipld",
Usage: "list IPLD datamodel links",
},
&cli.StringFlag{
Name: "data-selector",
Usage: "IPLD datamodel text-path selector, or IPLD json selector",
},
}, retrFlagsCommon...),
Action: func(cctx *cli.Context) error {
if cctx.NArg() != 1 {
return IncorrectNumArgs(cctx)
}
ainfo, err := GetAPIInfo(cctx, repo.FullNode)
if err != nil {
return xerrors.Errorf("could not get API info: %w", err)
}
fapi, closer, err := GetFullNodeAPIV1(cctx)
if err != nil {
return err
}
defer closer()
ctx := ReqContext(cctx)
afmt := NewAppFmt(cctx.App)
sel := lapi.Selector(cctx.String("data-selector"))
selp := &sel
if sel == "" {
selp = nil
}
eref, err := retrieve(ctx, cctx, fapi, selp, afmt.Printf)
if err != nil {
return err
}
fmt.Println() // separate retrieval events from results
if sel != "" {
eref.DAGs = append(eref.DAGs, lapi.DagSpec{DataSelector: &sel})
}
rc, err := cliutil.ClientExportStream(ainfo.Addr, ainfo.AuthHeader(), *eref, false)
if err != nil {
return err
}
defer rc.Close() // nolint
_, err = io.Copy(os.Stdout, rc)
return err
},
}
func pathToSel(psel string, matchTraversal bool, sub builder.SelectorSpec) (lapi.Selector, error) {
rs, err := textselector.SelectorSpecFromPath(textselector.Expression(psel), matchTraversal, sub)
if err != nil {
return "", xerrors.Errorf("failed to parse path-selector: %w", err)
}
var b bytes.Buffer
if err := dagjson.Encode(rs.Node(), &b); err != nil {
return "", err
}
return lapi.Selector(b.String()), nil
}
var clientRetrieveLsCmd = &cli.Command{
Name: "ls",
Usage: "List object links",
ArgsUsage: "[dataCid]",
Flags: append([]cli.Flag{
&cli.BoolFlag{
Name: "ipld",
Usage: "list IPLD datamodel links",
},
&cli.IntFlag{
Name: "depth",
Usage: "list links recursively up to the specified depth",
Value: 1,
},
&cli.StringFlag{
Name: "data-selector",
Usage: "IPLD datamodel text-path selector, or IPLD json selector",
},
}, retrFlagsCommon...),
Action: func(cctx *cli.Context) error {
if cctx.NArg() != 1 {
return IncorrectNumArgs(cctx)
}
ainfo, err := GetAPIInfo(cctx, repo.FullNode)
if err != nil {
return xerrors.Errorf("could not get API info: %w", err)
}
fapi, closer, err := GetFullNodeAPIV1(cctx)
if err != nil {
return err
}
defer closer()
ctx := ReqContext(cctx)
afmt := NewAppFmt(cctx.App)
dataSelector := lapi.Selector(fmt.Sprintf(`{"R":{"l":{"depth":%d},":>":{"a":{">":{"|":[{"@":{}},{".":{}}]}}}}}`, cctx.Int("depth")))
if cctx.IsSet("data-selector") {
ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any)
dataSelector, err = pathToSel(cctx.String("data-selector"), cctx.Bool("ipld"),
ssb.ExploreUnion(
ssb.Matcher(),
ssb.ExploreAll(
ssb.ExploreRecursive(selector.RecursionLimitDepth(int64(cctx.Int("depth"))), ssb.ExploreAll(ssb.ExploreUnion(ssb.Matcher(), ssb.ExploreRecursiveEdge()))),
)))
if err != nil {
return xerrors.Errorf("parsing datamodel path: %w", err)
}
}
eref, err := retrieve(ctx, cctx, fapi, &dataSelector, afmt.Printf)
if err != nil {
return xerrors.Errorf("retrieve: %w", err)
}
fmt.Println() // separate retrieval events from results
eref.DAGs = append(eref.DAGs, lapi.DagSpec{
DataSelector: &dataSelector,
})
rc, err := cliutil.ClientExportStream(ainfo.Addr, ainfo.AuthHeader(), *eref, true)
if err != nil {
return xerrors.Errorf("export: %w", err)
}
defer rc.Close() // nolint
var memcar bytes.Buffer
_, err = io.Copy(&memcar, rc)
if err != nil {
return err
}
cbs, err := blockstore.NewReadOnly(&bytesReaderAt{bytes.NewReader(memcar.Bytes())}, nil,
carv2.ZeroLengthSectionAsEOF(true),
blockstore.UseWholeCIDs(true))
if err != nil {
return xerrors.Errorf("opening car blockstore: %w", err)
}
roots, err := cbs.Roots()
if err != nil {
return xerrors.Errorf("getting roots: %w", err)
}
if len(roots) != 1 {
return xerrors.Errorf("expected 1 car root, got %d", len(roots))
}
dserv := merkledag.NewDAGService(blockservice.New(cbs, offline.Exchange(cbs)))
if !cctx.Bool("ipld") {
links, err := dserv.GetLinks(ctx, roots[0])
if err != nil {
return xerrors.Errorf("getting links: %w", err)
}
for _, link := range links {
fmt.Printf("%s %s\t%d\n", link.Cid, link.Name, link.Size)
}
} else {
jsel := lapi.Selector(fmt.Sprintf(`{"R":{"l":{"depth":%d},":>":{"a":{">":{"|":[{"@":{}},{".":{}}]}}}}}`, cctx.Int("depth")))
if cctx.IsSet("data-selector") {
ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any)
jsel, err = pathToSel(cctx.String("data-selector"), false,
ssb.ExploreRecursive(selector.RecursionLimitDepth(int64(cctx.Int("depth"))), ssb.ExploreAll(ssb.ExploreUnion(ssb.Matcher(), ssb.ExploreRecursiveEdge()))),
)
}
sel, _ := selectorparse.ParseJSONSelector(string(jsel))
if err := utils.TraverseDag(
ctx,
dserv,
roots[0],
sel,
nil,
func(p traversal.Progress, n ipld.Node, r traversal.VisitReason) error {
if r == traversal.VisitReason_SelectionMatch {
fmt.Println(p.Path)
}
return nil
},
); err != nil {
return err
}
}
return err
},
}
type bytesReaderAt struct {
btr *bytes.Reader
}
func (b bytesReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
return b.btr.ReadAt(p, off)
}
var _ io.ReaderAt = &bytesReaderAt{}
-1
View File
@@ -54,7 +54,6 @@ var GetFullNodeAPIV1 = cliutil.GetFullNodeAPIV1
var GetGatewayAPI = cliutil.GetGatewayAPI
var GetStorageMinerAPI = cliutil.GetStorageMinerAPI
var GetMarketsAPI = cliutil.GetMarketsAPI
var GetWorkerAPI = cliutil.GetWorkerAPI
var CommonCommands = []*cli.Command{
-54
View File
@@ -3,9 +3,7 @@ package cli
import (
"context"
"fmt"
"math"
"os"
"sort"
"strings"
"text/tabwriter"
"time"
@@ -14,7 +12,6 @@ import (
"github.com/fatih/color"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/lotus/api/v1api"
@@ -155,57 +152,6 @@ func infoCmdAct(cctx *cli.Context) error {
fmt.Printf("Payment Channels: %v channels\n", len(chs))
}
fmt.Println()
localDeals, err := fullapi.ClientListDeals(ctx)
if err != nil {
return err
}
var totalSize uint64
byState := map[storagemarket.StorageDealStatus][]uint64{}
for _, deal := range localDeals {
totalSize += deal.Size
byState[deal.State] = append(byState[deal.State], deal.Size)
}
fmt.Printf("Deals: %d, %s\n", len(localDeals), types.SizeStr(types.NewInt(totalSize)))
type stateStat struct {
state storagemarket.StorageDealStatus
count int
bytes uint64
}
stateStats := make([]stateStat, 0, len(byState))
for state, deals := range byState {
if state == storagemarket.StorageDealActive {
state = math.MaxUint64 // for sort
}
st := stateStat{
state: state,
count: len(deals),
}
for _, b := range deals {
st.bytes += b
}
stateStats = append(stateStats, st)
}
sort.Slice(stateStats, func(i, j int) bool {
return int64(stateStats[i].state) < int64(stateStats[j].state)
})
for _, st := range stateStats {
if st.state == math.MaxUint64 {
st.state = storagemarket.StorageDealActive
}
fmt.Printf(" %s: %d deals, %s\n", storagemarket.DealStates[st.state], st.count, types.SizeStr(types.NewInt(st.bytes)))
}
fmt.Println()
tw := tabwriter.NewWriter(os.Stdout, 6, 6, 2, ' ', 0)
s, err := fullapi.NetBandwidthStats(ctx)
-9
View File
@@ -39,11 +39,6 @@ var paychAddFundsCmd = &cli.Command{
Usage: "Add funds to the payment channel between fromAddress and toAddress. Creates the payment channel if it doesn't already exist.",
ArgsUsage: "[fromAddress toAddress amount]",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "restart-retrievals",
Usage: "restart stalled retrieval deals on this payment channel",
Value: true,
},
&cli.BoolFlag{
Name: "reserve",
Usage: "mark funds as reserved",
@@ -99,10 +94,6 @@ var paychAddFundsCmd = &cli.Command{
}
fmt.Fprintln(cctx.App.Writer, chAddr)
restartRetrievals := cctx.Bool("restart-retrievals")
if restartRetrievals {
return api.ClientRetrieveTryRestartInsufficientFunds(ctx, chAddr)
}
return nil
},
}
-21
View File
@@ -456,27 +456,6 @@ func GetWorkerAPI(ctx *cli.Context) (api.Worker, jsonrpc.ClientCloser, error) {
return client.NewWorkerRPCV0(ctx.Context, addr, headers)
}
func GetMarketsAPI(ctx *cli.Context) (api.StorageMiner, jsonrpc.ClientCloser, error) {
// to support lotus-miner cli tests.
if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
return tn.(api.StorageMiner), func() {}, nil
}
addr, headers, err := GetRawAPI(ctx, repo.Markets, "v0")
if err != nil {
return nil, nil, err
}
if IsVeryVerbose {
_, _ = fmt.Fprintln(ctx.App.Writer, "using markets API v0 endpoint:", addr)
}
// the markets node is a specialised miner's node, supporting only the
// markets API, which is a subset of the miner API. All non-markets
// operations will error out with "unsupported".
return client.NewStorageMinerRPCV0(ctx.Context, addr, headers)
}
func GetGatewayAPI(ctx *cli.Context) (api.Gateway, jsonrpc.ClientCloser, error) {
addr, headers, err := GetRawAPI(ctx, repo.FullNode, "v1")
if err != nil {