From 08e297a2178a4064b6afcb7201bcab90ae0288a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 9 Nov 2021 13:34:09 +0100 Subject: [PATCH 01/33] client: Support json selectors in retrieval --- node/impl/client/client.go | 61 ++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 199a2122d..46fe70cb4 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -8,6 +8,7 @@ import ( "io" "os" "sort" + "strings" "time" bstore "github.com/ipfs/go-ipfs-blockstore" @@ -15,6 +16,7 @@ import ( "github.com/ipld/go-car" carv2 "github.com/ipld/go-car/v2" carv2bs "github.com/ipld/go-car/v2/blockstore" + "github.com/ipld/go-ipld-prime/datamodel" "golang.org/x/xerrors" "github.com/filecoin-project/go-padreader" @@ -845,26 +847,35 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref sel := selectorparse.CommonSelector_ExploreAllRecursively if order.DatamodelPathSelector != nil { - ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) + if strings.HasPrefix(string(*order.DatamodelPathSelector), "{") { + var err error + sel, err = selectorparse.ParseJSONSelector(string(*order.DatamodelPathSelector)) + if err != nil { + finish(xerrors.Errorf("failed to parse json-selector '%s': %w", *order.DatamodelPathSelector, err)) + return + } + } else { + ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) - selspec, err := textselector.SelectorSpecFromPath( + selspec, err := textselector.SelectorSpecFromPath( - *order.DatamodelPathSelector, + *order.DatamodelPathSelector, - // URGH - this is a direct copy from https://github.com/filecoin-project/go-fil-markets/blob/v1.12.0/shared/selectors.go#L10-L16 - // Unable to use it because we need the SelectorSpec, and markets exposes just a reified node - ssb.ExploreRecursive( - selector.RecursionLimitNone(), - ssb.ExploreAll(ssb.ExploreRecursiveEdge()), - ), - ) - if err != nil { - finish(xerrors.Errorf("failed to parse text-selector '%s': %w", *order.DatamodelPathSelector, err)) - return + // URGH - this is a direct copy from https://github.com/filecoin-project/go-fil-markets/blob/v1.12.0/shared/selectors.go#L10-L16 + // Unable to use it because we need the SelectorSpec, and markets exposes just a reified node + ssb.ExploreRecursive( + selector.RecursionLimitNone(), + ssb.ExploreAll(ssb.ExploreRecursiveEdge()), + ), + ) + if err != nil { + finish(xerrors.Errorf("failed to parse text-selector '%s': %w", *order.DatamodelPathSelector, err)) + return + } + + sel = selspec.Node() + log.Infof("partial retrieval of datamodel-path-selector %s/*", *order.DatamodelPathSelector) } - - sel = selspec.Node() - log.Infof("partial retrieval of datamodel-path-selector %s/*", *order.DatamodelPathSelector) } // summary: @@ -1032,13 +1043,21 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref var subRootFound bool + var sel datamodel.Node + // no err check - we just compiled this before starting, but now we do not wrap a `*` - selspec, _ := textselector.SelectorSpecFromPath(*order.DatamodelPathSelector, nil) //nolint:errcheck + if strings.HasPrefix(string(*order.DatamodelPathSelector), "{") { + sel, _ = selectorparse.ParseJSONSelector(string(*order.DatamodelPathSelector)) + } else { + selspec, _ := textselector.SelectorSpecFromPath(*order.DatamodelPathSelector, nil) //nolint:errcheck + sel = selspec.Node() + } + if err := utils.TraverseDag( ctx, ds, root, - selspec.Node(), + sel, func(p traversal.Progress, n ipld.Node, r traversal.VisitReason) error { if r == traversal.VisitReason_SelectionMatch { @@ -1046,9 +1065,13 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref return xerrors.Errorf("unsupported selection path '%s' does not correspond to a block boundary (a.k.a. CID link)", p.Path.String()) } + if p.LastBlock.Link == nil { + return nil + } + cidLnk, castOK := p.LastBlock.Link.(cidlink.Link) if !castOK { - return xerrors.Errorf("cidlink cast unexpectedly failed on '%s'", p.LastBlock.Link.String()) + return xerrors.Errorf("cidlink cast unexpectedly failed on '%s'", p.LastBlock.Link) } root = cidLnk.Cid From 60ea33b1c7e55d34315d798a43c9f93d2462218a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 9 Nov 2021 17:27:42 +0100 Subject: [PATCH 02/33] client: cleanup clientRetrieve --- node/impl/client/client.go | 357 ++++++++++++++++++------------------- 1 file changed, 177 insertions(+), 180 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 46fe70cb4..3122e76a5 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -835,31 +835,21 @@ func consumeAllEvents(ctx context.Context, dealID rm.DealID, subscribeEvents cha } } -func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef, events chan marketevents.RetrievalEvent) { - defer close(events) - - finish := func(e error) { - if e != nil { - events <- marketevents.RetrievalEvent{Err: e.Error(), FundsSpent: big.Zero()} - } - } - +func getRetrievalSelector(dps *textselector.Expression) (datamodel.Node, error) { sel := selectorparse.CommonSelector_ExploreAllRecursively - if order.DatamodelPathSelector != nil { + if dps != nil { - if strings.HasPrefix(string(*order.DatamodelPathSelector), "{") { + if strings.HasPrefix(string(*dps), "{") { var err error - sel, err = selectorparse.ParseJSONSelector(string(*order.DatamodelPathSelector)) + sel, err = selectorparse.ParseJSONSelector(string(*dps)) if err != nil { - finish(xerrors.Errorf("failed to parse json-selector '%s': %w", *order.DatamodelPathSelector, err)) - return + return nil, xerrors.Errorf("failed to parse json-selector '%s': %w", *dps, err) } } else { ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) selspec, err := textselector.SelectorSpecFromPath( - - *order.DatamodelPathSelector, + *dps, // URGH - this is a direct copy from https://github.com/filecoin-project/go-fil-markets/blob/v1.12.0/shared/selectors.go#L10-L16 // Unable to use it because we need the SelectorSpec, and markets exposes just a reified node @@ -869,15 +859,179 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref ), ) if err != nil { - finish(xerrors.Errorf("failed to parse text-selector '%s': %w", *order.DatamodelPathSelector, err)) - return + return nil, xerrors.Errorf("failed to parse text-selector '%s': %w", *dps, err) } sel = selspec.Node() - log.Infof("partial retrieval of datamodel-path-selector %s/*", *order.DatamodelPathSelector) + log.Infof("partial retrieval of datamodel-path-selector %s/*", *dps) } } + return sel, nil +} + +func (a *API) doRetrieval(ctx context.Context, order api.RetrievalOrder, sel datamodel.Node, events chan marketevents.RetrievalEvent) (rm.DealID, error) { + if order.MinerPeer == nil || order.MinerPeer.ID == "" { + mi, err := a.StateMinerInfo(ctx, order.Miner, types.EmptyTSK) + if err != nil { + return 0, err + } + + order.MinerPeer = &rm.RetrievalPeer{ + ID: *mi.PeerId, + Address: order.Miner, + } + } + + if order.Total.Int == nil { + return 0, xerrors.Errorf("cannot make retrieval deal for null total") + } + + if order.Size == 0 { + return 0, xerrors.Errorf("cannot make retrieval deal for zero bytes") + } + + ppb := types.BigDiv(order.Total, types.NewInt(order.Size)) + + params, err := rm.NewParamsV1(ppb, order.PaymentInterval, order.PaymentIntervalIncrease, sel, order.Piece, order.UnsealPrice) + if err != nil { + return 0, xerrors.Errorf("Error in retrieval params: %s", err) + } + + // Subscribe to events before retrieving to avoid losing events. + subscribeEvents := make(chan retrievalSubscribeEvent, 1) + subscribeCtx, cancel := context.WithCancel(ctx) + defer cancel() + unsubscribe := a.Retrieval.SubscribeToEvents(func(event rm.ClientEvent, state rm.ClientDealState) { + // We'll check the deal IDs inside consumeAllEvents. + if state.PayloadCID.Equals(order.Root) { + select { + case <-subscribeCtx.Done(): + case subscribeEvents <- retrievalSubscribeEvent{event, state}: + } + } + }) + + id := a.Retrieval.NextID() + id, err = a.Retrieval.Retrieve( + ctx, + id, + order.Root, + params, + order.Total, + *order.MinerPeer, + order.Client, + order.Miner, + ) + + if err != nil { + unsubscribe() + return 0, xerrors.Errorf("Retrieve failed: %w", err) + } + + err = consumeAllEvents(ctx, id, subscribeEvents, events) + + unsubscribe() + if err != nil { + return 0, xerrors.Errorf("Retrieve: %w", err) + } + + return id, nil +} + +func (a *API) outputCAR(ctx context.Context, order api.RetrievalOrder, sel datamodel.Node, bs bstore.Blockstore, ref *api.FileRef) error { + // generating a CARv1 from the configured blockstore + f, err := os.OpenFile(ref.Path, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + + err = car.NewSelectiveCar( + ctx, + bs, + []car.Dag{{ + Root: order.Root, + Selector: sel, + }}, + car.MaxTraversalLinks(config.MaxTraversalLinks), + ).Write(f) + if err != nil { + return err + } + + return f.Close() +} + +func (a *API) outputUnixFS(ctx context.Context, order api.RetrievalOrder, sel datamodel.Node, bs bstore.Blockstore, ref *api.FileRef) error { + ds := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) + root := order.Root + + // if we used a selector - need to find the sub-root the user actually wanted to retrieve + if order.DatamodelPathSelector != nil { + var subRootFound bool + + if err := utils.TraverseDag( + ctx, + ds, + root, + sel, + func(p traversal.Progress, n ipld.Node, r traversal.VisitReason) error { + if r == traversal.VisitReason_SelectionMatch { + + if p.LastBlock.Path.String() != p.Path.String() { + return xerrors.Errorf("unsupported selection path '%s' does not correspond to a block boundary (a.k.a. CID link)", p.Path.String()) + } + + if p.LastBlock.Link == nil { + return nil + } + + cidLnk, castOK := p.LastBlock.Link.(cidlink.Link) + if !castOK { + return xerrors.Errorf("cidlink cast unexpectedly failed on '%s'", p.LastBlock.Link) + } + + root = cidLnk.Cid + subRootFound = true + } + return nil + }, + ); err != nil { + return xerrors.Errorf("error while locating partial retrieval sub-root: %w", err) + } + + if !subRootFound { + return xerrors.Errorf("path selection '%s' does not match a node within %s", *order.DatamodelPathSelector, root) + } + } + + nd, err := ds.Get(ctx, root) + if err != nil { + return xerrors.Errorf("ClientRetrieve: %w", err) + } + file, err := unixfile.NewUnixfsFile(ctx, ds, nd) + if err != nil { + return xerrors.Errorf("ClientRetrieve: %w", err) + } + + return files.WriteTo(file, ref.Path) +} + +func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef, events chan marketevents.RetrievalEvent) { + defer close(events) + + finish := func(e error) { + if e != nil { + events <- marketevents.RetrievalEvent{Err: e.Error(), FundsSpent: big.Zero()} + } + } + + sel, err := getRetrievalSelector(order.DatamodelPathSelector) + if err != nil { + finish(err) + return + } + // summary: // 1. if we're retrieving from an import, FromLocalCAR will be set. // Skip the retrieval itself, and use the provided car as a blockstore further down @@ -889,88 +1043,20 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref // this indicates we're proxying to IPFS. proxyBss, retrieveIntoIPFS := a.RtvlBlockstoreAccessor.(*retrievaladapter.ProxyBlockstoreAccessor) - carBss, retrieveIntoCAR := a.RtvlBlockstoreAccessor.(*retrievaladapter.CARBlockstoreAccessor) - carPath := order.FromLocalCAR // we actually need to retrieve from the network if carPath == "" { - if !retrieveIntoIPFS && !retrieveIntoCAR { // we don't recognize the blockstore accessor. finish(xerrors.Errorf("unsupported retrieval blockstore accessor")) return } - if order.MinerPeer == nil || order.MinerPeer.ID == "" { - mi, err := a.StateMinerInfo(ctx, order.Miner, types.EmptyTSK) - if err != nil { - finish(err) - return - } - - order.MinerPeer = &rm.RetrievalPeer{ - ID: *mi.PeerId, - Address: order.Miner, - } - } - - if order.Total.Int == nil { - finish(xerrors.Errorf("cannot make retrieval deal for null total")) - return - } - - if order.Size == 0 { - finish(xerrors.Errorf("cannot make retrieval deal for zero bytes")) - return - } - - ppb := types.BigDiv(order.Total, types.NewInt(order.Size)) - - params, err := rm.NewParamsV1(ppb, order.PaymentInterval, order.PaymentIntervalIncrease, sel, order.Piece, order.UnsealPrice) + id, err := a.doRetrieval(ctx, order, sel, events) if err != nil { - finish(xerrors.Errorf("Error in retrieval params: %s", err)) - return - } - - // Subscribe to events before retrieving to avoid losing events. - subscribeEvents := make(chan retrievalSubscribeEvent, 1) - subscribeCtx, cancel := context.WithCancel(ctx) - defer cancel() - unsubscribe := a.Retrieval.SubscribeToEvents(func(event rm.ClientEvent, state rm.ClientDealState) { - // We'll check the deal IDs inside consumeAllEvents. - if state.PayloadCID.Equals(order.Root) { - select { - case <-subscribeCtx.Done(): - case subscribeEvents <- retrievalSubscribeEvent{event, state}: - } - } - }) - - id := a.Retrieval.NextID() - id, err = a.Retrieval.Retrieve( - ctx, - id, - order.Root, - params, - order.Total, - *order.MinerPeer, - order.Client, - order.Miner, - ) - - if err != nil { - unsubscribe() - finish(xerrors.Errorf("Retrieve failed: %w", err)) - return - } - - err = consumeAllEvents(ctx, id, subscribeEvents, events) - - unsubscribe() - if err != nil { - finish(xerrors.Errorf("Retrieve: %w", err)) + finish(err) return } @@ -1002,106 +1088,17 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref // Are we outputting a CAR? if ref.IsCAR { - // not IPFS and we do full selection - just extract the CARv1 from the CARv2 we stored the retrieval in if !retrieveIntoIPFS && order.DatamodelPathSelector == nil { finish(carv2.ExtractV1File(carPath, ref.Path)) return } - // generating a CARv1 from the configured blockstore - f, err := os.OpenFile(ref.Path, os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - finish(err) - return - } - - err = car.NewSelectiveCar( - ctx, - retrievalBs, - []car.Dag{{ - Root: order.Root, - Selector: sel, - }}, - car.MaxTraversalLinks(config.MaxTraversalLinks), - ).Write(f) - if err != nil { - finish(err) - return - } - - finish(f.Close()) + finish(a.outputCAR(ctx, order, sel, retrievalBs, ref)) return } - // we are extracting a UnixFS file. - ds := merkledag.NewDAGService(blockservice.New(retrievalBs, offline.Exchange(retrievalBs))) - root := order.Root - - // if we used a selector - need to find the sub-root the user actually wanted to retrieve - if order.DatamodelPathSelector != nil { - - var subRootFound bool - - var sel datamodel.Node - - // no err check - we just compiled this before starting, but now we do not wrap a `*` - if strings.HasPrefix(string(*order.DatamodelPathSelector), "{") { - sel, _ = selectorparse.ParseJSONSelector(string(*order.DatamodelPathSelector)) - } else { - selspec, _ := textselector.SelectorSpecFromPath(*order.DatamodelPathSelector, nil) //nolint:errcheck - sel = selspec.Node() - } - - if err := utils.TraverseDag( - ctx, - ds, - root, - sel, - func(p traversal.Progress, n ipld.Node, r traversal.VisitReason) error { - if r == traversal.VisitReason_SelectionMatch { - - if p.LastBlock.Path.String() != p.Path.String() { - return xerrors.Errorf("unsupported selection path '%s' does not correspond to a block boundary (a.k.a. CID link)", p.Path.String()) - } - - if p.LastBlock.Link == nil { - return nil - } - - cidLnk, castOK := p.LastBlock.Link.(cidlink.Link) - if !castOK { - return xerrors.Errorf("cidlink cast unexpectedly failed on '%s'", p.LastBlock.Link) - } - - root = cidLnk.Cid - subRootFound = true - } - return nil - }, - ); err != nil { - finish(xerrors.Errorf("error while locating partial retrieval sub-root: %w", err)) - return - } - - if !subRootFound { - finish(xerrors.Errorf("path selection '%s' does not match a node within %s", *order.DatamodelPathSelector, root)) - return - } - } - - nd, err := ds.Get(ctx, root) - if err != nil { - finish(xerrors.Errorf("ClientRetrieve: %w", err)) - return - } - file, err := unixfile.NewUnixfsFile(ctx, ds, nd) - if err != nil { - finish(xerrors.Errorf("ClientRetrieve: %w", err)) - return - } - - finish(files.WriteTo(file, ref.Path)) + finish(a.outputUnixFS(ctx, order, sel, retrievalBs, ref)) } func (a *API) ClientListRetrievals(ctx context.Context) ([]api.RetrievalInfo, error) { From 89138bab4d5633a2939046e1c89d4d0485f6abd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 10 Nov 2021 15:45:46 +0100 Subject: [PATCH 03/33] Simplify retrieval APIs --- api/api_full.go | 10 +- api/proxy_gen.go | 39 +++--- api/types.go | 16 +++ api/v0api/full.go | 23 +++- api/v0api/proxy_gen.go | 12 +- api/v0api/v1_wrapper.go | 134 +++++++++++++++++++ cli/client.go | 114 +++++++++-------- gen/api/proxygen.go | 4 - itests/kit/deals.go | 44 +++++-- node/impl/client/client.go | 254 ++++++++++--------------------------- 10 files changed, 364 insertions(+), 286 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 6235e98d6..3bbfda178 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -28,7 +28,6 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/types" - marketevents "github.com/filecoin-project/lotus/markets/loggers" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo/imports" ) @@ -352,10 +351,9 @@ type FullNode interface { // ClientMinerQueryOffer returns a QueryOffer for the specific miner and file. ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (QueryOffer, error) //perm:read // ClientRetrieve initiates the retrieval of a file, as specified in the order. - ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *FileRef) error //perm:admin - // ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel - // of status updates. - ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin + ClientRetrieve(ctx context.Context, params RetrievalOrder) (*RestrievalRes, error) //perm:admin + // ClientExport exports a file stored in the local filestore to a system file + ClientExport(ctx context.Context, exportRef ExportRef, fileRef FileRef) error //perm:admin // ClientListRetrievals returns information about retrievals made by the local client ClientListRetrievals(ctx context.Context) ([]RetrievalInfo, error) //perm:write // ClientGetRetrievalUpdates returns status of updated retrieval deals @@ -940,8 +938,6 @@ type RetrievalOrder struct { DatamodelPathSelector *textselector.Expression Size uint64 - FromLocalCAR string // if specified, get data from a local CARv2 file. - // TODO: support offset Total types.BigInt UnsealPrice types.BigInt PaymentInterval uint64 diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 78ae607bf..d78304397 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -28,7 +28,6 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" "github.com/filecoin-project/lotus/journal/alerting" - marketevents "github.com/filecoin-project/lotus/markets/loggers" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo/imports" "github.com/filecoin-project/specs-storage/storage" @@ -162,6 +161,8 @@ type FullNodeStruct struct { ClientDealSize func(p0 context.Context, p1 cid.Cid) (DataSize, error) `perm:"read"` + ClientExport func(p0 context.Context, p1 ExportRef, p2 FileRef) error `perm:"admin"` + ClientFindData func(p0 context.Context, p1 cid.Cid, p2 *cid.Cid) ([]QueryOffer, error) `perm:"read"` ClientGenCar func(p0 context.Context, p1 FileRef, p2 string) error `perm:"write"` @@ -194,12 +195,10 @@ type FullNodeStruct struct { ClientRestartDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"` - ClientRetrieve func(p0 context.Context, p1 RetrievalOrder, p2 *FileRef) error `perm:"admin"` + ClientRetrieve func(p0 context.Context, p1 RetrievalOrder) (*RestrievalRes, error) `perm:"admin"` ClientRetrieveTryRestartInsufficientFunds func(p0 context.Context, p1 address.Address) error `perm:"write"` - ClientRetrieveWithEvents func(p0 context.Context, p1 RetrievalOrder, p2 *FileRef) (<-chan marketevents.RetrievalEvent, error) `perm:"admin"` - ClientStartDeal func(p0 context.Context, p1 *StartDealParams) (*cid.Cid, error) `perm:"admin"` ClientStatelessDeal func(p0 context.Context, p1 *StartDealParams) (*cid.Cid, error) `perm:"write"` @@ -1357,6 +1356,17 @@ func (s *FullNodeStub) ClientDealSize(p0 context.Context, p1 cid.Cid) (DataSize, return *new(DataSize), ErrNotSupported } +func (s *FullNodeStruct) ClientExport(p0 context.Context, p1 ExportRef, p2 FileRef) error { + if s.Internal.ClientExport == nil { + return ErrNotSupported + } + return s.Internal.ClientExport(p0, p1, p2) +} + +func (s *FullNodeStub) ClientExport(p0 context.Context, p1 ExportRef, p2 FileRef) error { + return ErrNotSupported +} + func (s *FullNodeStruct) ClientFindData(p0 context.Context, p1 cid.Cid, p2 *cid.Cid) ([]QueryOffer, error) { if s.Internal.ClientFindData == nil { return *new([]QueryOffer), ErrNotSupported @@ -1533,15 +1543,15 @@ func (s *FullNodeStub) ClientRestartDataTransfer(p0 context.Context, p1 datatran return ErrNotSupported } -func (s *FullNodeStruct) ClientRetrieve(p0 context.Context, p1 RetrievalOrder, p2 *FileRef) error { +func (s *FullNodeStruct) ClientRetrieve(p0 context.Context, p1 RetrievalOrder) (*RestrievalRes, error) { if s.Internal.ClientRetrieve == nil { - return ErrNotSupported + return nil, ErrNotSupported } - return s.Internal.ClientRetrieve(p0, p1, p2) + return s.Internal.ClientRetrieve(p0, p1) } -func (s *FullNodeStub) ClientRetrieve(p0 context.Context, p1 RetrievalOrder, p2 *FileRef) error { - return ErrNotSupported +func (s *FullNodeStub) ClientRetrieve(p0 context.Context, p1 RetrievalOrder) (*RestrievalRes, error) { + return nil, ErrNotSupported } func (s *FullNodeStruct) ClientRetrieveTryRestartInsufficientFunds(p0 context.Context, p1 address.Address) error { @@ -1555,17 +1565,6 @@ func (s *FullNodeStub) ClientRetrieveTryRestartInsufficientFunds(p0 context.Cont return ErrNotSupported } -func (s *FullNodeStruct) ClientRetrieveWithEvents(p0 context.Context, p1 RetrievalOrder, p2 *FileRef) (<-chan marketevents.RetrievalEvent, error) { - if s.Internal.ClientRetrieveWithEvents == nil { - return nil, ErrNotSupported - } - return s.Internal.ClientRetrieveWithEvents(p0, p1, p2) -} - -func (s *FullNodeStub) ClientRetrieveWithEvents(p0 context.Context, p1 RetrievalOrder, p2 *FileRef) (<-chan marketevents.RetrievalEvent, error) { - return nil, ErrNotSupported -} - func (s *FullNodeStruct) ClientStartDeal(p0 context.Context, p1 *StartDealParams) (*cid.Cid, error) { if s.Internal.ClientStartDeal == nil { return nil, ErrNotSupported diff --git a/api/types.go b/api/types.go index 9d887b0a1..a4c477545 100644 --- a/api/types.go +++ b/api/types.go @@ -7,6 +7,7 @@ import ( "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/lotus/chain/types" + textselector "github.com/ipld/go-ipld-selector-text-lite" datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-state-types/abi" @@ -194,4 +195,19 @@ type RetrievalInfo struct { TransferChannelID *datatransfer.ChannelID DataTransfer *DataTransferChannel + + // optional event if part of ClientGetRetrievalUpdates + Event *retrievalmarket.ClientEvent +} + +type RestrievalRes struct { + DealID retrievalmarket.DealID +} + +type ExportRef struct { + Root cid.Cid + DatamodelPathSelector *textselector.Expression + + FromLocalCAR string // if specified, get data from a local CARv2 file. + DealID retrievalmarket.DealID } diff --git a/api/v0api/full.go b/api/v0api/full.go index d7e38ce97..20e5a7179 100644 --- a/api/v0api/full.go +++ b/api/v0api/full.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/dline" "github.com/ipfs/go-cid" + textselector "github.com/ipld/go-ipld-selector-text-lite" "github.com/libp2p/go-libp2p-core/peer" "github.com/filecoin-project/lotus/api" @@ -325,10 +326,10 @@ type FullNode interface { // ClientMinerQueryOffer returns a QueryOffer for the specific miner and file. ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error) //perm:read // ClientRetrieve initiates the retrieval of a file, as specified in the order. - ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error //perm:admin + ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *api.FileRef) error //perm:admin // ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel // of status updates. - ClientRetrieveWithEvents(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin + ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin // ClientQueryAsk returns a signed StorageAsk from the specified miner. // ClientListRetrievals returns information about retrievals made by the local client ClientListRetrievals(ctx context.Context) ([]api.RetrievalInfo, error) //perm:write @@ -714,3 +715,21 @@ type FullNode interface { // the path specified when calling CreateBackup is within the base path CreateBackup(ctx context.Context, fpath string) error //perm:admin } + +type RetrievalOrder struct { + // TODO: make this less unixfs specific + Root cid.Cid + Piece *cid.Cid + DatamodelPathSelector *textselector.Expression + Size uint64 + + FromLocalCAR string // if specified, get data from a local CARv2 file. + // TODO: support offset + Total types.BigInt + UnsealPrice types.BigInt + PaymentInterval uint64 + PaymentIntervalIncrease uint64 + Client address.Address + Miner address.Address + MinerPeer *retrievalmarket.RetrievalPeer +} diff --git a/api/v0api/proxy_gen.go b/api/v0api/proxy_gen.go index dd6330a02..af0687fe5 100644 --- a/api/v0api/proxy_gen.go +++ b/api/v0api/proxy_gen.go @@ -125,11 +125,11 @@ type FullNodeStruct struct { ClientRestartDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"` - ClientRetrieve func(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) error `perm:"admin"` + ClientRetrieve func(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) error `perm:"admin"` ClientRetrieveTryRestartInsufficientFunds func(p0 context.Context, p1 address.Address) error `perm:"write"` - ClientRetrieveWithEvents func(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) `perm:"admin"` + ClientRetrieveWithEvents func(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) `perm:"admin"` ClientStartDeal func(p0 context.Context, p1 *api.StartDealParams) (*cid.Cid, error) `perm:"admin"` @@ -965,14 +965,14 @@ func (s *FullNodeStub) ClientRestartDataTransfer(p0 context.Context, p1 datatran return ErrNotSupported } -func (s *FullNodeStruct) ClientRetrieve(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) error { +func (s *FullNodeStruct) ClientRetrieve(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) error { if s.Internal.ClientRetrieve == nil { return ErrNotSupported } return s.Internal.ClientRetrieve(p0, p1, p2) } -func (s *FullNodeStub) ClientRetrieve(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) error { +func (s *FullNodeStub) ClientRetrieve(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) error { return ErrNotSupported } @@ -987,14 +987,14 @@ func (s *FullNodeStub) ClientRetrieveTryRestartInsufficientFunds(p0 context.Cont return ErrNotSupported } -func (s *FullNodeStruct) ClientRetrieveWithEvents(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) { +func (s *FullNodeStruct) ClientRetrieveWithEvents(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) { if s.Internal.ClientRetrieveWithEvents == nil { return nil, ErrNotSupported } return s.Internal.ClientRetrieveWithEvents(p0, p1, p2) } -func (s *FullNodeStub) ClientRetrieveWithEvents(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) { +func (s *FullNodeStub) ClientRetrieveWithEvents(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) { return nil, ErrNotSupported } diff --git a/api/v0api/v1_wrapper.go b/api/v0api/v1_wrapper.go index e36f478f5..0a1a463e5 100644 --- a/api/v0api/v1_wrapper.go +++ b/api/v0api/v1_wrapper.go @@ -3,7 +3,10 @@ package v0api import ( "context" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" + marketevents "github.com/filecoin-project/lotus/markets/loggers" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" @@ -194,4 +197,135 @@ func (w *WrapperV1Full) ChainGetRandomnessFromBeacon(ctx context.Context, tsk ty return w.StateGetRandomnessFromBeacon(ctx, personalization, randEpoch, entropy, tsk) } +func (w *WrapperV1Full) ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *api.FileRef) error { + events := make(chan marketevents.RetrievalEvent) + go w.clientRetrieve(ctx, order, ref, events) + + for { + select { + case evt, ok := <-events: + if !ok { // done successfully + return nil + } + + if evt.Err != "" { + return xerrors.Errorf("retrieval failed: %s", evt.Err) + } + case <-ctx.Done(): + return xerrors.Errorf("retrieval timed out") + } + } +} + +func (w *WrapperV1Full) ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) { + events := make(chan marketevents.RetrievalEvent) + go w.clientRetrieve(ctx, order, ref, events) + return events, nil +} + +func readSubscribeEvents(ctx context.Context, dealID retrievalmarket.DealID, subscribeEvents <-chan api.RetrievalInfo, events chan marketevents.RetrievalEvent) error { + for { + var subscribeEvent api.RetrievalInfo + var evt retrievalmarket.ClientEvent + select { + case <-ctx.Done(): + return xerrors.New("Retrieval Timed Out") + case subscribeEvent = <-subscribeEvents: + if subscribeEvent.ID != dealID { + // we can't check the deal ID ahead of time because: + // 1. We need to subscribe before retrieving. + // 2. We won't know the deal ID until after retrieving. + continue + } + if subscribeEvent.Event != nil { + evt = *subscribeEvent.Event + } + } + + select { + case <-ctx.Done(): + return xerrors.New("Retrieval Timed Out") + case events <- marketevents.RetrievalEvent{ + Event: evt, + Status: subscribeEvent.Status, + BytesReceived: subscribeEvent.BytesReceived, + FundsSpent: subscribeEvent.TotalPaid, + }: + } + + switch subscribeEvent.Status { + case retrievalmarket.DealStatusCompleted: + return nil + case retrievalmarket.DealStatusRejected: + return xerrors.Errorf("Retrieval Proposal Rejected: %s", subscribeEvent.Message) + case + retrievalmarket.DealStatusDealNotFound, + retrievalmarket.DealStatusErrored: + return xerrors.Errorf("Retrieval Error: %s", subscribeEvent.Message) + } + } +} + +func (w *WrapperV1Full) clientRetrieve(ctx context.Context, order RetrievalOrder, ref *api.FileRef, events chan marketevents.RetrievalEvent) { + defer close(events) + + finish := func(e error) { + if e != nil { + events <- marketevents.RetrievalEvent{Err: e.Error(), FundsSpent: big.Zero()} + } + } + + var dealID retrievalmarket.DealID + if order.FromLocalCAR == "" { + // Subscribe to events before retrieving to avoid losing events. + subscribeCtx, cancel := context.WithCancel(ctx) + defer cancel() + retrievalEvents, err := w.ClientGetRetrievalUpdates(subscribeCtx) + + if err != nil { + finish(xerrors.Errorf("GetRetrievalUpdates failed: %w", err)) + return + } + + retrievalRes, err := w.FullNode.ClientRetrieve(ctx, api.RetrievalOrder{ + Root: order.Root, + Piece: order.Piece, + Size: order.Size, + Total: order.Total, + UnsealPrice: order.UnsealPrice, + PaymentInterval: order.PaymentInterval, + PaymentIntervalIncrease: order.PaymentIntervalIncrease, + Client: order.Client, + Miner: order.Miner, + MinerPeer: order.MinerPeer, + }) + + if err != nil { + finish(xerrors.Errorf("Retrieve failed: %w", err)) + return + } + + dealID = retrievalRes.DealID + + err = readSubscribeEvents(ctx, retrievalRes.DealID, retrievalEvents, events) + if err != nil { + finish(xerrors.Errorf("Retrieve: %w", err)) + return + } + } + + // If ref is nil, it only fetches the data into the configured blockstore. + if ref == nil { + finish(nil) + return + } + + finish(w.ClientExport(ctx, api.ExportRef{ + Root: order.Root, + DatamodelPathSelector: order.DatamodelPathSelector, + FromLocalCAR: order.FromLocalCAR, + DealID: dealID, + }, *ref)) +} + var _ FullNode = &WrapperV1Full{} diff --git a/cli/client.go b/cli/client.go index daaf5f3fe..e715ec9fb 100644 --- a/cli/client.go +++ b/cli/client.go @@ -1069,7 +1069,7 @@ var clientRetrieveCmd = &cli.Command{ return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments")) } - fapi, closer, err := GetFullNodeAPI(cctx) + fapi, closer, err := GetFullNodeAPIV1(cctx) if err != nil { return err } @@ -1101,7 +1101,7 @@ var clientRetrieveCmd = &cli.Command{ pieceCid = &parsed } - var order *lapi.RetrievalOrder + var eref *lapi.ExportRef if cctx.Bool("allow-local") { imports, err := fapi.ClientListImports(ctx) if err != nil { @@ -1110,19 +1110,17 @@ var clientRetrieveCmd = &cli.Command{ for _, i := range imports { if i.Root != nil && i.Root.Equals(file) { - order = &lapi.RetrievalOrder{ - Root: file, - FromLocalCAR: i.CARPath, - - Total: big.Zero(), - UnsealPrice: big.Zero(), + eref = &lapi.ExportRef{ + Root: file, + FromLocalCAR: i.CARPath, } break } } } - if order == nil { + // no local found, so make a retrieval + if eref == nil { var offer api.QueryOffer minerStrAddr := cctx.String("miner") if minerStrAddr == "" { // Local discovery @@ -1163,7 +1161,7 @@ var clientRetrieveCmd = &cli.Command{ } } if offer.Err != "" { - return fmt.Errorf("The received offer errored: %s", offer.Err) + return fmt.Errorf("offer error: %s", offer.Err) } maxPrice := types.MustParseFIL(DefaultMaxRetrievePrice) @@ -1180,55 +1178,67 @@ var clientRetrieveCmd = &cli.Command{ } o := offer.Order(payer) - order = &o - } - ref := &lapi.FileRef{ - Path: cctx.Args().Get(1), - IsCAR: cctx.Bool("car"), + + subscribeEvents, err := fapi.ClientGetRetrievalUpdates(ctx) + if err != nil { + return xerrors.Errorf("error setting up retrieval updates: %w", err) + } + retrievalRes, err := fapi.ClientRetrieve(ctx, o) + if err != nil { + return xerrors.Errorf("error setting up retrieval: %w", err) + } + + readEvents: + for { + var evt api.RetrievalInfo + select { + case <-ctx.Done(): + return xerrors.New("Retrieval Timed Out") + case evt = <-subscribeEvents: + if evt.ID != retrievalRes.DealID { + // we can't check the deal ID ahead of time because: + // 1. We need to subscribe before retrieving. + // 2. We won't know the deal ID until after retrieving. + continue + } + } + afmt.Printf("> Recv: %s, Paid %s, %s (%s)\n", + types.SizeStr(types.NewInt(evt.BytesReceived)), + types.FIL(evt.TotalPaid), + retrievalmarket.ClientEvents[*evt.Event], + retrievalmarket.DealStatuses[evt.Status], + ) + switch evt.Status { + case retrievalmarket.DealStatusCompleted: + break readEvents + case retrievalmarket.DealStatusRejected: + return xerrors.Errorf("Retrieval Proposal Rejected: %s", evt.Message) + case + retrievalmarket.DealStatusDealNotFound, + retrievalmarket.DealStatusErrored: + return xerrors.Errorf("Retrieval Error: %s", evt.Message) + } + } + + eref = &lapi.ExportRef{ + Root: file, + DealID: retrievalRes.DealID, + } } if sel := textselector.Expression(cctx.String("datamodel-path-selector")); sel != "" { - order.DatamodelPathSelector = &sel + eref.DatamodelPathSelector = &sel } - updates, err := fapi.ClientRetrieveWithEvents(ctx, *order, ref) + err = fapi.ClientExport(ctx, *eref, lapi.FileRef{ + Path: cctx.Args().Get(1), + IsCAR: cctx.Bool("car"), + }) if err != nil { - return xerrors.Errorf("error setting up retrieval: %w", err) - } - - var prevStatus retrievalmarket.DealStatus - - for { - select { - case evt, ok := <-updates: - if ok { - afmt.Printf("> Recv: %s, Paid %s, %s (%s)\n", - types.SizeStr(types.NewInt(evt.BytesReceived)), - types.FIL(evt.FundsSpent), - retrievalmarket.ClientEvents[evt.Event], - retrievalmarket.DealStatuses[evt.Status], - ) - prevStatus = evt.Status - } - - if evt.Err != "" { - return xerrors.Errorf("retrieval failed: %s", evt.Err) - } - - if !ok { - if prevStatus == retrievalmarket.DealStatusCompleted { - afmt.Println("Success") - } else { - afmt.Printf("saw final deal state %s instead of expected success state DealStatusCompleted\n", - retrievalmarket.DealStatuses[prevStatus]) - } - return nil - } - - case <-ctx.Done(): - return xerrors.Errorf("retrieval timed out") - } + return err } + afmt.Println("Success") + return nil }, } diff --git a/gen/api/proxygen.go b/gen/api/proxygen.go index 3e0766c31..df39132ff 100644 --- a/gen/api/proxygen.go +++ b/gen/api/proxygen.go @@ -10,7 +10,6 @@ import ( "path/filepath" "strings" "text/template" - "unicode" "golang.org/x/xerrors" ) @@ -71,9 +70,6 @@ func typeName(e ast.Expr, pkg string) (string, error) { return t.X.(*ast.Ident).Name + "." + t.Sel.Name, nil case *ast.Ident: pstr := t.Name - if !unicode.IsLower(rune(pstr[0])) && pkg != "api" { - pstr = "api." + pstr // todo src pkg name - } return pstr, nil case *ast.ArrayType: subt, err := typeName(t.Elt, pkg) diff --git a/itests/kit/deals.go b/itests/kit/deals.go index 4a9af69e6..b8534982a 100644 --- a/itests/kit/deals.go +++ b/itests/kit/deals.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" @@ -320,17 +321,44 @@ func (dh *DealHarness) PerformRetrieval(ctx context.Context, deal *cid.Cid, root caddr, err := dh.client.WalletDefaultAddress(ctx) require.NoError(dh.t, err) - ref := &api.FileRef{ - Path: carFile.Name(), - IsCAR: carExport, - } + updatesCtx, cancel := context.WithCancel(ctx) + updates, err := dh.client.ClientGetRetrievalUpdates(updatesCtx) - updates, err := dh.client.ClientRetrieveWithEvents(ctx, offers[0].Order(caddr), ref) + retrievalRes, err := dh.client.ClientRetrieve(ctx, offers[0].Order(caddr)) require.NoError(dh.t, err) - - for update := range updates { - require.Emptyf(dh.t, update.Err, "retrieval failed: %s", update.Err) +consumeEvents: + for { + var evt api.RetrievalInfo + select { + case <-updatesCtx.Done(): + dh.t.Fatal("Retrieval Timed Out") + case evt = <-updates: + if evt.ID != retrievalRes.DealID { + continue + } + } + switch evt.Status { + case retrievalmarket.DealStatusCompleted: + break consumeEvents + case retrievalmarket.DealStatusRejected: + dh.t.Fatalf("Retrieval Proposal Rejected: %s", evt.Message) + case + retrievalmarket.DealStatusDealNotFound, + retrievalmarket.DealStatusErrored: + dh.t.Fatalf("Retrieval Error: %s", evt.Message) + } } + cancel() + + require.NoError(dh.t, dh.client.ClientExport(ctx, + api.ExportRef{ + Root: root, + DealID: retrievalRes.DealID, + }, + api.FileRef{ + Path: carFile.Name(), + IsCAR: carExport, + })) ret := carFile.Name() if carExport { diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 3122e76a5..8f82e6513 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -60,7 +60,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-actors/v3/actors/builtin/market" - marketevents "github.com/filecoin-project/lotus/markets/loggers" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/repo/imports" @@ -762,79 +761,6 @@ func (a *API) ClientCancelRetrievalDeal(ctx context.Context, dealID rm.DealID) e } } -func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error { - events := make(chan marketevents.RetrievalEvent) - go a.clientRetrieve(ctx, order, ref, events) - - for { - select { - case evt, ok := <-events: - if !ok { // done successfully - return nil - } - - if evt.Err != "" { - return xerrors.Errorf("retrieval failed: %s", evt.Err) - } - case <-ctx.Done(): - return xerrors.Errorf("retrieval timed out") - } - } -} - -func (a *API) ClientRetrieveWithEvents(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) { - events := make(chan marketevents.RetrievalEvent) - go a.clientRetrieve(ctx, order, ref, events) - return events, nil -} - -type retrievalSubscribeEvent struct { - event rm.ClientEvent - state rm.ClientDealState -} - -func consumeAllEvents(ctx context.Context, dealID rm.DealID, subscribeEvents chan retrievalSubscribeEvent, events chan marketevents.RetrievalEvent) error { - for { - var subscribeEvent retrievalSubscribeEvent - select { - case <-ctx.Done(): - return xerrors.New("Retrieval Timed Out") - case subscribeEvent = <-subscribeEvents: - if subscribeEvent.state.ID != dealID { - // we can't check the deal ID ahead of time because: - // 1. We need to subscribe before retrieving. - // 2. We won't know the deal ID until after retrieving. - continue - } - } - - select { - case <-ctx.Done(): - return xerrors.New("Retrieval Timed Out") - case events <- marketevents.RetrievalEvent{ - Event: subscribeEvent.event, - Status: subscribeEvent.state.Status, - BytesReceived: subscribeEvent.state.TotalReceived, - FundsSpent: subscribeEvent.state.FundsSpent, - }: - } - - state := subscribeEvent.state - switch state.Status { - case rm.DealStatusCompleted: - return nil - case rm.DealStatusRejected: - return xerrors.Errorf("Retrieval Proposal Rejected: %s", state.Message) - case rm.DealStatusCancelled: - return xerrors.Errorf("Retrieval was cancelled externally: %s", state.Message) - case - rm.DealStatusDealNotFound, - rm.DealStatusErrored: - return xerrors.Errorf("Retrieval Error: %s", state.Message) - } - } -} - func getRetrievalSelector(dps *textselector.Expression) (datamodel.Node, error) { sel := selectorparse.CommonSelector_ExploreAllRecursively if dps != nil { @@ -870,7 +796,23 @@ func getRetrievalSelector(dps *textselector.Expression) (datamodel.Node, error) return sel, nil } -func (a *API) doRetrieval(ctx context.Context, order api.RetrievalOrder, sel datamodel.Node, events chan marketevents.RetrievalEvent) (rm.DealID, error) { +func (a *API) ClientRetrieve(ctx context.Context, params api.RetrievalOrder) (*api.RestrievalRes, error) { + sel, err := getRetrievalSelector(params.DatamodelPathSelector) + if err != nil { + return nil, err + } + + di, err := a.doRetrieval(ctx, params, sel) + if err != nil { + return nil, err + } + + return &api.RestrievalRes{ + DealID: di, + }, nil +} + +func (a *API) doRetrieval(ctx context.Context, order api.RetrievalOrder, sel datamodel.Node) (rm.DealID, error) { if order.MinerPeer == nil || order.MinerPeer.ID == "" { mi, err := a.StateMinerInfo(ctx, order.Miner, types.EmptyTSK) if err != nil { @@ -898,20 +840,6 @@ func (a *API) doRetrieval(ctx context.Context, order api.RetrievalOrder, sel dat return 0, xerrors.Errorf("Error in retrieval params: %s", err) } - // Subscribe to events before retrieving to avoid losing events. - subscribeEvents := make(chan retrievalSubscribeEvent, 1) - subscribeCtx, cancel := context.WithCancel(ctx) - defer cancel() - unsubscribe := a.Retrieval.SubscribeToEvents(func(event rm.ClientEvent, state rm.ClientDealState) { - // We'll check the deal IDs inside consumeAllEvents. - if state.PayloadCID.Equals(order.Root) { - select { - case <-subscribeCtx.Done(): - case subscribeEvents <- retrievalSubscribeEvent{event, state}: - } - } - }) - id := a.Retrieval.NextID() id, err = a.Retrieval.Retrieve( ctx, @@ -925,21 +853,58 @@ func (a *API) doRetrieval(ctx context.Context, order api.RetrievalOrder, sel dat ) if err != nil { - unsubscribe() return 0, xerrors.Errorf("Retrieve failed: %w", err) } - err = consumeAllEvents(ctx, id, subscribeEvents, events) - - unsubscribe() - if err != nil { - return 0, xerrors.Errorf("Retrieve: %w", err) - } - return id, nil } -func (a *API) outputCAR(ctx context.Context, order api.RetrievalOrder, sel datamodel.Node, bs bstore.Blockstore, ref *api.FileRef) error { +func (a *API) ClientExport(ctx context.Context, exportRef api.ExportRef, ref api.FileRef) error { + proxyBss, retrieveIntoIPFS := a.RtvlBlockstoreAccessor.(*retrievaladapter.ProxyBlockstoreAccessor) + carBss, retrieveIntoCAR := a.RtvlBlockstoreAccessor.(*retrievaladapter.CARBlockstoreAccessor) + carPath := exportRef.FromLocalCAR + + sel, err := getRetrievalSelector(exportRef.DatamodelPathSelector) + if err != nil { + return err + } + + if carPath == "" { + if !retrieveIntoIPFS && !retrieveIntoCAR { + return xerrors.Errorf("unsupported retrieval blockstore accessor") + } + + if retrieveIntoCAR { + carPath = carBss.PathFor(exportRef.DealID) + } + } + + var retrievalBs bstore.Blockstore + if retrieveIntoIPFS { + retrievalBs = proxyBss.Blockstore + } else { + cbs, err := stores.ReadOnlyFilestore(carPath) + if err != nil { + return err + } + defer cbs.Close() //nolint:errcheck + retrievalBs = cbs + } + + // Are we outputting a CAR? + if ref.IsCAR { + // not IPFS and we do full selection - just extract the CARv1 from the CARv2 we stored the retrieval in + if !retrieveIntoIPFS && exportRef.DatamodelPathSelector == nil { + return carv2.ExtractV1File(carPath, ref.Path) + } + + return a.outputCAR(ctx, exportRef.Root, sel, retrievalBs, ref) + } + + return a.outputUnixFS(ctx, exportRef.Root, exportRef.DatamodelPathSelector != nil, sel, retrievalBs, ref) +} + +func (a *API) outputCAR(ctx context.Context, root cid.Cid, sel datamodel.Node, bs bstore.Blockstore, ref api.FileRef) error { // generating a CARv1 from the configured blockstore f, err := os.OpenFile(ref.Path, os.O_CREATE|os.O_WRONLY, 0644) if err != nil { @@ -950,7 +915,7 @@ func (a *API) outputCAR(ctx context.Context, order api.RetrievalOrder, sel datam ctx, bs, []car.Dag{{ - Root: order.Root, + Root: root, Selector: sel, }}, car.MaxTraversalLinks(config.MaxTraversalLinks), @@ -962,12 +927,11 @@ func (a *API) outputCAR(ctx context.Context, order api.RetrievalOrder, sel datam return f.Close() } -func (a *API) outputUnixFS(ctx context.Context, order api.RetrievalOrder, sel datamodel.Node, bs bstore.Blockstore, ref *api.FileRef) error { +func (a *API) outputUnixFS(ctx context.Context, root cid.Cid, findSubroot bool, sel datamodel.Node, bs bstore.Blockstore, ref api.FileRef) error { ds := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) - root := order.Root // if we used a selector - need to find the sub-root the user actually wanted to retrieve - if order.DatamodelPathSelector != nil { + if findSubroot { var subRootFound bool if err := utils.TraverseDag( @@ -1001,7 +965,7 @@ func (a *API) outputUnixFS(ctx context.Context, order api.RetrievalOrder, sel da } if !subRootFound { - return xerrors.Errorf("path selection '%s' does not match a node within %s", *order.DatamodelPathSelector, root) + return xerrors.Errorf("path selection does not match a node within %s", root) } } @@ -1017,90 +981,6 @@ func (a *API) outputUnixFS(ctx context.Context, order api.RetrievalOrder, sel da return files.WriteTo(file, ref.Path) } -func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef, events chan marketevents.RetrievalEvent) { - defer close(events) - - finish := func(e error) { - if e != nil { - events <- marketevents.RetrievalEvent{Err: e.Error(), FundsSpent: big.Zero()} - } - } - - sel, err := getRetrievalSelector(order.DatamodelPathSelector) - if err != nil { - finish(err) - return - } - - // summary: - // 1. if we're retrieving from an import, FromLocalCAR will be set. - // Skip the retrieval itself, and use the provided car as a blockstore further down - // to extract a CAR or UnixFS export from. - // 2. if we're using an IPFS blockstore for retrieval, retrieve into it, - // then use the virtual blockstore to extract a CAR or UnixFS export from it. - // 3. if we have to retrieve, perform a CARv2 retrieval, then either - // extract the CARv1 (with ExtractV1File) or use it as a blockstore further down. - - // this indicates we're proxying to IPFS. - proxyBss, retrieveIntoIPFS := a.RtvlBlockstoreAccessor.(*retrievaladapter.ProxyBlockstoreAccessor) - carBss, retrieveIntoCAR := a.RtvlBlockstoreAccessor.(*retrievaladapter.CARBlockstoreAccessor) - carPath := order.FromLocalCAR - - // we actually need to retrieve from the network - if carPath == "" { - if !retrieveIntoIPFS && !retrieveIntoCAR { - // we don't recognize the blockstore accessor. - finish(xerrors.Errorf("unsupported retrieval blockstore accessor")) - return - } - - id, err := a.doRetrieval(ctx, order, sel, events) - if err != nil { - finish(err) - return - } - - if retrieveIntoCAR { - carPath = carBss.PathFor(id) - } - } - - if ref == nil { - // If ref is nil, it only fetches the data into the configured blockstore - // (if fetching from network). - finish(nil) - return - } - - // determine where did the retrieval go - var retrievalBs bstore.Blockstore - if retrieveIntoIPFS { - retrievalBs = proxyBss.Blockstore - } else { - cbs, err := stores.ReadOnlyFilestore(carPath) - if err != nil { - finish(err) - return - } - defer cbs.Close() //nolint:errcheck - retrievalBs = cbs - } - - // Are we outputting a CAR? - if ref.IsCAR { - // not IPFS and we do full selection - just extract the CARv1 from the CARv2 we stored the retrieval in - if !retrieveIntoIPFS && order.DatamodelPathSelector == nil { - finish(carv2.ExtractV1File(carPath, ref.Path)) - return - } - - finish(a.outputCAR(ctx, order, sel, retrievalBs, ref)) - return - } - - finish(a.outputUnixFS(ctx, order, sel, retrievalBs, ref)) -} - func (a *API) ClientListRetrievals(ctx context.Context) ([]api.RetrievalInfo, error) { deals, err := a.Retrieval.ListDeals() if err != nil { From b868769ec8e313918e77bef91955ca91a6a0f0c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 10 Nov 2021 17:51:16 +0100 Subject: [PATCH 04/33] more retrieval api work --- api/api_full.go | 2 + api/docgen/docgen.go | 3 + api/mocks/mock_full.go | 43 +++++----- api/proxy_gen.go | 13 +++ api/v0api/v0mocks/mock_full.go | 5 +- cli/client.go | 4 +- documentation/en/api-v0-methods.md | 3 +- documentation/en/api-v1-unstable-methods.md | 91 ++++++++------------- itests/deals_partial_retrieval_test.go | 38 +++++++-- node/impl/client/client.go | 50 +++++++++++ node/impl/client/client_test.go | 6 +- 11 files changed, 163 insertions(+), 95 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 3bbfda178..48b5d0d3c 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -352,6 +352,8 @@ type FullNode interface { ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (QueryOffer, error) //perm:read // ClientRetrieve initiates the retrieval of a file, as specified in the order. ClientRetrieve(ctx context.Context, params RetrievalOrder) (*RestrievalRes, error) //perm:admin + // ClientRetrieveWait waits for retrieval to be complete + ClientRetrieveWait(ctx context.Context, deal retrievalmarket.DealID) error //perm:admin // ClientExport exports a file stored in the local filestore to a system file ClientExport(ctx context.Context, exportRef ExportRef, fileRef FileRef) error //perm:admin // ClientListRetrievals returns information about retrievals made by the local client diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index 25b9ac8c9..498a0747c 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -91,6 +91,7 @@ func init() { storeIDExample := imports.ID(50) textSelExample := textselector.Expression("Links/21/Hash/Links/42/Hash") + clientEvent := retrievalmarket.ClientEventDealAccepted addExample(bitfield.NewFromSet([]uint64{5})) addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1) @@ -122,6 +123,8 @@ func init() { addExample(datatransfer.Ongoing) addExample(storeIDExample) addExample(&storeIDExample) + addExample(clientEvent) + addExample(&clientEvent) addExample(retrievalmarket.ClientEventDealAccepted) addExample(retrievalmarket.DealStatusNew) addExample(&textSelExample) diff --git a/api/mocks/mock_full.go b/api/mocks/mock_full.go index 44fe82b60..4b18eb365 100644 --- a/api/mocks/mock_full.go +++ b/api/mocks/mock_full.go @@ -25,7 +25,6 @@ import ( miner "github.com/filecoin-project/lotus/chain/actors/builtin/miner" types "github.com/filecoin-project/lotus/chain/types" alerting "github.com/filecoin-project/lotus/journal/alerting" - marketevents "github.com/filecoin-project/lotus/markets/loggers" dtypes "github.com/filecoin-project/lotus/node/modules/dtypes" imports "github.com/filecoin-project/lotus/node/repo/imports" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" @@ -537,6 +536,20 @@ func (mr *MockFullNodeMockRecorder) ClientDealSize(arg0, arg1 interface{}) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientDealSize", reflect.TypeOf((*MockFullNode)(nil).ClientDealSize), arg0, arg1) } +// ClientExport mocks base method. +func (m *MockFullNode) ClientExport(arg0 context.Context, arg1 api.ExportRef, arg2 api.FileRef) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClientExport", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// ClientExport indicates an expected call of ClientExport. +func (mr *MockFullNodeMockRecorder) ClientExport(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientExport", reflect.TypeOf((*MockFullNode)(nil).ClientExport), arg0, arg1, arg2) +} + // ClientFindData mocks base method. func (m *MockFullNode) ClientFindData(arg0 context.Context, arg1 cid.Cid, arg2 *cid.Cid) ([]api.QueryOffer, error) { m.ctrl.T.Helper() @@ -775,17 +788,18 @@ func (mr *MockFullNodeMockRecorder) ClientRestartDataTransfer(arg0, arg1, arg2, } // ClientRetrieve mocks base method. -func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) error { +func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 api.RetrievalOrder) (*api.RestrievalRes, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ClientRetrieve", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 + ret := m.ctrl.Call(m, "ClientRetrieve", arg0, arg1) + ret0, _ := ret[0].(*api.RestrievalRes) + ret1, _ := ret[1].(error) + return ret0, ret1 } // ClientRetrieve indicates an expected call of ClientRetrieve. -func (mr *MockFullNodeMockRecorder) ClientRetrieve(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockFullNodeMockRecorder) ClientRetrieve(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieve", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieve), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieve", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieve), arg0, arg1) } // ClientRetrieveTryRestartInsufficientFunds mocks base method. @@ -802,21 +816,6 @@ func (mr *MockFullNodeMockRecorder) ClientRetrieveTryRestartInsufficientFunds(ar return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieveTryRestartInsufficientFunds", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieveTryRestartInsufficientFunds), arg0, arg1) } -// ClientRetrieveWithEvents mocks base method. -func (m *MockFullNode) ClientRetrieveWithEvents(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ClientRetrieveWithEvents", arg0, arg1, arg2) - ret0, _ := ret[0].(<-chan marketevents.RetrievalEvent) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ClientRetrieveWithEvents indicates an expected call of ClientRetrieveWithEvents. -func (mr *MockFullNodeMockRecorder) ClientRetrieveWithEvents(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieveWithEvents", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieveWithEvents), arg0, arg1, arg2) -} - // ClientStartDeal mocks base method. func (m *MockFullNode) ClientStartDeal(arg0 context.Context, arg1 *api.StartDealParams) (*cid.Cid, error) { m.ctrl.T.Helper() diff --git a/api/proxy_gen.go b/api/proxy_gen.go index d78304397..feb08531f 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -199,6 +199,8 @@ type FullNodeStruct struct { ClientRetrieveTryRestartInsufficientFunds func(p0 context.Context, p1 address.Address) error `perm:"write"` + ClientRetrieveWait func(p0 context.Context, p1 retrievalmarket.DealID) error `perm:"admin"` + ClientStartDeal func(p0 context.Context, p1 *StartDealParams) (*cid.Cid, error) `perm:"admin"` ClientStatelessDeal func(p0 context.Context, p1 *StartDealParams) (*cid.Cid, error) `perm:"write"` @@ -1565,6 +1567,17 @@ func (s *FullNodeStub) ClientRetrieveTryRestartInsufficientFunds(p0 context.Cont return ErrNotSupported } +func (s *FullNodeStruct) ClientRetrieveWait(p0 context.Context, p1 retrievalmarket.DealID) error { + if s.Internal.ClientRetrieveWait == nil { + return ErrNotSupported + } + return s.Internal.ClientRetrieveWait(p0, p1) +} + +func (s *FullNodeStub) ClientRetrieveWait(p0 context.Context, p1 retrievalmarket.DealID) error { + return ErrNotSupported +} + func (s *FullNodeStruct) ClientStartDeal(p0 context.Context, p1 *StartDealParams) (*cid.Cid, error) { if s.Internal.ClientStartDeal == nil { return nil, ErrNotSupported diff --git a/api/v0api/v0mocks/mock_full.go b/api/v0api/v0mocks/mock_full.go index 0344eebf3..3e9caaee8 100644 --- a/api/v0api/v0mocks/mock_full.go +++ b/api/v0api/v0mocks/mock_full.go @@ -21,6 +21,7 @@ import ( network "github.com/filecoin-project/go-state-types/network" api "github.com/filecoin-project/lotus/api" apitypes "github.com/filecoin-project/lotus/api/types" + v0api "github.com/filecoin-project/lotus/api/v0api" miner "github.com/filecoin-project/lotus/chain/actors/builtin/miner" types "github.com/filecoin-project/lotus/chain/types" alerting "github.com/filecoin-project/lotus/journal/alerting" @@ -760,7 +761,7 @@ func (mr *MockFullNodeMockRecorder) ClientRestartDataTransfer(arg0, arg1, arg2, } // ClientRetrieve mocks base method. -func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) error { +func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 v0api.RetrievalOrder, arg2 *api.FileRef) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ClientRetrieve", arg0, arg1, arg2) ret0, _ := ret[0].(error) @@ -788,7 +789,7 @@ func (mr *MockFullNodeMockRecorder) ClientRetrieveTryRestartInsufficientFunds(ar } // ClientRetrieveWithEvents mocks base method. -func (m *MockFullNode) ClientRetrieveWithEvents(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) { +func (m *MockFullNode) ClientRetrieveWithEvents(arg0 context.Context, arg1 v0api.RetrievalOrder, arg2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ClientRetrieveWithEvents", arg0, arg1, arg2) ret0, _ := ret[0].(<-chan marketevents.RetrievalEvent) diff --git a/cli/client.go b/cli/client.go index e715ec9fb..4aa64ef55 100644 --- a/cli/client.go +++ b/cli/client.go @@ -1111,8 +1111,8 @@ var clientRetrieveCmd = &cli.Command{ for _, i := range imports { if i.Root != nil && i.Root.Equals(file) { eref = &lapi.ExportRef{ - Root: file, - FromLocalCAR: i.CARPath, + Root: file, + FromLocalCAR: i.CARPath, } break } diff --git a/documentation/en/api-v0-methods.md b/documentation/en/api-v0-methods.md index 4d9530821..2e57b8d7d 100644 --- a/documentation/en/api-v0-methods.md +++ b/documentation/en/api-v0-methods.md @@ -1269,7 +1269,8 @@ Response: "Stages": { "Stages": null } - } + }, + "Event": 5 } ``` diff --git a/documentation/en/api-v1-unstable-methods.md b/documentation/en/api-v1-unstable-methods.md index 24eba2d06..17cb16339 100644 --- a/documentation/en/api-v1-unstable-methods.md +++ b/documentation/en/api-v1-unstable-methods.md @@ -41,6 +41,7 @@ * [ClientDataTransferUpdates](#ClientDataTransferUpdates) * [ClientDealPieceCID](#ClientDealPieceCID) * [ClientDealSize](#ClientDealSize) + * [ClientExport](#ClientExport) * [ClientFindData](#ClientFindData) * [ClientGenCar](#ClientGenCar) * [ClientGetDealInfo](#ClientGetDealInfo) @@ -59,7 +60,6 @@ * [ClientRestartDataTransfer](#ClientRestartDataTransfer) * [ClientRetrieve](#ClientRetrieve) * [ClientRetrieveTryRestartInsufficientFunds](#ClientRetrieveTryRestartInsufficientFunds) - * [ClientRetrieveWithEvents](#ClientRetrieveWithEvents) * [ClientStartDeal](#ClientStartDeal) * [ClientStatelessDeal](#ClientStatelessDeal) * [Create](#Create) @@ -1055,6 +1055,32 @@ Response: } ``` +### ClientExport +ClientExport exports a file stored in the local filestore to a system file + + +Perms: admin + +Inputs: +```json +[ + { + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "DatamodelPathSelector": "Links/21/Hash/Links/42/Hash", + "FromLocalCAR": "string value", + "DealID": 5 + }, + { + "Path": "string value", + "IsCAR": true + } +] +``` + +Response: `{}` + ### ClientFindData ClientFindData identifies peers that have a certain file, and returns QueryOffers (one per peer). @@ -1282,7 +1308,8 @@ Response: "Stages": { "Stages": null } - } + }, + "Event": 5 } ``` @@ -1484,7 +1511,6 @@ Inputs: "Piece": null, "DatamodelPathSelector": "Links/21/Hash/Links/42/Hash", "Size": 42, - "FromLocalCAR": "string value", "Total": "0", "UnsealPrice": "0", "PaymentInterval": 42, @@ -1496,15 +1522,16 @@ Inputs: "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", "PieceCID": null } - }, - { - "Path": "string value", - "IsCAR": true } ] ``` -Response: `{}` +Response: +```json +{ + "DealID": 5 +} +``` ### ClientRetrieveTryRestartInsufficientFunds ClientRetrieveTryRestartInsufficientFunds attempts to restart stalled retrievals on a given payment channel @@ -1522,54 +1549,6 @@ Inputs: Response: `{}` -### ClientRetrieveWithEvents -ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel -of status updates. - - -Perms: admin - -Inputs: -```json -[ - { - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Piece": null, - "DatamodelPathSelector": "Links/21/Hash/Links/42/Hash", - "Size": 42, - "FromLocalCAR": "string value", - "Total": "0", - "UnsealPrice": "0", - "PaymentInterval": 42, - "PaymentIntervalIncrease": 42, - "Client": "f01234", - "Miner": "f01234", - "MinerPeer": { - "Address": "f01234", - "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", - "PieceCID": null - } - }, - { - "Path": "string value", - "IsCAR": true - } -] -``` - -Response: -```json -{ - "Event": 5, - "Status": 0, - "BytesReceived": 42, - "FundsSpent": "0", - "Err": "string value" -} -``` - ### ClientStartDeal ClientStartDeal proposes a deal with a miner. diff --git a/itests/deals_partial_retrieval_test.go b/itests/deals_partial_retrieval_test.go index ffc8c5e2c..8c7775588 100644 --- a/itests/deals_partial_retrieval_test.go +++ b/itests/deals_partial_retrieval_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "golang.org/x/xerrors" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" @@ -56,14 +58,11 @@ func TestPartialRetrieval(t *testing.T) { for _, fullCycle := range []bool{false, true} { var retOrder api.RetrievalOrder + var eref api.ExportRef if !fullCycle { - - retOrder.FromLocalCAR = sourceCar - retOrder.Root = carRoot - + eref.FromLocalCAR = sourceCar } else { - dp := dh.DefaultStartDealParams() dp.Data = &storagemarket.DataRef{ // FIXME: figure out how to do this with an online partial transfer @@ -96,6 +95,8 @@ func TestPartialRetrieval(t *testing.T) { } retOrder.DatamodelPathSelector = &textSelector + eref.DatamodelPathSelector = &textSelector + eref.Root = carRoot // test retrieval of either data or constructing a partial selective-car for _, retrieveAsCar := range []bool{false, true} { @@ -107,6 +108,7 @@ func TestPartialRetrieval(t *testing.T) { ctx, client, retOrder, + eref, &api.FileRef{ Path: outFile.Name(), IsCAR: retrieveAsCar, @@ -131,10 +133,13 @@ func TestPartialRetrieval(t *testing.T) { ctx, client, api.RetrievalOrder{ - FromLocalCAR: sourceCar, Root: carRoot, DatamodelPathSelector: &textSelectorNonexistent, }, + api.ExportRef{ + FromLocalCAR: sourceCar, + DatamodelPathSelector: &textSelectorNonexistent, + }, &api.FileRef{}, nil, ), @@ -148,10 +153,13 @@ func TestPartialRetrieval(t *testing.T) { ctx, client, api.RetrievalOrder{ - FromLocalCAR: sourceCar, Root: carRoot, DatamodelPathSelector: &textSelectorNonLink, }, + api.ExportRef{ + FromLocalCAR: sourceCar, + DatamodelPathSelector: &textSelectorNonLink, + }, &api.FileRef{}, nil, ), @@ -159,7 +167,7 @@ func TestPartialRetrieval(t *testing.T) { ) } -func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrder api.RetrievalOrder, retRef *api.FileRef, outFile *os.File) error { +func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrder api.RetrievalOrder, eref api.ExportRef, retRef *api.FileRef, outFile *os.File) error { if retOrder.Total.Nil() { retOrder.Total = big.Zero() @@ -168,7 +176,19 @@ func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrde retOrder.UnsealPrice = big.Zero() } - err := client.ClientRetrieve(ctx, retOrder, retRef) + if eref.FromLocalCAR == "" { + rr, err := client.ClientRetrieve(ctx, retOrder) + if err != nil { + return err + } + eref.DealID = rr.DealID + + if err := client.ClientRetrieveWait(ctx, rr.DealID); err != nil { + return xerrors.Errorf("retrieval wait: %w", err) + } + } + + err := client.ClientExport(ctx, eref, *retRef) if err != nil { return err } diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 8f82e6513..34bde7e26 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -859,6 +859,56 @@ func (a *API) doRetrieval(ctx context.Context, order api.RetrievalOrder, sel dat return id, nil } +func (a *API) ClientRetrieveWait(ctx context.Context, deal rm.DealID) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + subscribeEvents := make(chan rm.ClientDealState, 1) + + unsubscribe := a.Retrieval.SubscribeToEvents(func(event rm.ClientEvent, state rm.ClientDealState) { + // We'll check the deal IDs inside consumeAllEvents. + if state.ID != deal { + return + } + select { + case <-ctx.Done(): + case subscribeEvents <- state: + } + }) + defer unsubscribe() + + { + state, err := a.Retrieval.GetDeal(deal) + if err != nil { + return xerrors.Errorf("getting deal state: %w", err) + } + select { + case subscribeEvents <- state: + default: // already have an event queued from the subscription + } + } + + for { + select { + case <-ctx.Done(): + return xerrors.New("Retrieval Timed Out") + case state := <-subscribeEvents: + switch state.Status { + case rm.DealStatusCompleted: + return nil + case rm.DealStatusRejected: + return xerrors.Errorf("Retrieval Proposal Rejected: %s", state.Message) + case rm.DealStatusCancelled: + return xerrors.Errorf("Retrieval was cancelled externally: %s", state.Message) + case + rm.DealStatusDealNotFound, + rm.DealStatusErrored: + return xerrors.Errorf("Retrieval Error: %s", state.Message) + } + } + } +} + func (a *API) ClientExport(ctx context.Context, exportRef api.ExportRef, ref api.FileRef) error { proxyBss, retrieveIntoIPFS := a.RtvlBlockstoreAccessor.(*retrievaladapter.ProxyBlockstoreAccessor) carBss, retrieveIntoCAR := a.RtvlBlockstoreAccessor.(*retrievaladapter.CARBlockstoreAccessor) diff --git a/node/impl/client/client_test.go b/node/impl/client/client_test.go index 834c980ab..bf7ff7735 100644 --- a/node/impl/client/client_test.go +++ b/node/impl/client/client_test.go @@ -60,7 +60,7 @@ func TestImportLocal(t *testing.T) { require.NoError(t, err) require.True(t, local) - order := api.RetrievalOrder{ + order := api.ExportRef{ Root: root, FromLocalCAR: it.CARPath, } @@ -68,7 +68,7 @@ func TestImportLocal(t *testing.T) { // retrieve as UnixFS. out1 := filepath.Join(dir, "retrieval1.data") // as unixfs out2 := filepath.Join(dir, "retrieval2.data") // as car - err = a.ClientRetrieve(ctx, order, &api.FileRef{ + err = a.ClientExport(ctx, order, api.FileRef{ Path: out1, }) require.NoError(t, err) @@ -77,7 +77,7 @@ func TestImportLocal(t *testing.T) { require.NoError(t, err) require.Equal(t, b, outBytes) - err = a.ClientRetrieve(ctx, order, &api.FileRef{ + err = a.ClientExport(ctx, order, api.FileRef{ Path: out2, IsCAR: true, }) From b9bd061bdd88c94c9d7a67a09c06ad3e67fd15b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 10 Nov 2021 18:39:06 +0100 Subject: [PATCH 05/33] fix unixfs selector root node selection --- node/impl/client/client.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 34bde7e26..97ed9f289 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -951,7 +951,7 @@ func (a *API) ClientExport(ctx context.Context, exportRef api.ExportRef, ref api return a.outputCAR(ctx, exportRef.Root, sel, retrievalBs, ref) } - return a.outputUnixFS(ctx, exportRef.Root, exportRef.DatamodelPathSelector != nil, sel, retrievalBs, ref) + return a.outputUnixFS(ctx, exportRef.Root, exportRef.DatamodelPathSelector, retrievalBs, ref) } func (a *API) outputCAR(ctx context.Context, root cid.Cid, sel datamodel.Node, bs bstore.Blockstore, ref api.FileRef) error { @@ -977,12 +977,24 @@ func (a *API) outputCAR(ctx context.Context, root cid.Cid, sel datamodel.Node, b return f.Close() } -func (a *API) outputUnixFS(ctx context.Context, root cid.Cid, findSubroot bool, sel datamodel.Node, bs bstore.Blockstore, ref api.FileRef) error { +func (a *API) outputUnixFS(ctx context.Context, root cid.Cid, sels *textselector.Expression, bs bstore.Blockstore, ref api.FileRef) error { ds := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) // if we used a selector - need to find the sub-root the user actually wanted to retrieve - if findSubroot { + if sels != nil { var subRootFound bool + sel := selectorparse.CommonSelector_ExploreAllRecursively + + if strings.HasPrefix(string(*sels), "{") { + var err error + sel, err = selectorparse.ParseJSONSelector(string(*sels)) + if err != nil { + return xerrors.Errorf("failed to parse json-selector '%s': %w", *sels, err) + } + } else { + selspec, _ := textselector.SelectorSpecFromPath(*sels, nil) //nolint:errcheck + sel = selspec.Node() + } if err := utils.TraverseDag( ctx, From d0503d409fef1ba77ba6cf5726ff1918141c2dec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 10 Nov 2021 18:57:10 +0100 Subject: [PATCH 06/33] fix TestPartialRetrieval --- itests/deals_partial_retrieval_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/itests/deals_partial_retrieval_test.go b/itests/deals_partial_retrieval_test.go index 8c7775588..4b3b90f02 100644 --- a/itests/deals_partial_retrieval_test.go +++ b/itests/deals_partial_retrieval_test.go @@ -137,13 +137,14 @@ func TestPartialRetrieval(t *testing.T) { DatamodelPathSelector: &textSelectorNonexistent, }, api.ExportRef{ + Root: carRoot, FromLocalCAR: sourceCar, DatamodelPathSelector: &textSelectorNonexistent, }, &api.FileRef{}, nil, ), - fmt.Sprintf("retrieval failed: path selection '%s' does not match a node within %s", textSelectorNonexistent, carRoot), + fmt.Sprintf("path selection does not match a node within %s", carRoot), ) // ensure non-boundary retrievals fail @@ -157,13 +158,14 @@ func TestPartialRetrieval(t *testing.T) { DatamodelPathSelector: &textSelectorNonLink, }, api.ExportRef{ + Root: carRoot, FromLocalCAR: sourceCar, DatamodelPathSelector: &textSelectorNonLink, }, &api.FileRef{}, nil, ), - fmt.Sprintf("retrieval failed: error while locating partial retrieval sub-root: unsupported selection path '%s' does not correspond to a block boundary (a.k.a. CID link)", textSelectorNonLink), + fmt.Sprintf("error while locating partial retrieval sub-root: unsupported selection path '%s' does not correspond to a block boundary (a.k.a. CID link)", textSelectorNonLink), ) } From b0c043cc2ff7ed0fdcc44468f5a7975be57bfbd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 10 Nov 2021 18:59:58 +0100 Subject: [PATCH 07/33] make gen --- api/mocks/mock_full.go | 14 ++++++++++++++ documentation/en/api-v1-unstable-methods.md | 16 ++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/api/mocks/mock_full.go b/api/mocks/mock_full.go index 4b18eb365..3f9d75433 100644 --- a/api/mocks/mock_full.go +++ b/api/mocks/mock_full.go @@ -816,6 +816,20 @@ func (mr *MockFullNodeMockRecorder) ClientRetrieveTryRestartInsufficientFunds(ar return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieveTryRestartInsufficientFunds", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieveTryRestartInsufficientFunds), arg0, arg1) } +// ClientRetrieveWait mocks base method. +func (m *MockFullNode) ClientRetrieveWait(arg0 context.Context, arg1 retrievalmarket.DealID) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClientRetrieveWait", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// ClientRetrieveWait indicates an expected call of ClientRetrieveWait. +func (mr *MockFullNodeMockRecorder) ClientRetrieveWait(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieveWait", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieveWait), arg0, arg1) +} + // ClientStartDeal mocks base method. func (m *MockFullNode) ClientStartDeal(arg0 context.Context, arg1 *api.StartDealParams) (*cid.Cid, error) { m.ctrl.T.Helper() diff --git a/documentation/en/api-v1-unstable-methods.md b/documentation/en/api-v1-unstable-methods.md index 17cb16339..54cfda089 100644 --- a/documentation/en/api-v1-unstable-methods.md +++ b/documentation/en/api-v1-unstable-methods.md @@ -60,6 +60,7 @@ * [ClientRestartDataTransfer](#ClientRestartDataTransfer) * [ClientRetrieve](#ClientRetrieve) * [ClientRetrieveTryRestartInsufficientFunds](#ClientRetrieveTryRestartInsufficientFunds) + * [ClientRetrieveWait](#ClientRetrieveWait) * [ClientStartDeal](#ClientStartDeal) * [ClientStatelessDeal](#ClientStatelessDeal) * [Create](#Create) @@ -1549,6 +1550,21 @@ Inputs: Response: `{}` +### ClientRetrieveWait +ClientRetrieveWait waits for retrieval to be complete + + +Perms: admin + +Inputs: +```json +[ + 5 +] +``` + +Response: `{}` + ### ClientStartDeal ClientStartDeal proposes a deal with a miner. From b26906963b5a19c69c340861b61af86fe4fa4bb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 11 Nov 2021 16:17:39 +0100 Subject: [PATCH 08/33] retrieval: Support multi-root export --- api/api_full.go | 7 +- api/types.go | 34 ++++- api/v0api/v1_wrapper.go | 14 +- cli/client.go | 13 +- itests/deals_partial_retrieval_test.go | 43 ++++--- node/impl/client/client.go | 171 +++++++++++++++---------- 6 files changed, 182 insertions(+), 100 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 48b5d0d3c..139f0326c 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -7,7 +7,6 @@ import ( "time" "github.com/ipfs/go-cid" - textselector "github.com/ipld/go-ipld-selector-text-lite" "github.com/libp2p/go-libp2p-core/peer" "github.com/filecoin-project/go-address" @@ -936,9 +935,9 @@ type MarketDeal struct { type RetrievalOrder struct { // TODO: make this less unixfs specific Root cid.Cid - Piece *cid.Cid - DatamodelPathSelector *textselector.Expression - Size uint64 + Piece *cid.Cid + DataSelector *Selector + Size uint64 Total types.BigInt UnsealPrice types.BigInt diff --git a/api/types.go b/api/types.go index a4c477545..05f86fa88 100644 --- a/api/types.go +++ b/api/types.go @@ -5,12 +5,10 @@ import ( "fmt" "time" - "github.com/filecoin-project/go-fil-markets/retrievalmarket" - "github.com/filecoin-project/lotus/chain/types" - textselector "github.com/ipld/go-ipld-selector-text-lite" - datatransfer "github.com/filecoin-project/go-data-transfer" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" @@ -204,9 +202,35 @@ type RestrievalRes struct { DealID retrievalmarket.DealID } +// Selector specifies ipld selector string +// - if the string starts with '{', it's interpreted as json selector string +// see https://ipld.io/specs/selectors/ and https://ipld.io/specs/selectors/fixtures/selector-fixtures-1/ +// - otherwise the string is interpreted as ipld-selector-text-lite (simple ipld path) +// see https://github.com/ipld/go-ipld-selector-text-lite +type Selector string + +type DagSpec struct { + // RootSelector specifies root node + // - when using textselector, the path specifies the root node + // - if nil then RootSelector is inferred from DataSelector + // - must match a single node + RootSelector *Selector + + // DataSelector matches data to be retrieved + // - when using textselector, the path specifies subtree + DataSelector *Selector +} + type ExportRef struct { Root cid.Cid - DatamodelPathSelector *textselector.Expression + + // DAGs array specifies a list of DAGs to export + // - If exporting into a car file, defines car roots + // - If exporting into unixfs files, only one DAG is supported, DataSelector is ignored + // - When not specified defaults to a single DAG: + // - Root - the root node: `{".": {}}` + // - Data - the entire DAG: `{"R":{"l":{"none":{}},":>":{"a":{">":{"@":{}}}}}}` + DAGs []DagSpec FromLocalCAR string // if specified, get data from a local CARv2 file. DealID retrievalmarket.DealID diff --git a/api/v0api/v1_wrapper.go b/api/v0api/v1_wrapper.go index 0a1a463e5..5418d99c7 100644 --- a/api/v0api/v1_wrapper.go +++ b/api/v0api/v1_wrapper.go @@ -320,12 +320,20 @@ func (w *WrapperV1Full) clientRetrieve(ctx context.Context, order RetrievalOrder return } - finish(w.ClientExport(ctx, api.ExportRef{ + eref := api.ExportRef{ Root: order.Root, - DatamodelPathSelector: order.DatamodelPathSelector, FromLocalCAR: order.FromLocalCAR, DealID: dealID, - }, *ref)) + } + + if order.DatamodelPathSelector != nil { + s := api.Selector(*order.DatamodelPathSelector) + eref.DAGs = append(eref.DAGs, api.DagSpec{ + DataSelector: &s, + }) + } + + finish(w.ClientExport(ctx, eref, *ref)) } var _ FullNode = &WrapperV1Full{} diff --git a/cli/client.go b/cli/client.go index 4aa64ef55..91e431eb0 100644 --- a/cli/client.go +++ b/cli/client.go @@ -26,7 +26,6 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/ipfs/go-cid" "github.com/ipfs/go-cidutil/cidenc" - textselector "github.com/ipld/go-ipld-selector-text-lite" "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multibase" "github.com/urfave/cli/v2" @@ -1202,10 +1201,16 @@ var clientRetrieveCmd = &cli.Command{ continue } } + + event := "New" + if evt.Event != nil { + event = retrievalmarket.ClientEvents[*evt.Event] + } + afmt.Printf("> Recv: %s, Paid %s, %s (%s)\n", types.SizeStr(types.NewInt(evt.BytesReceived)), types.FIL(evt.TotalPaid), - retrievalmarket.ClientEvents[*evt.Event], + event, retrievalmarket.DealStatuses[evt.Status], ) switch evt.Status { @@ -1226,8 +1231,8 @@ var clientRetrieveCmd = &cli.Command{ } } - if sel := textselector.Expression(cctx.String("datamodel-path-selector")); sel != "" { - eref.DatamodelPathSelector = &sel + if sel := api.Selector(cctx.String("datamodel-path-selector")); sel != "" { + eref.DAGs = append(eref.DAGs, api.DagSpec{DataSelector: &sel}) } err = fapi.ClientExport(ctx, *eref, lapi.FileRef{ diff --git a/itests/deals_partial_retrieval_test.go b/itests/deals_partial_retrieval_test.go index 4b3b90f02..9eeae3692 100644 --- a/itests/deals_partial_retrieval_test.go +++ b/itests/deals_partial_retrieval_test.go @@ -20,7 +20,6 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipld/go-car" - textselector "github.com/ipld/go-ipld-selector-text-lite" "github.com/stretchr/testify/require" ) @@ -31,9 +30,10 @@ var ( carRoot, _ = cid.Parse("bafy2bzacecnamqgqmifpluoeldx7zzglxcljo6oja4vrmtj7432rphldpdmm2") carCommp, _ = cid.Parse("baga6ea4seaqmrivgzei3fmx5qxtppwankmtou6zvigyjaveu3z2zzwhysgzuina") carPieceSize = abi.PaddedPieceSize(2097152) - textSelector = textselector.Expression("8/1/8/1/0/1/0") - textSelectorNonLink = textselector.Expression("8/1/8/1/0/1") - textSelectorNonexistent = textselector.Expression("42") + textSelector = api.Selector("8/1/8/1/0/1/0") + storPowCid, _ = cid.Parse("bafkqaetgnfwc6mjpon2g64tbm5sxa33xmvza") + textSelectorNonLink = api.Selector("8/1/8/1/0/1") + textSelectorNonexistent = api.Selector("42") expectedResult = "fil/1/storagepower" ) @@ -94,8 +94,10 @@ func TestPartialRetrieval(t *testing.T) { retOrder = offers[0].Order(caddr) } - retOrder.DatamodelPathSelector = &textSelector - eref.DatamodelPathSelector = &textSelector + retOrder.DataSelector = &textSelector + eref.DAGs = append(eref.DAGs, api.DagSpec{ + DataSelector: &textSelector, + }) eref.Root = carRoot // test retrieval of either data or constructing a partial selective-car @@ -113,6 +115,7 @@ func TestPartialRetrieval(t *testing.T) { Path: outFile.Name(), IsCAR: retrieveAsCar, }, + storPowCid, outFile, )) @@ -133,18 +136,19 @@ func TestPartialRetrieval(t *testing.T) { ctx, client, api.RetrievalOrder{ - Root: carRoot, - DatamodelPathSelector: &textSelectorNonexistent, + Root: carRoot, + DataSelector: &textSelectorNonexistent, }, api.ExportRef{ Root: carRoot, FromLocalCAR: sourceCar, - DatamodelPathSelector: &textSelectorNonexistent, + DAGs: []api.DagSpec{{DataSelector: &textSelectorNonexistent}}, }, &api.FileRef{}, + storPowCid, nil, ), - fmt.Sprintf("path selection does not match a node within %s", carRoot), + fmt.Sprintf("parsing dag spec: path selection does not match a node within %s", carRoot), ) // ensure non-boundary retrievals fail @@ -154,22 +158,23 @@ func TestPartialRetrieval(t *testing.T) { ctx, client, api.RetrievalOrder{ - Root: carRoot, - DatamodelPathSelector: &textSelectorNonLink, + Root: carRoot, + DataSelector: &textSelectorNonLink, }, api.ExportRef{ Root: carRoot, FromLocalCAR: sourceCar, - DatamodelPathSelector: &textSelectorNonLink, + DAGs: []api.DagSpec{{DataSelector: &textSelectorNonLink}}, }, &api.FileRef{}, + storPowCid, nil, ), - fmt.Sprintf("error while locating partial retrieval sub-root: unsupported selection path '%s' does not correspond to a block boundary (a.k.a. CID link)", textSelectorNonLink), + fmt.Sprintf("parsing dag spec: error while locating partial retrieval sub-root: unsupported selection path '%s' does not correspond to a block boundary (a.k.a. CID link)", textSelectorNonLink), ) } -func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrder api.RetrievalOrder, eref api.ExportRef, retRef *api.FileRef, outFile *os.File) error { +func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrder api.RetrievalOrder, eref api.ExportRef, retRef *api.FileRef, expRootCid cid.Cid, outFile *os.File) error { if retOrder.Total.Nil() { retOrder.Total = big.Zero() @@ -212,7 +217,7 @@ func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrde if len(cr.Header.Roots) != 1 { return fmt.Errorf("expected a single root in result car, got %d", len(cr.Header.Roots)) - } else if cr.Header.Roots[0].String() != carRoot.String() { + } else if cr.Header.Roots[0].String() != expRootCid.String() { return fmt.Errorf("expected root cid '%s', got '%s'", carRoot.String(), cr.Header.Roots[0].String()) } @@ -228,11 +233,11 @@ func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrde blks = append(blks, b) } - if len(blks) != 3 { - return fmt.Errorf("expected a car file with 3 blocks, got one with %d instead", len(blks)) + if len(blks) != 1 { + return fmt.Errorf("expected a car file with 1 blocks, got one with %d instead", len(blks)) } - data = blks[2].RawData() + data = blks[0].RawData() } if string(data) != expectedResult { diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 97ed9f289..5ea620110 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -12,6 +12,7 @@ import ( "time" bstore "github.com/ipfs/go-ipfs-blockstore" + format "github.com/ipfs/go-ipld-format" unixfile "github.com/ipfs/go-unixfs/file" "github.com/ipld/go-car" carv2 "github.com/ipld/go-car/v2" @@ -761,7 +762,7 @@ func (a *API) ClientCancelRetrievalDeal(ctx context.Context, dealID rm.DealID) e } } -func getRetrievalSelector(dps *textselector.Expression) (datamodel.Node, error) { +func getDataSelector(dps *api.Selector) (datamodel.Node, error) { sel := selectorparse.CommonSelector_ExploreAllRecursively if dps != nil { @@ -775,7 +776,7 @@ func getRetrievalSelector(dps *textselector.Expression) (datamodel.Node, error) ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) selspec, err := textselector.SelectorSpecFromPath( - *dps, + textselector.Expression(*dps), // URGH - this is a direct copy from https://github.com/filecoin-project/go-fil-markets/blob/v1.12.0/shared/selectors.go#L10-L16 // Unable to use it because we need the SelectorSpec, and markets exposes just a reified node @@ -797,7 +798,7 @@ func getRetrievalSelector(dps *textselector.Expression) (datamodel.Node, error) } func (a *API) ClientRetrieve(ctx context.Context, params api.RetrievalOrder) (*api.RestrievalRes, error) { - sel, err := getRetrievalSelector(params.DatamodelPathSelector) + sel, err := getDataSelector(params.DataSelector) if err != nil { return nil, err } @@ -914,11 +915,6 @@ func (a *API) ClientExport(ctx context.Context, exportRef api.ExportRef, ref api carBss, retrieveIntoCAR := a.RtvlBlockstoreAccessor.(*retrievaladapter.CARBlockstoreAccessor) carPath := exportRef.FromLocalCAR - sel, err := getRetrievalSelector(exportRef.DatamodelPathSelector) - if err != nil { - return err - } - if carPath == "" { if !retrieveIntoIPFS && !retrieveIntoCAR { return xerrors.Errorf("unsupported retrieval blockstore accessor") @@ -941,33 +937,48 @@ func (a *API) ClientExport(ctx context.Context, exportRef api.ExportRef, ref api retrievalBs = cbs } + dserv := merkledag.NewDAGService(blockservice.New(retrievalBs, offline.Exchange(retrievalBs))) + roots, err := parseDagSpec(ctx, exportRef.Root, exportRef.DAGs, dserv) + if err != nil { + return xerrors.Errorf("parsing dag spec: %w", err) + } + // Are we outputting a CAR? if ref.IsCAR { // not IPFS and we do full selection - just extract the CARv1 from the CARv2 we stored the retrieval in - if !retrieveIntoIPFS && exportRef.DatamodelPathSelector == nil { + if !retrieveIntoIPFS && len(exportRef.DAGs) == 0 { return carv2.ExtractV1File(carPath, ref.Path) } - return a.outputCAR(ctx, exportRef.Root, sel, retrievalBs, ref) + return a.outputCAR(ctx, roots, retrievalBs, ref) } - return a.outputUnixFS(ctx, exportRef.Root, exportRef.DatamodelPathSelector, retrievalBs, ref) + if len(roots) != 1 { + return xerrors.Errorf("unixfs retrieval requires one root node, got %d", len(roots)) + } + + return a.outputUnixFS(ctx, roots[0].root, dserv, ref) } -func (a *API) outputCAR(ctx context.Context, root cid.Cid, sel datamodel.Node, bs bstore.Blockstore, ref api.FileRef) error { +func (a *API) outputCAR(ctx context.Context, dags []dagSpec, bs bstore.Blockstore, ref api.FileRef) error { // generating a CARv1 from the configured blockstore f, err := os.OpenFile(ref.Path, os.O_CREATE|os.O_WRONLY, 0644) if err != nil { return err } + carDags := make([]car.Dag, len(dags)) + for i, dag := range dags { + carDags[i] = car.Dag{ + Root: dag.root, + Selector: dag.selector, + } + } + err = car.NewSelectiveCar( ctx, bs, - []car.Dag{{ - Root: root, - Selector: sel, - }}, + carDags, car.MaxTraversalLinks(config.MaxTraversalLinks), ).Write(f) if err != nil { @@ -977,60 +988,88 @@ func (a *API) outputCAR(ctx context.Context, root cid.Cid, sel datamodel.Node, b return f.Close() } -func (a *API) outputUnixFS(ctx context.Context, root cid.Cid, sels *textselector.Expression, bs bstore.Blockstore, ref api.FileRef) error { - ds := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) +type dagSpec struct { + root cid.Cid + selector ipld.Node +} - // if we used a selector - need to find the sub-root the user actually wanted to retrieve - if sels != nil { - var subRootFound bool - sel := selectorparse.CommonSelector_ExploreAllRecursively - - if strings.HasPrefix(string(*sels), "{") { - var err error - sel, err = selectorparse.ParseJSONSelector(string(*sels)) - if err != nil { - return xerrors.Errorf("failed to parse json-selector '%s': %w", *sels, err) - } - } else { - selspec, _ := textselector.SelectorSpecFromPath(*sels, nil) //nolint:errcheck - sel = selspec.Node() - } - - if err := utils.TraverseDag( - ctx, - ds, - root, - sel, - func(p traversal.Progress, n ipld.Node, r traversal.VisitReason) error { - if r == traversal.VisitReason_SelectionMatch { - - if p.LastBlock.Path.String() != p.Path.String() { - return xerrors.Errorf("unsupported selection path '%s' does not correspond to a block boundary (a.k.a. CID link)", p.Path.String()) - } - - if p.LastBlock.Link == nil { - return nil - } - - cidLnk, castOK := p.LastBlock.Link.(cidlink.Link) - if !castOK { - return xerrors.Errorf("cidlink cast unexpectedly failed on '%s'", p.LastBlock.Link) - } - - root = cidLnk.Cid - subRootFound = true - } - return nil +func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds format.DAGService) ([]dagSpec, error) { + if len(dsp) == 0 { + return []dagSpec{ + { + root: root, + selector: nil, }, - ); err != nil { - return xerrors.Errorf("error while locating partial retrieval sub-root: %w", err) + }, nil + } + + out := make([]dagSpec, len(dsp)) + for i, spec := range dsp { + if spec.RootSelector == nil { + spec.RootSelector = spec.DataSelector } - if !subRootFound { - return xerrors.Errorf("path selection does not match a node within %s", root) + if spec.RootSelector != nil { + var rsn ipld.Node + + if strings.HasPrefix(string(*spec.RootSelector), "{") { + var err error + rsn, err = selectorparse.ParseJSONSelector(string(*spec.RootSelector)) + if err != nil { + return nil, xerrors.Errorf("failed to parse json-selector '%s': %w", *spec.RootSelector, err) + } + } else { + selspec, _ := textselector.SelectorSpecFromPath(textselector.Expression(*spec.RootSelector), nil) //nolint:errcheck + rsn = selspec.Node() + } + + if err := utils.TraverseDag( + ctx, + ds, + root, + rsn, + func(p traversal.Progress, n ipld.Node, r traversal.VisitReason) error { + if r == traversal.VisitReason_SelectionMatch { + + if p.LastBlock.Path.String() != p.Path.String() { + return xerrors.Errorf("unsupported selection path '%s' does not correspond to a block boundary (a.k.a. CID link)", p.Path.String()) + } + + if p.LastBlock.Link == nil { + return nil + } + + cidLnk, castOK := p.LastBlock.Link.(cidlink.Link) + if !castOK { + return xerrors.Errorf("cidlink cast unexpectedly failed on '%s'", p.LastBlock.Link) + } + + out[i].root = cidLnk.Cid + } + return nil + }, + ); err != nil { + return nil, xerrors.Errorf("error while locating partial retrieval sub-root: %w", err) + } + + if out[i].root == cid.Undef { + return nil, xerrors.Errorf("path selection does not match a node within %s", root) + } + } + + if spec.DataSelector != nil { + var err error + out[i].selector, err = getDataSelector(spec.DataSelector) + if err != nil { + return nil, err + } } } + return out, nil +} + +func (a *API) outputUnixFS(ctx context.Context, root cid.Cid, ds format.DAGService, ref api.FileRef) error { nd, err := ds.Get(ctx, root) if err != nil { return xerrors.Errorf("ClientRetrieve: %w", err) @@ -1072,8 +1111,10 @@ func (a *API) ClientListRetrievals(ctx context.Context) ([]api.RetrievalInfo, er func (a *API) ClientGetRetrievalUpdates(ctx context.Context) (<-chan api.RetrievalInfo, error) { updates := make(chan api.RetrievalInfo) - unsub := a.Retrieval.SubscribeToEvents(func(_ rm.ClientEvent, deal rm.ClientDealState) { - updates <- a.newRetrievalInfo(ctx, deal) + unsub := a.Retrieval.SubscribeToEvents(func(evt rm.ClientEvent, deal rm.ClientDealState) { + update := a.newRetrievalInfo(ctx, deal) + update.Event = &evt + updates <- update }) go func() { From b83a9b902a149838e188002d011f78c51b8a57da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 11 Nov 2021 16:17:54 +0100 Subject: [PATCH 09/33] gofmt --- api/api_full.go | 2 +- api/types.go | 2 +- api/v0api/v1_wrapper.go | 6 +++--- itests/deals_partial_retrieval_test.go | 14 +++++++------- node/impl/client/client.go | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 139f0326c..8c720588b 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -934,7 +934,7 @@ type MarketDeal struct { type RetrievalOrder struct { // TODO: make this less unixfs specific - Root cid.Cid + Root cid.Cid Piece *cid.Cid DataSelector *Selector Size uint64 diff --git a/api/types.go b/api/types.go index 05f86fa88..330263f8a 100644 --- a/api/types.go +++ b/api/types.go @@ -222,7 +222,7 @@ type DagSpec struct { } type ExportRef struct { - Root cid.Cid + Root cid.Cid // DAGs array specifies a list of DAGs to export // - If exporting into a car file, defines car roots diff --git a/api/v0api/v1_wrapper.go b/api/v0api/v1_wrapper.go index 5418d99c7..4626f0d06 100644 --- a/api/v0api/v1_wrapper.go +++ b/api/v0api/v1_wrapper.go @@ -321,9 +321,9 @@ func (w *WrapperV1Full) clientRetrieve(ctx context.Context, order RetrievalOrder } eref := api.ExportRef{ - Root: order.Root, - FromLocalCAR: order.FromLocalCAR, - DealID: dealID, + Root: order.Root, + FromLocalCAR: order.FromLocalCAR, + DealID: dealID, } if order.DatamodelPathSelector != nil { diff --git a/itests/deals_partial_retrieval_test.go b/itests/deals_partial_retrieval_test.go index 9eeae3692..bb9611594 100644 --- a/itests/deals_partial_retrieval_test.go +++ b/itests/deals_partial_retrieval_test.go @@ -31,7 +31,7 @@ var ( carCommp, _ = cid.Parse("baga6ea4seaqmrivgzei3fmx5qxtppwankmtou6zvigyjaveu3z2zzwhysgzuina") carPieceSize = abi.PaddedPieceSize(2097152) textSelector = api.Selector("8/1/8/1/0/1/0") - storPowCid, _ = cid.Parse("bafkqaetgnfwc6mjpon2g64tbm5sxa33xmvza") + storPowCid, _ = cid.Parse("bafkqaetgnfwc6mjpon2g64tbm5sxa33xmvza") textSelectorNonLink = api.Selector("8/1/8/1/0/1") textSelectorNonexistent = api.Selector("42") expectedResult = "fil/1/storagepower" @@ -140,9 +140,9 @@ func TestPartialRetrieval(t *testing.T) { DataSelector: &textSelectorNonexistent, }, api.ExportRef{ - Root: carRoot, - FromLocalCAR: sourceCar, - DAGs: []api.DagSpec{{DataSelector: &textSelectorNonexistent}}, + Root: carRoot, + FromLocalCAR: sourceCar, + DAGs: []api.DagSpec{{DataSelector: &textSelectorNonexistent}}, }, &api.FileRef{}, storPowCid, @@ -162,9 +162,9 @@ func TestPartialRetrieval(t *testing.T) { DataSelector: &textSelectorNonLink, }, api.ExportRef{ - Root: carRoot, - FromLocalCAR: sourceCar, - DAGs: []api.DagSpec{{DataSelector: &textSelectorNonLink}}, + Root: carRoot, + FromLocalCAR: sourceCar, + DAGs: []api.DagSpec{{DataSelector: &textSelectorNonLink}}, }, &api.FileRef{}, storPowCid, diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 5ea620110..a32f13d06 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -989,7 +989,7 @@ func (a *API) outputCAR(ctx context.Context, dags []dagSpec, bs bstore.Blockstor } type dagSpec struct { - root cid.Cid + root cid.Cid selector ipld.Node } From 450d0687dacc71fa2e0605426838a8f6eae1f898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 11 Nov 2021 17:17:11 +0100 Subject: [PATCH 10/33] retrieval: REST export endpoint --- node/impl/client/client.go | 80 ++++++++++++++++++++++++++------------ node/rpc.go | 37 ++++++++++++++++++ 2 files changed, 93 insertions(+), 24 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index a32f13d06..e72ca7eae 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -910,7 +910,34 @@ func (a *API) ClientRetrieveWait(ctx context.Context, deal rm.DealID) error { } } +type ExportDest struct { + Writer io.Writer + Path string +} + +func (ed *ExportDest) doWrite(cb func(io.Writer) error) error { + if ed.Writer != nil { + return cb(ed.Writer) + } + + f, err := os.OpenFile(ed.Path, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + + if err := cb(f); err != nil { + _ = f.Close() + return err + } + + return f.Close() +} + func (a *API) ClientExport(ctx context.Context, exportRef api.ExportRef, ref api.FileRef) error { + return a.ClientExportInto(ctx, exportRef, ref.IsCAR, ExportDest{Path: ref.Path}) +} + +func (a *API) ClientExportInto(ctx context.Context, exportRef api.ExportRef, car bool, dest ExportDest) error { proxyBss, retrieveIntoIPFS := a.RtvlBlockstoreAccessor.(*retrievaladapter.ProxyBlockstoreAccessor) carBss, retrieveIntoCAR := a.RtvlBlockstoreAccessor.(*retrievaladapter.CARBlockstoreAccessor) carPath := exportRef.FromLocalCAR @@ -944,29 +971,24 @@ func (a *API) ClientExport(ctx context.Context, exportRef api.ExportRef, ref api } // Are we outputting a CAR? - if ref.IsCAR { + if car { // not IPFS and we do full selection - just extract the CARv1 from the CARv2 we stored the retrieval in - if !retrieveIntoIPFS && len(exportRef.DAGs) == 0 { - return carv2.ExtractV1File(carPath, ref.Path) + if !retrieveIntoIPFS && len(exportRef.DAGs) == 0 && dest.Writer == nil { + return carv2.ExtractV1File(carPath, dest.Path) } - return a.outputCAR(ctx, roots, retrievalBs, ref) + return a.outputCAR(ctx, roots, retrievalBs, dest) } if len(roots) != 1 { return xerrors.Errorf("unixfs retrieval requires one root node, got %d", len(roots)) } - return a.outputUnixFS(ctx, roots[0].root, dserv, ref) + return a.outputUnixFS(ctx, roots[0].root, dserv, dest) } -func (a *API) outputCAR(ctx context.Context, dags []dagSpec, bs bstore.Blockstore, ref api.FileRef) error { +func (a *API) outputCAR(ctx context.Context, dags []dagSpec, bs bstore.Blockstore, dest ExportDest) error { // generating a CARv1 from the configured blockstore - f, err := os.OpenFile(ref.Path, os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return err - } - carDags := make([]car.Dag, len(dags)) for i, dag := range dags { carDags[i] = car.Dag{ @@ -975,17 +997,14 @@ func (a *API) outputCAR(ctx context.Context, dags []dagSpec, bs bstore.Blockstor } } - err = car.NewSelectiveCar( - ctx, - bs, - carDags, - car.MaxTraversalLinks(config.MaxTraversalLinks), - ).Write(f) - if err != nil { - return err - } - - return f.Close() + return dest.doWrite(func(w io.Writer) error { + return car.NewSelectiveCar( + ctx, + bs, + carDags, + car.MaxTraversalLinks(config.MaxTraversalLinks), + ).Write(w) + }) } type dagSpec struct { @@ -1069,7 +1088,7 @@ func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds forma return out, nil } -func (a *API) outputUnixFS(ctx context.Context, root cid.Cid, ds format.DAGService, ref api.FileRef) error { +func (a *API) outputUnixFS(ctx context.Context, root cid.Cid, ds format.DAGService, dest ExportDest) error { nd, err := ds.Get(ctx, root) if err != nil { return xerrors.Errorf("ClientRetrieve: %w", err) @@ -1079,7 +1098,20 @@ func (a *API) outputUnixFS(ctx context.Context, root cid.Cid, ds format.DAGServi return xerrors.Errorf("ClientRetrieve: %w", err) } - return files.WriteTo(file, ref.Path) + if dest.Writer == nil { + return files.WriteTo(file, dest.Path) + } + + switch f := file.(type) { + case files.File: + _, err = io.Copy(dest.Writer, f) + if err != nil { + return err + } + return nil + default: + return fmt.Errorf("file type %T is not supported", nd) + } } func (a *API) ClientListRetrievals(ctx context.Context) ([]api.RetrievalInfo, error) { diff --git a/node/rpc.go b/node/rpc.go index 9bcdb7388..6a3e55115 100644 --- a/node/rpc.go +++ b/node/rpc.go @@ -27,6 +27,7 @@ import ( "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/metrics/proxy" "github.com/filecoin-project/lotus/node/impl" + "github.com/filecoin-project/lotus/node/impl/client" ) var rpclog = logging.Logger("rpc") @@ -89,14 +90,22 @@ func FullNodeHandler(a v1api.FullNode, permissioned bool, opts ...jsonrpc.Server // Import handler handleImportFunc := handleImport(a.(*impl.FullNodeAPI)) + handleExportFunc := handleExport(a.(*impl.FullNodeAPI)) if permissioned { importAH := &auth.Handler{ Verify: a.AuthVerify, Next: handleImportFunc, } m.Handle("/rest/v0/import", importAH) + + exportAH := &auth.Handler{ + Verify: a.AuthVerify, + Next: handleExportFunc, + } + m.Handle("/rest/v0/export", exportAH) } else { m.HandleFunc("/rest/v0/import", handleImportFunc) + m.HandleFunc("/rest/v0/export", handleExportFunc) } // debugging @@ -169,6 +178,34 @@ func handleImport(a *impl.FullNodeAPI) func(w http.ResponseWriter, r *http.Reque } } +func handleExport(a *impl.FullNodeAPI) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" { + w.WriteHeader(404) + return + } + if !auth.HasPerm(r.Context(), nil, api.PermWrite) { + w.WriteHeader(401) + _ = json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing write permission"}) + return + } + + var eref api.ExportRef + if err := json.Unmarshal([]byte(r.FormValue("export")), &eref); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + car := r.FormValue("car") == "true" + + err := a.ClientExportInto(r.Context(), eref, car, client.ExportDest{Writer: w}) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } +} + func handleFractionOpt(name string, setter func(int)) http.HandlerFunc { return func(rw http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { From a1d5b2a29375fb6840e27028488516054ddd5a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 15 Nov 2021 13:53:28 +0100 Subject: [PATCH 11/33] retrieval: wip improved retrieval commands --- cli/client.go | 219 ----------------------- cli/client_retr.go | 423 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 423 insertions(+), 219 deletions(-) create mode 100644 cli/client_retr.go diff --git a/cli/client.go b/cli/client.go index 91e431eb0..9a8f20899 100644 --- a/cli/client.go +++ b/cli/client.go @@ -1028,225 +1028,6 @@ var clientFindCmd = &cli.Command{ }, } -const DefaultMaxRetrievePrice = "0.01" - -var clientRetrieveCmd = &cli.Command{ - Name: "retrieve", - Usage: "Retrieve data from network", - ArgsUsage: "[dataCid outputPath]", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "from", - Usage: "address to send transactions from", - }, - &cli.BoolFlag{ - Name: "car", - Usage: "export to a car file instead of a regular file", - }, - &cli.StringFlag{ - Name: "miner", - Usage: "miner address for retrieval, if not present it'll use local discovery", - }, - &cli.StringFlag{ - Name: "datamodel-path-selector", - Usage: "a rudimentary (DM-level-only) text-path selector, allowing for sub-selection within a deal", - }, - &cli.StringFlag{ - Name: "maxPrice", - Usage: fmt.Sprintf("maximum price the client is willing to consider (default: %s FIL)", DefaultMaxRetrievePrice), - }, - &cli.StringFlag{ - Name: "pieceCid", - Usage: "require data to be retrieved from a specific Piece CID", - }, - &cli.BoolFlag{ - Name: "allow-local", - }, - }, - Action: func(cctx *cli.Context) error { - if cctx.NArg() != 2 { - return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments")) - } - - fapi, closer, err := GetFullNodeAPIV1(cctx) - if err != nil { - return err - } - defer closer() - ctx := ReqContext(cctx) - afmt := NewAppFmt(cctx.App) - - var payer address.Address - if cctx.String("from") != "" { - payer, err = address.NewFromString(cctx.String("from")) - } else { - payer, err = fapi.WalletDefaultAddress(ctx) - } - if err != nil { - return err - } - - file, err := cid.Parse(cctx.Args().Get(0)) - if err != nil { - return err - } - - var pieceCid *cid.Cid - if cctx.String("pieceCid") != "" { - parsed, err := cid.Parse(cctx.String("pieceCid")) - if err != nil { - return err - } - pieceCid = &parsed - } - - var eref *lapi.ExportRef - if cctx.Bool("allow-local") { - imports, err := fapi.ClientListImports(ctx) - if err != nil { - return err - } - - for _, i := range imports { - if i.Root != nil && i.Root.Equals(file) { - eref = &lapi.ExportRef{ - Root: file, - FromLocalCAR: i.CARPath, - } - break - } - } - } - - // no local found, so make a retrieval - if eref == nil { - var offer api.QueryOffer - minerStrAddr := cctx.String("miner") - if minerStrAddr == "" { // Local discovery - offers, err := fapi.ClientFindData(ctx, file, pieceCid) - - var cleaned []api.QueryOffer - // filter out offers that errored - for _, o := range offers { - if o.Err == "" { - cleaned = append(cleaned, o) - } - } - - offers = cleaned - - // sort by price low to high - sort.Slice(offers, func(i, j int) bool { - return offers[i].MinPrice.LessThan(offers[j].MinPrice) - }) - if err != nil { - return err - } - - // TODO: parse offer strings from `client find`, make this smarter - if len(offers) < 1 { - fmt.Println("Failed to find file") - return nil - } - offer = offers[0] - } else { // Directed retrieval - minerAddr, err := address.NewFromString(minerStrAddr) - if err != nil { - return err - } - offer, err = fapi.ClientMinerQueryOffer(ctx, minerAddr, file, pieceCid) - if err != nil { - return err - } - } - if offer.Err != "" { - return fmt.Errorf("offer error: %s", offer.Err) - } - - maxPrice := types.MustParseFIL(DefaultMaxRetrievePrice) - - if cctx.String("maxPrice") != "" { - maxPrice, err = types.ParseFIL(cctx.String("maxPrice")) - if err != nil { - return xerrors.Errorf("parsing maxPrice: %w", err) - } - } - - if offer.MinPrice.GreaterThan(big.Int(maxPrice)) { - return xerrors.Errorf("failed to find offer satisfying maxPrice: %s", maxPrice) - } - - o := offer.Order(payer) - - subscribeEvents, err := fapi.ClientGetRetrievalUpdates(ctx) - if err != nil { - return xerrors.Errorf("error setting up retrieval updates: %w", err) - } - retrievalRes, err := fapi.ClientRetrieve(ctx, o) - if err != nil { - return xerrors.Errorf("error setting up retrieval: %w", err) - } - - readEvents: - for { - var evt api.RetrievalInfo - select { - case <-ctx.Done(): - return xerrors.New("Retrieval Timed Out") - case evt = <-subscribeEvents: - if evt.ID != retrievalRes.DealID { - // we can't check the deal ID ahead of time because: - // 1. We need to subscribe before retrieving. - // 2. We won't know the deal ID until after retrieving. - continue - } - } - - event := "New" - if evt.Event != nil { - event = retrievalmarket.ClientEvents[*evt.Event] - } - - afmt.Printf("> Recv: %s, Paid %s, %s (%s)\n", - types.SizeStr(types.NewInt(evt.BytesReceived)), - types.FIL(evt.TotalPaid), - event, - retrievalmarket.DealStatuses[evt.Status], - ) - switch evt.Status { - case retrievalmarket.DealStatusCompleted: - break readEvents - case retrievalmarket.DealStatusRejected: - return xerrors.Errorf("Retrieval Proposal Rejected: %s", evt.Message) - case - retrievalmarket.DealStatusDealNotFound, - retrievalmarket.DealStatusErrored: - return xerrors.Errorf("Retrieval Error: %s", evt.Message) - } - } - - eref = &lapi.ExportRef{ - Root: file, - DealID: retrievalRes.DealID, - } - } - - if sel := api.Selector(cctx.String("datamodel-path-selector")); sel != "" { - eref.DAGs = append(eref.DAGs, api.DagSpec{DataSelector: &sel}) - } - - err = fapi.ClientExport(ctx, *eref, lapi.FileRef{ - Path: cctx.Args().Get(1), - IsCAR: cctx.Bool("car"), - }) - if err != nil { - return err - } - afmt.Println("Success") - return nil - }, -} - var clientListRetrievalsCmd = &cli.Command{ Name: "list-retrievals", Usage: "List retrieval market deals", diff --git a/cli/client_retr.go b/cli/client_retr.go new file mode 100644 index 000000000..2639ab7cb --- /dev/null +++ b/cli/client_retr.go @@ -0,0 +1,423 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "sort" + + "github.com/filecoin-project/lotus/node/repo" + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cid" + offline "github.com/ipfs/go-ipfs-exchange-offline" + "github.com/ipfs/go-merkledag" + carv2 "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/blockstore" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-state-types/big" + lapi "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" +) + +const DefaultMaxRetrievePrice = "0.01" + +func retrieve(ctx context.Context, cctx *cli.Context, fapi lapi.FullNode, sel *lapi.Selector, printf func(string, ...interface{})) (*lapi.ExportRef, error) { + var payer address.Address + var err error + if cctx.String("from") != "" { + payer, err = address.NewFromString(cctx.String("from")) + } else { + payer, err = fapi.WalletDefaultAddress(ctx) + } + if err != nil { + return nil, err + } + + file, err := cid.Parse(cctx.Args().Get(0)) + if err != nil { + return nil, err + } + + var pieceCid *cid.Cid + if cctx.String("pieceCid") != "" { + parsed, err := cid.Parse(cctx.String("pieceCid")) + if err != nil { + return nil, err + } + pieceCid = &parsed + } + + var eref *lapi.ExportRef + if cctx.Bool("allow-local") { + imports, err := fapi.ClientListImports(ctx) + if err != nil { + return nil, err + } + + for _, i := range imports { + if i.Root != nil && i.Root.Equals(file) { + eref = &lapi.ExportRef{ + Root: file, + FromLocalCAR: i.CARPath, + } + break + } + } + } + + // no local found, so make a retrieval + if eref == nil { + var offer lapi.QueryOffer + minerStrAddr := cctx.String("miner") + if minerStrAddr == "" { // Local discovery + offers, err := fapi.ClientFindData(ctx, file, pieceCid) + + var cleaned []lapi.QueryOffer + // filter out offers that errored + for _, o := range offers { + if o.Err == "" { + cleaned = append(cleaned, o) + } + } + + offers = cleaned + + // sort by price low to high + sort.Slice(offers, func(i, j int) bool { + return offers[i].MinPrice.LessThan(offers[j].MinPrice) + }) + if err != nil { + return nil, err + } + + // TODO: parse offer strings from `client find`, make this smarter + if len(offers) < 1 { + fmt.Println("Failed to find file") + return nil, nil + } + offer = offers[0] + } else { // Directed retrieval + minerAddr, err := address.NewFromString(minerStrAddr) + if err != nil { + return nil, err + } + offer, err = fapi.ClientMinerQueryOffer(ctx, minerAddr, file, pieceCid) + if err != nil { + return nil, err + } + } + if offer.Err != "" { + return nil, fmt.Errorf("offer error: %s", offer.Err) + } + + maxPrice := types.MustParseFIL(DefaultMaxRetrievePrice) + + if cctx.String("maxPrice") != "" { + maxPrice, err = types.ParseFIL(cctx.String("maxPrice")) + if err != nil { + return nil, xerrors.Errorf("parsing maxPrice: %w", err) + } + } + + if offer.MinPrice.GreaterThan(big.Int(maxPrice)) { + return nil, xerrors.Errorf("failed to find offer satisfying maxPrice: %s", maxPrice) + } + + o := offer.Order(payer) + o.DataSelector = sel + + subscribeEvents, err := fapi.ClientGetRetrievalUpdates(ctx) + if err != nil { + return nil, xerrors.Errorf("error setting up retrieval updates: %w", err) + } + retrievalRes, err := fapi.ClientRetrieve(ctx, o) + if err != nil { + return nil, xerrors.Errorf("error setting up retrieval: %w", err) + } + + readEvents: + for { + var evt lapi.RetrievalInfo + select { + case <-ctx.Done(): + return nil, xerrors.New("Retrieval Timed Out") + case evt = <-subscribeEvents: + if evt.ID != retrievalRes.DealID { + // we can't check the deal ID ahead of time because: + // 1. We need to subscribe before retrieving. + // 2. We won't know the deal ID until after retrieving. + continue + } + } + + event := "New" + if evt.Event != nil { + event = retrievalmarket.ClientEvents[*evt.Event] + } + + printf("> Recv: %s, Paid %s, %s (%s)\n", + types.SizeStr(types.NewInt(evt.BytesReceived)), + types.FIL(evt.TotalPaid), + event, + retrievalmarket.DealStatuses[evt.Status], + ) + switch evt.Status { + case retrievalmarket.DealStatusCompleted: + break readEvents + case retrievalmarket.DealStatusRejected: + return nil, xerrors.Errorf("Retrieval Proposal Rejected: %s", evt.Message) + case + retrievalmarket.DealStatusDealNotFound, + retrievalmarket.DealStatusErrored: + return nil, xerrors.Errorf("Retrieval Error: %s", evt.Message) + } + } + + eref = &lapi.ExportRef{ + Root: file, + DealID: retrievalRes.DealID, + } + } + + return eref, nil +} + +var clientRetrieveCmd = &cli.Command{ + Name: "retrieve", + Subcommands: []*cli.Command{ + clientRetrieveCatCmd, + clientRetrieveLsCmd, + }, + Usage: "Retrieve data from network", + ArgsUsage: "[dataCid outputPath]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "from", + Usage: "address to send transactions from", + }, + &cli.BoolFlag{ + Name: "car", + Usage: "export to a car file instead of a regular file", + }, + &cli.StringFlag{ + Name: "miner", + Usage: "miner address for retrieval, if not present it'll use local discovery", + }, + &cli.StringFlag{ + Name: "datamodel-path-selector", + Usage: "a rudimentary (DM-level-only) text-path selector, allowing for sub-selection within a deal", + }, + &cli.StringFlag{ + Name: "maxPrice", + Usage: fmt.Sprintf("maximum price the client is willing to consider (default: %s FIL)", DefaultMaxRetrievePrice), + }, + &cli.StringFlag{ + Name: "pieceCid", + Usage: "require data to be retrieved from a specific Piece CID", + }, + &cli.BoolFlag{ + Name: "allow-local", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.NArg() != 2 { + return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments")) + } + + fapi, closer, err := GetFullNodeAPIV1(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + afmt := NewAppFmt(cctx.App) + + var s *lapi.Selector + if sel := lapi.Selector(cctx.String("datamodel-path-selector")); sel != "" { + s = &sel + } + + eref, err := retrieve(ctx, cctx, fapi, s, afmt.Printf) + if err != nil { + return err + } + + if s != nil { + eref.DAGs = append(eref.DAGs, lapi.DagSpec{DataSelector: s}) + } + + err = fapi.ClientExport(ctx, *eref, lapi.FileRef{ + Path: cctx.Args().Get(1), + IsCAR: cctx.Bool("car"), + }) + if err != nil { + return err + } + afmt.Println("Success") + return nil + }, +} + +func ClientExportStream(apiAddr string, apiAuth http.Header, eref lapi.ExportRef, car bool) (io.ReadCloser, error) { + rj, err := json.Marshal(eref) + if err != nil { + return nil, xerrors.Errorf("marshaling export ref: %w", err) + } + + aa, err := url.Parse(apiAddr) + if err != nil { + return nil, xerrors.Errorf("parsing api address: %w", err) + } + switch aa.Scheme { + case "ws": + aa.Scheme = "http" + case "wss": + aa.Scheme = "https" + } + + aa.Path = path.Join(aa.Path, "rest/v0/export") + req, err := http.NewRequest("GET", fmt.Sprintf("%s?car=%t&export=%s", aa, car, url.QueryEscape(string(rj))), nil) + if err != nil { + return nil, err + } + + req.Header = apiAuth + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + + return resp.Body, nil +} + +var clientRetrieveCatCmd = &cli.Command{ + Name: "cat", + Usage: "Show data from network", + ArgsUsage: "[dataCid]", + Action: func(cctx *cli.Context) error { + if cctx.NArg() != 1 { + return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments")) + } + + ainfo, err := GetAPIInfo(cctx, repo.FullNode) + if err != nil { + return xerrors.Errorf("could not get API info: %w", err) + } + + fapi, closer, err := GetFullNodeAPIV1(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + afmt := NewAppFmt(cctx.App) + + // todo selector + eref, err := retrieve(ctx, cctx, fapi, nil, afmt.Printf) + if err != nil { + return err + } + + if sel := lapi.Selector(cctx.String("datamodel-path-selector")); sel != "" { + eref.DAGs = append(eref.DAGs, lapi.DagSpec{DataSelector: &sel}) + } + + rc, err := ClientExportStream(ainfo.Addr, ainfo.AuthHeader(), *eref, false) + if err != nil { + return err + } + defer rc.Close() // nolint + + _, err = io.Copy(os.Stdout, rc) + return err + }, +} + +var clientRetrieveLsCmd = &cli.Command{ + Name: "ls", + Usage: "Show object links", + ArgsUsage: "[dataCid]", + Action: func(cctx *cli.Context) error { + if cctx.NArg() != 1 { + return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments")) + } + + ainfo, err := GetAPIInfo(cctx, repo.FullNode) + if err != nil { + return xerrors.Errorf("could not get API info: %w", err) + } + + fapi, closer, err := GetFullNodeAPIV1(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + afmt := NewAppFmt(cctx.App) + + rootSelector := lapi.Selector(`{".": {}}`) + dataSelector := lapi.Selector(`{"R":{"l":{"depth":1},":>":{"a":{">":{"@":{}}}}}}`) + + eref, err := retrieve(ctx, cctx, fapi, &dataSelector, afmt.Printf) + if err != nil { + return err + } + + eref.DAGs = append(eref.DAGs, lapi.DagSpec{ + RootSelector: &rootSelector, + DataSelector: &dataSelector, + }) + + rc, err := ClientExportStream(ainfo.Addr, ainfo.AuthHeader(), *eref, true) + if err != nil { + return err + } + defer rc.Close() // nolint + + var memcar bytes.Buffer + _, err = io.Copy(&memcar, rc) + if err != nil { + return err + } + + cbs, err := blockstore.NewReadOnly(bytes.NewReader(memcar.Bytes()), nil, + carv2.ZeroLengthSectionAsEOF(true), + blockstore.UseWholeCIDs(true)) + if err != nil { + return xerrors.Errorf("opening car blockstore: %w", err) + } + + roots, err := cbs.Roots() + if err != nil { + return xerrors.Errorf("getting roots: %w", err) + } + + if len(roots) != 1 { + return xerrors.Errorf("expected 1 car root, got %d") + } + + dserv := merkledag.NewDAGService(blockservice.New(cbs, offline.Exchange(cbs))) + + links, err := dserv.GetLinks(ctx, roots[0]) + if err != nil { + return xerrors.Errorf("getting links: %w", err) + } + + for _, link := range links { + fmt.Printf("%s %s\t%d\n", link.Cid, link.Name, link.Size) + } + + return err + }, +} From 9c119bfdad3e78da1145476c0f9f4eeb7a965b36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 15 Nov 2021 17:27:20 +0100 Subject: [PATCH 12/33] retrieval: Make the ls command work --- cli/client_retr.go | 32 ++++++++++++++++++++++++++++++-- node/impl/client/client.go | 5 +++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/cli/client_retr.go b/cli/client_retr.go index 2639ab7cb..4d28cca00 100644 --- a/cli/client_retr.go +++ b/cli/client_retr.go @@ -5,6 +5,8 @@ import ( "context" "encoding/json" "fmt" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" "io" "net/http" "net/url" @@ -274,6 +276,17 @@ func ClientExportStream(apiAddr string, apiAuth http.Header, eref lapi.ExportRef return nil, xerrors.Errorf("marshaling export ref: %w", err) } + ma, err := multiaddr.NewMultiaddr(apiAddr) + if err == nil { + _, addr, err := manet.DialArgs(ma) + if err != nil { + return nil, err + } + + // todo: make cliutil helpers for this + apiAddr = "http://" + addr + } + aa, err := url.Parse(apiAddr) if err != nil { return nil, xerrors.Errorf("parsing api address: %w", err) @@ -298,6 +311,11 @@ func ClientExportStream(apiAddr string, apiAuth http.Header, eref lapi.ExportRef return nil, err } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() // nolint + return nil, xerrors.Errorf("getting root car: http %d", resp.StatusCode) + } + return resp.Body, nil } @@ -376,7 +394,7 @@ var clientRetrieveLsCmd = &cli.Command{ eref.DAGs = append(eref.DAGs, lapi.DagSpec{ RootSelector: &rootSelector, - DataSelector: &dataSelector, + DataSelector: &rootSelector, }) rc, err := ClientExportStream(ainfo.Addr, ainfo.AuthHeader(), *eref, true) @@ -391,7 +409,7 @@ var clientRetrieveLsCmd = &cli.Command{ return err } - cbs, err := blockstore.NewReadOnly(bytes.NewReader(memcar.Bytes()), nil, + cbs, err := blockstore.NewReadOnly(&bytesReaderAt{bytes.NewReader(memcar.Bytes())}, nil, carv2.ZeroLengthSectionAsEOF(true), blockstore.UseWholeCIDs(true)) if err != nil { @@ -421,3 +439,13 @@ var clientRetrieveLsCmd = &cli.Command{ return err }, } + +type bytesReaderAt struct { + btr *bytes.Reader +} + +func (b bytesReaderAt) ReadAt(p []byte, off int64) (n int, err error) { + return b.btr.ReadAt(p, off) +} + +var _ io.ReaderAt = &bytesReaderAt{} diff --git a/node/impl/client/client.go b/node/impl/client/client.go index e72ca7eae..28da94146 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -1055,6 +1055,11 @@ func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds forma } if p.LastBlock.Link == nil { + // this is likely the root node that we've matched here + // todo: is this a correct assumption + // todo: is the n ipld.Node above the node we want as the (sub)root? + // todo: how to go from ipld.Node to a cid? + out[i].root = root return nil } From 8ea5162ad9cf6ea8d7b44d1b70703ef78baedd12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 15 Nov 2021 17:40:05 +0100 Subject: [PATCH 13/33] retrieval: Cleanup retrieve command output --- cli/client_retr.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cli/client_retr.go b/cli/client_retr.go index 4d28cca00..3299cafbe 100644 --- a/cli/client_retr.go +++ b/cli/client_retr.go @@ -13,6 +13,8 @@ import ( "os" "path" "sort" + "strings" + "time" "github.com/filecoin-project/lotus/node/repo" "github.com/ipfs/go-blockservice" @@ -147,6 +149,7 @@ func retrieve(ctx context.Context, cctx *cli.Context, fapi lapi.FullNode, sel *l return nil, xerrors.Errorf("error setting up retrieval: %w", err) } + start := time.Now() readEvents: for { var evt lapi.RetrievalInfo @@ -167,12 +170,14 @@ func retrieve(ctx context.Context, cctx *cli.Context, fapi lapi.FullNode, sel *l event = retrievalmarket.ClientEvents[*evt.Event] } - printf("> Recv: %s, Paid %s, %s (%s)\n", + printf("Recv %s, Paid %s, %s (%s), %s\n", types.SizeStr(types.NewInt(evt.BytesReceived)), types.FIL(evt.TotalPaid), - event, - retrievalmarket.DealStatuses[evt.Status], + strings.TrimPrefix(event, "ClientEvent"), + strings.TrimPrefix(retrievalmarket.DealStatuses[evt.Status], "DealStatus"), + time.Now().Sub(start).Truncate(time.Millisecond), ) + switch evt.Status { case retrievalmarket.DealStatusCompleted: break readEvents From ec2bfb99bb59699d52a9a3170797f497e2a75784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 15 Nov 2021 18:27:36 +0100 Subject: [PATCH 14/33] make gen --- api/docgen/docgen.go | 2 ++ documentation/en/api-v1-unstable-methods.md | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index 498a0747c..11440619c 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -91,6 +91,7 @@ func init() { storeIDExample := imports.ID(50) textSelExample := textselector.Expression("Links/21/Hash/Links/42/Hash") + apiSelExample := api.Selector("Links/21/Hash/Links/42/Hash") clientEvent := retrievalmarket.ClientEventDealAccepted addExample(bitfield.NewFromSet([]uint64{5})) @@ -128,6 +129,7 @@ func init() { addExample(retrievalmarket.ClientEventDealAccepted) addExample(retrievalmarket.DealStatusNew) addExample(&textSelExample) + addExample(&apiSelExample) addExample(network.ReachabilityPublic) addExample(build.NewestNetworkVersion) addExample(map[string]int{"name": 42}) diff --git a/documentation/en/api-v1-unstable-methods.md b/documentation/en/api-v1-unstable-methods.md index 54cfda089..1a14dbd71 100644 --- a/documentation/en/api-v1-unstable-methods.md +++ b/documentation/en/api-v1-unstable-methods.md @@ -1069,7 +1069,7 @@ Inputs: "Root": { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" }, - "DatamodelPathSelector": "Links/21/Hash/Links/42/Hash", + "DAGs": null, "FromLocalCAR": "string value", "DealID": 5 }, @@ -1510,7 +1510,7 @@ Inputs: "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" }, "Piece": null, - "DatamodelPathSelector": "Links/21/Hash/Links/42/Hash", + "DataSelector": "Links/21/Hash/Links/42/Hash", "Size": 42, "Total": "0", "UnsealPrice": "0", From 597b72e28601c5b0b468ffd24a4e9200345a3d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 16 Nov 2021 12:04:03 +0100 Subject: [PATCH 15/33] retrieval: Fix lint, cli docsgen --- cli/client_retr.go | 9 +++++---- documentation/en/cli-lotus.md | 35 ++++++++++++++++++++++++++++++++--- itests/kit/deals.go | 1 + node/impl/client/client.go | 8 ++++++-- 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/cli/client_retr.go b/cli/client_retr.go index 3299cafbe..eba741802 100644 --- a/cli/client_retr.go +++ b/cli/client_retr.go @@ -5,8 +5,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr/net" "io" "net/http" "net/url" @@ -16,21 +14,24 @@ import ( "strings" "time" - "github.com/filecoin-project/lotus/node/repo" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" offline "github.com/ipfs/go-ipfs-exchange-offline" "github.com/ipfs/go-merkledag" carv2 "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/blockstore" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" "github.com/urfave/cli/v2" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-state-types/big" + lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/repo" ) const DefaultMaxRetrievePrice = "0.01" @@ -427,7 +428,7 @@ var clientRetrieveLsCmd = &cli.Command{ } if len(roots) != 1 { - return xerrors.Errorf("expected 1 car root, got %d") + return xerrors.Errorf("expected 1 car root, got %d", len(roots)) } dserv := merkledag.NewDAGService(blockservice.New(cbs, offline.Exchange(cbs))) diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index d617ac684..4cfa82b73 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -539,10 +539,12 @@ NAME: lotus client retrieve - Retrieve data from network USAGE: - lotus client retrieve [command options] [dataCid outputPath] + lotus client retrieve command [command options] [dataCid outputPath] -CATEGORY: - RETRIEVAL +COMMANDS: + cat Show data from network + ls Show object links + help, h Shows a list of commands or help for one command OPTIONS: --from value address to send transactions from @@ -553,6 +555,33 @@ OPTIONS: --pieceCid value require data to be retrieved from a specific Piece CID --allow-local (default: false) --help, -h show help (default: false) + --version, -v print the version (default: false) + +``` + +#### lotus client retrieve cat +``` +NAME: + lotus client retrieve cat - Show data from network + +USAGE: + lotus client retrieve cat [command options] [dataCid] + +OPTIONS: + --help, -h show help (default: false) + +``` + +#### lotus client retrieve ls +``` +NAME: + lotus client retrieve ls - Show object links + +USAGE: + lotus client retrieve ls [command options] [dataCid] + +OPTIONS: + --help, -h show help (default: false) ``` diff --git a/itests/kit/deals.go b/itests/kit/deals.go index b8534982a..651c15901 100644 --- a/itests/kit/deals.go +++ b/itests/kit/deals.go @@ -323,6 +323,7 @@ func (dh *DealHarness) PerformRetrieval(ctx context.Context, deal *cid.Cid, root updatesCtx, cancel := context.WithCancel(ctx) updates, err := dh.client.ClientGetRetrievalUpdates(updatesCtx) + require.NoError(dh.t, err) retrievalRes, err := dh.client.ClientRetrieve(ctx, offers[0].Order(caddr)) require.NoError(dh.t, err) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 28da94146..a4d626206 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -1042,6 +1042,8 @@ func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds forma rsn = selspec.Node() } + var newRoot cid.Cid + if err := utils.TraverseDag( ctx, ds, @@ -1059,7 +1061,7 @@ func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds forma // todo: is this a correct assumption // todo: is the n ipld.Node above the node we want as the (sub)root? // todo: how to go from ipld.Node to a cid? - out[i].root = root + newRoot = root return nil } @@ -1068,7 +1070,7 @@ func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds forma return xerrors.Errorf("cidlink cast unexpectedly failed on '%s'", p.LastBlock.Link) } - out[i].root = cidLnk.Cid + newRoot = cidLnk.Cid } return nil }, @@ -1079,6 +1081,8 @@ func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds forma if out[i].root == cid.Undef { return nil, xerrors.Errorf("path selection does not match a node within %s", root) } + + out[i].root = newRoot } if spec.DataSelector != nil { From 53a48df77e3232b5537f0591e646ee998fc43d9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 16 Nov 2021 12:40:25 +0100 Subject: [PATCH 16/33] retrieval: Don't use subcommands --- cli/client.go | 2 ++ cli/client_retr.go | 54 ++++++++++++++++++----------------- documentation/en/cli-lotus.md | 47 +++++++++++++++++++----------- 3 files changed, 61 insertions(+), 42 deletions(-) diff --git a/cli/client.go b/cli/client.go index 9a8f20899..f5b38788b 100644 --- a/cli/client.go +++ b/cli/client.go @@ -93,6 +93,8 @@ var clientCmd = &cli.Command{ WithCategory("data", clientStat), WithCategory("retrieval", clientFindCmd), WithCategory("retrieval", clientRetrieveCmd), + WithCategory("retrieval", clientRetrieveCatCmd), + WithCategory("retrieval", clientRetrieveLsCmd), WithCategory("retrieval", clientCancelRetrievalDealCmd), WithCategory("retrieval", clientListRetrievalsCmd), WithCategory("util", clientCommPCmd), diff --git a/cli/client_retr.go b/cli/client_retr.go index eba741802..fa5dc2d9d 100644 --- a/cli/client_retr.go +++ b/cli/client_retr.go @@ -200,43 +200,43 @@ func retrieve(ctx context.Context, cctx *cli.Context, fapi lapi.FullNode, sel *l return eref, nil } -var clientRetrieveCmd = &cli.Command{ - Name: "retrieve", - Subcommands: []*cli.Command{ - clientRetrieveCatCmd, - clientRetrieveLsCmd, +var retrFlagsCommon = []cli.Flag{ + &cli.StringFlag{ + Name: "from", + Usage: "address to send transactions from", }, + &cli.StringFlag{ + Name: "miner", + Usage: "miner address for retrieval, if not present it'll use local discovery", + }, + &cli.StringFlag{ + Name: "maxPrice", + Usage: fmt.Sprintf("maximum price the client is willing to consider (default: %s FIL)", DefaultMaxRetrievePrice), + }, + &cli.StringFlag{ + Name: "pieceCid", + Usage: "require data to be retrieved from a specific Piece CID", + }, + &cli.BoolFlag{ + Name: "allow-local", + // todo: default to true? + }, +} + +var clientRetrieveCmd = &cli.Command{ + Name: "retrieve", Usage: "Retrieve data from network", ArgsUsage: "[dataCid outputPath]", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "from", - Usage: "address to send transactions from", - }, + Flags: append([]cli.Flag{ &cli.BoolFlag{ Name: "car", Usage: "export to a car file instead of a regular file", }, - &cli.StringFlag{ - Name: "miner", - Usage: "miner address for retrieval, if not present it'll use local discovery", - }, &cli.StringFlag{ Name: "datamodel-path-selector", Usage: "a rudimentary (DM-level-only) text-path selector, allowing for sub-selection within a deal", }, - &cli.StringFlag{ - Name: "maxPrice", - Usage: fmt.Sprintf("maximum price the client is willing to consider (default: %s FIL)", DefaultMaxRetrievePrice), - }, - &cli.StringFlag{ - Name: "pieceCid", - Usage: "require data to be retrieved from a specific Piece CID", - }, - &cli.BoolFlag{ - Name: "allow-local", - }, - }, + }, retrFlagsCommon...), Action: func(cctx *cli.Context) error { if cctx.NArg() != 2 { return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments")) @@ -329,6 +329,7 @@ var clientRetrieveCatCmd = &cli.Command{ Name: "cat", Usage: "Show data from network", ArgsUsage: "[dataCid]", + Flags: retrFlagsCommon, Action: func(cctx *cli.Context) error { if cctx.NArg() != 1 { return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments")) @@ -372,6 +373,7 @@ var clientRetrieveLsCmd = &cli.Command{ Name: "ls", Usage: "Show object links", ArgsUsage: "[dataCid]", + Flags: retrFlagsCommon, Action: func(cctx *cli.Context) error { if cctx.NArg() != 1 { return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments")) diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index 4cfa82b73..d279cd00d 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -426,6 +426,8 @@ COMMANDS: RETRIEVAL: find Find data in the network retrieve Retrieve data from network + cat Show data from network + ls Show object links cancel-retrieval Cancel a retrieval deal by deal ID; this also cancels the associated transfer list-retrievals List retrieval market deals STORAGE: @@ -539,49 +541,62 @@ NAME: lotus client retrieve - Retrieve data from network USAGE: - lotus client retrieve command [command options] [dataCid outputPath] + lotus client retrieve [command options] [dataCid outputPath] -COMMANDS: - cat Show data from network - ls Show object links - help, h Shows a list of commands or help for one command +CATEGORY: + RETRIEVAL OPTIONS: - --from value address to send transactions from --car export to a car file instead of a regular file (default: false) - --miner value miner address for retrieval, if not present it'll use local discovery --datamodel-path-selector value a rudimentary (DM-level-only) text-path selector, allowing for sub-selection within a deal + --from value address to send transactions from + --miner value miner address for retrieval, if not present it'll use local discovery --maxPrice value maximum price the client is willing to consider (default: 0.01 FIL) --pieceCid value require data to be retrieved from a specific Piece CID --allow-local (default: false) --help, -h show help (default: false) - --version, -v print the version (default: false) ``` -#### lotus client retrieve cat +### lotus client cat ``` NAME: - lotus client retrieve cat - Show data from network + lotus client cat - Show data from network USAGE: - lotus client retrieve cat [command options] [dataCid] + lotus client cat [command options] [dataCid] + +CATEGORY: + RETRIEVAL OPTIONS: - --help, -h show help (default: false) + --from value address to send transactions from + --miner value miner address for retrieval, if not present it'll use local discovery + --maxPrice value maximum price the client is willing to consider (default: 0.01 FIL) + --pieceCid value require data to be retrieved from a specific Piece CID + --allow-local (default: false) + --help, -h show help (default: false) ``` -#### lotus client retrieve ls +### lotus client ls ``` NAME: - lotus client retrieve ls - Show object links + lotus client ls - Show object links USAGE: - lotus client retrieve ls [command options] [dataCid] + lotus client ls [command options] [dataCid] + +CATEGORY: + RETRIEVAL OPTIONS: - --help, -h show help (default: false) + --from value address to send transactions from + --miner value miner address for retrieval, if not present it'll use local discovery + --maxPrice value maximum price the client is willing to consider (default: 0.01 FIL) + --pieceCid value require data to be retrieved from a specific Piece CID + --allow-local (default: false) + --help, -h show help (default: false) ``` From 74f645a098989275c5b4b0e16d7799f4351a8e8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 16 Nov 2021 12:58:52 +0100 Subject: [PATCH 17/33] retrieval: Fix parseDagSpec root check --- node/impl/client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index a4d626206..8ac067779 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -1078,7 +1078,7 @@ func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds forma return nil, xerrors.Errorf("error while locating partial retrieval sub-root: %w", err) } - if out[i].root == cid.Undef { + if newRoot == cid.Undef { return nil, xerrors.Errorf("path selection does not match a node within %s", root) } From 763659b8a3559b44699589b792aa350863b3f9f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 16 Nov 2021 14:30:14 +0100 Subject: [PATCH 18/33] retrieval: Update lotus-soup --- api/v0api/full.go | 16 ++++++++++++++++ testplans/lotus-soup/testkit/retrieval.go | 3 ++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/api/v0api/full.go b/api/v0api/full.go index 20e5a7179..b37e89155 100644 --- a/api/v0api/full.go +++ b/api/v0api/full.go @@ -716,6 +716,22 @@ type FullNode interface { CreateBackup(ctx context.Context, fpath string) error //perm:admin } +func OfferOrder(o api.QueryOffer, client address.Address) RetrievalOrder { + return RetrievalOrder{ + Root: o.Root, + Piece: o.Piece, + Size: o.Size, + Total: o.MinPrice, + UnsealPrice: o.UnsealPrice, + PaymentInterval: o.PaymentInterval, + PaymentIntervalIncrease: o.PaymentIntervalIncrease, + Client: client, + + Miner: o.Miner, + MinerPeer: &o.MinerPeer, + } +} + type RetrievalOrder struct { // TODO: make this less unixfs specific Root cid.Cid diff --git a/testplans/lotus-soup/testkit/retrieval.go b/testplans/lotus-soup/testkit/retrieval.go index de3dee6be..3d6683d00 100644 --- a/testplans/lotus-soup/testkit/retrieval.go +++ b/testplans/lotus-soup/testkit/retrieval.go @@ -11,6 +11,7 @@ import ( "time" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/v0api" "github.com/ipfs/go-cid" files "github.com/ipfs/go-ipfs-files" ipld "github.com/ipfs/go-ipld-format" @@ -51,7 +52,7 @@ func RetrieveData(t *TestEnvironment, ctx context.Context, client api.FullNode, IsCAR: carExport, } t1 = time.Now() - err = client.ClientRetrieve(ctx, offers[0].Order(caddr), ref) + err = (&v0api.WrapperV1Full{FullNode: client}).ClientRetrieve(ctx, v0api.OfferOrder(offers[0], caddr), ref) if err != nil { return err } From 46ee3a0c464ac195957d06591e9cfd1bcb2be5c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 16 Nov 2021 15:06:05 +0100 Subject: [PATCH 19/33] retrieval: Don't default to non-zero cost retrieval --- cli/client_retr.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/client_retr.go b/cli/client_retr.go index fa5dc2d9d..a9df144d3 100644 --- a/cli/client_retr.go +++ b/cli/client_retr.go @@ -34,7 +34,7 @@ import ( "github.com/filecoin-project/lotus/node/repo" ) -const DefaultMaxRetrievePrice = "0.01" +const DefaultMaxRetrievePrice = "0" func retrieve(ctx context.Context, cctx *cli.Context, fapi lapi.FullNode, sel *lapi.Selector, printf func(string, ...interface{})) (*lapi.ExportRef, error) { var payer address.Address From c6101aa02f2286e3fff0bf8c0bab88fe9d9c2468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 17 Nov 2021 17:39:27 +0100 Subject: [PATCH 20/33] retrieval: Support listing ipld links in ls --- cli/client_retr.go | 44 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/cli/client_retr.go b/cli/client_retr.go index a9df144d3..2f7f7972a 100644 --- a/cli/client_retr.go +++ b/cli/client_retr.go @@ -5,6 +5,10 @@ import ( "context" "encoding/json" "fmt" + "github.com/filecoin-project/lotus/markets/utils" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/traversal" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "io" "net/http" "net/url" @@ -373,7 +377,12 @@ var clientRetrieveLsCmd = &cli.Command{ Name: "ls", Usage: "Show object links", ArgsUsage: "[dataCid]", - Flags: retrFlagsCommon, + Flags: append([]cli.Flag{ + &cli.BoolFlag{ + Name: "ipld", + Usage: "list IPLD-level links", + }, + }, retrFlagsCommon...), Action: func(cctx *cli.Context) error { if cctx.NArg() != 1 { return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments")) @@ -432,16 +441,35 @@ var clientRetrieveLsCmd = &cli.Command{ if len(roots) != 1 { return xerrors.Errorf("expected 1 car root, got %d", len(roots)) } - dserv := merkledag.NewDAGService(blockservice.New(cbs, offline.Exchange(cbs))) - links, err := dserv.GetLinks(ctx, roots[0]) - if err != nil { - return xerrors.Errorf("getting links: %w", err) - } + if !cctx.Bool("ipld") { - for _, link := range links { - fmt.Printf("%s %s\t%d\n", link.Cid, link.Name, link.Size) + links, err := dserv.GetLinks(ctx, roots[0]) + if err != nil { + return xerrors.Errorf("getting links: %w", err) + } + + for _, link := range links { + fmt.Printf("%s %s\t%d\n", link.Cid, link.Name, link.Size) + } + } else { + sel, _ := selectorparse.ParseJSONSelector(`{"a":{">":{".":{}}}}`) + + if err := utils.TraverseDag( + ctx, + dserv, + roots[0], + sel, + func(p traversal.Progress, n ipld.Node, r traversal.VisitReason) error { + if r == traversal.VisitReason_SelectionMatch { + fmt.Println(p.Path) + } + return nil + }, + ); err != nil { + return err + } } return err From dd78e75dd69a6113183d20d0cd3f9941eeb4fd10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 17 Nov 2021 19:17:59 +0100 Subject: [PATCH 21/33] retrieval: add depth parameter to ls --- cli/client_retr.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/cli/client_retr.go b/cli/client_retr.go index 2f7f7972a..d70b5ee1b 100644 --- a/cli/client_retr.go +++ b/cli/client_retr.go @@ -380,7 +380,16 @@ var clientRetrieveLsCmd = &cli.Command{ Flags: append([]cli.Flag{ &cli.BoolFlag{ Name: "ipld", - Usage: "list IPLD-level links", + Usage: "list IPLD datamodel links", + }, + &cli.IntFlag{ + Name: "depth", + Usage: "list links recursively up to the specified depth", + Value: 1, + }, + &cli.StringFlag{ + Name: "path-selector", + Usage: "a rudimentary (DM-level-only) text-path selector", }, }, retrFlagsCommon...), Action: func(cctx *cli.Context) error { @@ -402,7 +411,7 @@ var clientRetrieveLsCmd = &cli.Command{ afmt := NewAppFmt(cctx.App) rootSelector := lapi.Selector(`{".": {}}`) - dataSelector := lapi.Selector(`{"R":{"l":{"depth":1},":>":{"a":{">":{"@":{}}}}}}`) + dataSelector := lapi.Selector(fmt.Sprintf(`{"a":{">":{"R":{"l":{"depth":%d},":>":{"a":{">":{"|": [{"@":{}}, {".": {}}]}}}}}}}`, cctx.Int("depth"))) eref, err := retrieve(ctx, cctx, fapi, &dataSelector, afmt.Printf) if err != nil { @@ -411,7 +420,7 @@ var clientRetrieveLsCmd = &cli.Command{ eref.DAGs = append(eref.DAGs, lapi.DagSpec{ RootSelector: &rootSelector, - DataSelector: &rootSelector, + DataSelector: &dataSelector, }) rc, err := ClientExportStream(ainfo.Addr, ainfo.AuthHeader(), *eref, true) @@ -454,7 +463,7 @@ var clientRetrieveLsCmd = &cli.Command{ fmt.Printf("%s %s\t%d\n", link.Cid, link.Name, link.Size) } } else { - sel, _ := selectorparse.ParseJSONSelector(`{"a":{">":{".":{}}}}`) + sel, _ := selectorparse.ParseJSONSelector(fmt.Sprintf(`{"R":{"l":{"depth":%d},":>":{"a":{">":{"|":[{"@":{}},{".":{}}]}}}}}`, cctx.Int("depth"))) if err := utils.TraverseDag( ctx, From 2c583b03ffc83a72bfebd6b0dc24c13373ba0ede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 17 Nov 2021 19:52:31 +0100 Subject: [PATCH 22/33] retrieval: Support DM-paths in ls --- cli/client_retr.go | 68 +++++++++++++++++++++++++++++++++----- node/impl/client/client.go | 4 --- 2 files changed, 59 insertions(+), 13 deletions(-) diff --git a/cli/client_retr.go b/cli/client_retr.go index d70b5ee1b..5b18d09b7 100644 --- a/cli/client_retr.go +++ b/cli/client_retr.go @@ -5,11 +5,8 @@ import ( "context" "encoding/json" "fmt" - "github.com/filecoin-project/lotus/markets/utils" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/traversal" - selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "io" + "io/ioutil" "net/http" "net/url" "os" @@ -24,6 +21,14 @@ import ( "github.com/ipfs/go-merkledag" carv2 "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/blockstore" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/codec/dagjson" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/ipld/go-ipld-prime/traversal" + "github.com/ipld/go-ipld-prime/traversal/selector" + "github.com/ipld/go-ipld-prime/traversal/selector/builder" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" + textselector "github.com/ipld/go-ipld-selector-text-lite" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" "github.com/urfave/cli/v2" @@ -35,6 +40,7 @@ import ( lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/node/repo" ) @@ -322,8 +328,13 @@ func ClientExportStream(apiAddr string, apiAuth http.Header, eref lapi.ExportRef } if resp.StatusCode != http.StatusOK { + em, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, xerrors.Errorf("reading error body: %w", err) + } + resp.Body.Close() // nolint - return nil, xerrors.Errorf("getting root car: http %d", resp.StatusCode) + return nil, xerrors.Errorf("getting root car: http %d: %s", resp.StatusCode, string(em)) } return resp.Body, nil @@ -373,6 +384,20 @@ var clientRetrieveCatCmd = &cli.Command{ }, } +func pathToSel(psel string, sub builder.SelectorSpec) (lapi.Selector, error) { + rs, err := textselector.SelectorSpecFromPath(textselector.Expression(psel), sub) + if err != nil { + return "", xerrors.Errorf("failed to parse path-selector '%s': %w", err) + } + + var b bytes.Buffer + if err := dagjson.Encode(rs.Node(), &b); err != nil { + return "", err + } + + return lapi.Selector(b.String()), nil +} + var clientRetrieveLsCmd = &cli.Command{ Name: "ls", Usage: "Show object links", @@ -388,7 +413,7 @@ var clientRetrieveLsCmd = &cli.Command{ Value: 1, }, &cli.StringFlag{ - Name: "path-selector", + Name: "datamodel-path", Usage: "a rudimentary (DM-level-only) text-path selector", }, }, retrFlagsCommon...), @@ -411,13 +436,30 @@ var clientRetrieveLsCmd = &cli.Command{ afmt := NewAppFmt(cctx.App) rootSelector := lapi.Selector(`{".": {}}`) - dataSelector := lapi.Selector(fmt.Sprintf(`{"a":{">":{"R":{"l":{"depth":%d},":>":{"a":{">":{"|": [{"@":{}}, {".": {}}]}}}}}}}`, cctx.Int("depth"))) + dataSelector := lapi.Selector(fmt.Sprintf(`{"a":{">":{"R":{"l":{"depth":%d},":>":{"a":{">":{"|":[{"@":{}},{".":{}}]}}}}}}}`, cctx.Int("depth"))) + + if cctx.IsSet("datamodel-path") { + rootSelector, err = pathToSel(cctx.String("datamodel-path"), nil) + if err != nil { + return err + } + + ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) + dataSelector, err = pathToSel(cctx.String("datamodel-path"), ssb.ExploreAll( + ssb.ExploreRecursive(selector.RecursionLimitDepth(int64(cctx.Int("depth"))), ssb.ExploreAll(ssb.ExploreUnion(ssb.Matcher(), ssb.ExploreRecursiveEdge()))), + )) + if err != nil { + return err + } + } eref, err := retrieve(ctx, cctx, fapi, &dataSelector, afmt.Printf) if err != nil { return err } + fmt.Println() // separate retrieval events from results + eref.DAGs = append(eref.DAGs, lapi.DagSpec{ RootSelector: &rootSelector, DataSelector: &dataSelector, @@ -453,7 +495,6 @@ var clientRetrieveLsCmd = &cli.Command{ dserv := merkledag.NewDAGService(blockservice.New(cbs, offline.Exchange(cbs))) if !cctx.Bool("ipld") { - links, err := dserv.GetLinks(ctx, roots[0]) if err != nil { return xerrors.Errorf("getting links: %w", err) @@ -463,7 +504,16 @@ var clientRetrieveLsCmd = &cli.Command{ fmt.Printf("%s %s\t%d\n", link.Cid, link.Name, link.Size) } } else { - sel, _ := selectorparse.ParseJSONSelector(fmt.Sprintf(`{"R":{"l":{"depth":%d},":>":{"a":{">":{"|":[{"@":{}},{".":{}}]}}}}}`, cctx.Int("depth"))) + jsel := lapi.Selector(fmt.Sprintf(`{"R":{"l":{"depth":%d},":>":{"a":{">":{"|":[{"@":{}},{".":{}}]}}}}}`, cctx.Int("depth"))) + + if cctx.IsSet("datamodel-path") { + ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) + jsel, err = pathToSel(cctx.String("datamodel-path"), + ssb.ExploreRecursive(selector.RecursionLimitDepth(int64(cctx.Int("depth"))), ssb.ExploreAll(ssb.ExploreUnion(ssb.Matcher(), ssb.ExploreRecursiveEdge()))), + ) + } + + sel, _ := selectorparse.ParseJSONSelector(string(jsel)) if err := utils.TraverseDag( ctx, diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 8ac067779..abfd835cf 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -1052,10 +1052,6 @@ func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds forma func(p traversal.Progress, n ipld.Node, r traversal.VisitReason) error { if r == traversal.VisitReason_SelectionMatch { - if p.LastBlock.Path.String() != p.Path.String() { - return xerrors.Errorf("unsupported selection path '%s' does not correspond to a block boundary (a.k.a. CID link)", p.Path.String()) - } - if p.LastBlock.Link == nil { // this is likely the root node that we've matched here // todo: is this a correct assumption From 3e70b8420ea9ba5236e403f6ee4d4178bb40a80d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 17 Nov 2021 20:24:12 +0100 Subject: [PATCH 23/33] retrieval: Make the cat command work --- cli/client_retr.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/cli/client_retr.go b/cli/client_retr.go index 5b18d09b7..b8119d816 100644 --- a/cli/client_retr.go +++ b/cli/client_retr.go @@ -344,7 +344,16 @@ var clientRetrieveCatCmd = &cli.Command{ Name: "cat", Usage: "Show data from network", ArgsUsage: "[dataCid]", - Flags: retrFlagsCommon, + Flags: append([]cli.Flag{ + &cli.BoolFlag{ + Name: "ipld", + Usage: "list IPLD datamodel links", + }, + &cli.StringFlag{ + Name: "datamodel-path", + Usage: "a rudimentary (DM-level-only) text-path selector", + }, + }, retrFlagsCommon...), Action: func(cctx *cli.Context) error { if cctx.NArg() != 1 { return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments")) @@ -363,13 +372,20 @@ var clientRetrieveCatCmd = &cli.Command{ ctx := ReqContext(cctx) afmt := NewAppFmt(cctx.App) - // todo selector - eref, err := retrieve(ctx, cctx, fapi, nil, afmt.Printf) + sel := lapi.Selector(cctx.String("datamodel-path")) + selp := &sel + if sel == "" { + selp = nil + } + + eref, err := retrieve(ctx, cctx, fapi, selp, afmt.Printf) if err != nil { return err } - if sel := lapi.Selector(cctx.String("datamodel-path-selector")); sel != "" { + fmt.Println() // separate retrieval events from results + + if sel != "" { eref.DAGs = append(eref.DAGs, lapi.DagSpec{DataSelector: &sel}) } From f88c514be95aa10dea6b4ee5dd298b814f75f60a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 22 Nov 2021 12:52:11 +0100 Subject: [PATCH 24/33] make lint happy --- build/openrpc/full.json.gz | Bin 25494 -> 25677 bytes build/openrpc/miner.json.gz | Bin 10466 -> 10466 bytes build/openrpc/worker.json.gz | Bin 2712 -> 2710 bytes cli/client_retr.go | 2 +- 4 files changed, 1 insertion(+), 1 deletion(-) diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index 14fbc4354e44049bc00722763001c21120a4baac..3477fdc45dbf338ee327f4d3c43adb5d2a97299a 100644 GIT binary patch delta 24449 zcmb4~W0NjT7p2>_ZTD&0K5g5!?Y`T#ZQHhO+cr;oo_FRKOyq~ms1H?<89OR=u63={ zG2oeT;CM1X)E#F;-A%@}^WioR60{_iGEf*0owd^Y?eE~AFS3MU>mTFGmmq%5tAz@p|C{6@IF%B^tTT_OeV^H zes%co_p>}>FWJAuj{PhUdfkY8cPy?8Xm3WE#yd)#WjuK|6a`$+cq z@8oCWVL)(rAq8qD*v3fvTjij|fdlkn|K8UDfCKqd?=vC8 z!+?!llNGP#5duv*Ewy(dGUvdgL9fPa%i z@6vh9fBH)Nax}qzam7bblAbd?KXp+J3&+6)q!+IUZVnojq9{#{shQfZr>9J|R>WU9OdR z9>nrEkXSKaNvFQEGJ@~Zz5i>>}0&{a8rCVKQA;D3A$5Rtw zg7PE0i{kg^*_?^r#XZBZ!_YgOCj~zscaxiPIYDH6_6BmA@&F0ULW&E7fP$mPj{&ZG zSIB;_V-CUZ+x1VB4VTM@bu*XV2Qa$J_a(I4>(zYGkbYUbBia{??^hM!eAoML4Dtrf z((j#(xpzJ3sLHXSQx8Ez&LAiJe1$=hEVia3F8TH~mj`>u5625#uknEF-+3W`fZu9E zU1I5!=Q?bl8TyTY{owZGn9}UdWO$SYUXe0bXuKgL)B>tZLAnf380&H5@5l?zLtf*z zHvjdYdG#N&#<+v|3;gSAgkiH6V$3Gj+>KOihT-3|JL4A3XN09K%Okh-UAq|so?N0c z3Qd}uR*l+(MLvo)r4zZy7KL6w3XWYWi7Hzcxx9ZINCOv^KNxS2ja;^*AT<1jGo#S_ z5$b+8jz0?dwis?7GlHiZXhgx^=fDGxa?LHuaS=RU?4K1o)v=W-r23$p8slqvtu05Moa##etLn->_O%Au zUzGc0F(N@yG@?T?TBU-mqot%-q_W)15e4O0%7gu7vEU#PCHxJY^MN?5i>Q=(0#-br59kZv1Ks?5B4kra}Y+y291Nl zILmX#>+m)P-}7xEj$1Lnb?zAu3Z~$_WOAk;`^F^60b!3m`8Mv1d)Tia~PSQQgW+e_)YuKWEEDKb& z?gTbJ5xu7`&nVK0-Z-XFibnEJcz#r1NIJ7sH);&0{4Gc*@>>pIA?yV8lvLcNoMg^| zqUFz$hp0?ZyASQloeVPS_75~F+B1Sz(M`_IfL`5R*WfICV$ZMR!w+x1i%MnEoA({> zarmxl24PQKI^I(J{qDflPvK-lfglY~;KGX-=-G4PmDlA47u@jV#Lrjcy~m(8WY3LW z-<#rZyjfiNaEU*_A2WF}z~zCjC{%!WbRSEX2+}aG={9o>-hKUWVdLb$123QHSfbgB ze9btgK+NZx+6Yh1*EBKV>H;NBwyLizfJ!c(l8&wrn$1=`zTgdNLVk5VH;Hi6HzRP_ z16L!^Xr`sU8}Q61uI!e#%)wz&GQNgrfYjfCCufDt8+rj?s4s;OrHrm3KY+%y5%;k} zq_ZP444^Pw@+nn|$Al;q8wuR_6HPe=w5iHq*fkT*fDCJk^ABgvB}>snbKpo+YQ~Ii zP7=k%5un{@lT>n?JyAMAlH_*HyVQS3Z@N&#rJllr_`2Zm#h9V8 zX5xL0=Vhd4N8|)Y^@ysCh!S7?7)@%32H{p6@>d3cVUf_o03A<|3P%LB_OC&cL^eY| z`-*D(W-1&~yZQu!Vzfc$PV}an@(=ld@N$bpgVJ~GKY1`Bki&IRArUGn#NNiEMw(Ka z-%qV{fjYcAJfrw{xxYM2LAQVXJ9%qDy+3){=o?tzx&J)rLcRZ)0{?e^dwKc%1bKh4 zb8-SW2y;){e`etXOFO%ZmEX%F2r6n~80(9`kni=M)2|ink!e!b7lpmp95e6B>*i6M z(aFix@9pZO=wm^%4~e!h^UVi~#!F$}1`UG%c1omnRD3=;zOSAz5|PDt>2AH>99D5W zZUF11aqcH~`pWtmGQPg-%d*P6K-h$Se**^Ou#1Ng5_!kXGd*jo@(n55K>Y{^x)J14 zBWNSNOL}oT zQ1rmENdr(c5YJ?-kUFSI4^Sf7&&3|Aym%kt*04>n&^2dd4g0NWxZ#92}{<4L}FPvm28>gR;OzKjN*O%bCk(=w49bP&-<~B2+4KoZ}ZXBT!n085kol_5&<=Cit zIc(DxnWxzRp3BNq=d{-bFhcp5x#YMpm%l!|;RMkZ@r*AhcC{qM#~oblnfb-=H=}2U z7?w+cip+x0UK1D6Qt^*QNr8r$=+ww($p@;OHAr2Kz9)*55uBs7DKxl$RHfQvBcdy4 z+O#5`ZV{Xxn4)Ee`dCaX_Il!RjWzFXX3!}{d;sRDZT%~ z^SyZkhE<^Vle~Zvo7a?V#+f3RpfWL`;rJs)g7uMeTlvuu!*2L z#?hm_quY5ux+wzpDE*e=75IgO2&o`VJKhmY-qyg%e>r1*!G67EkAZ1`zU~!xfA-Fq z1h|s{A&7Cc(Es+*81#ER-X8(3V&4pX?elbil7h2KLw|uj>c_l?qMg@Z;n6a)^-dgftVlBqWkYe32k}55o$ZWBt);s*0}8Po>+-z3Ucd zLwsCotXkC-lTZhN`US&DXGMc6OVxQJd13UJZhqNyj(M0F)C0@_#U3S}XF5?t#9RDy z0`DS1Ogmlk+kpueg|DO>`2~qAE;|8my{tR;8J@qD7(;aK<_M|Uyo4{aq_Xk=zo5U_ zj_J8CKMZ@KRmvIK4_B{wNoojur zp%w8`J!AHZ@bHePqQxcbUMb6-uW?OuASti`294y#U+5PqntwX=C!RIJu8!}FAJ*P? zFV+3PqdJ<|SZWEIf6miV3g7wW<~Wo_@U#{zpk)9;TSUtAk75|=6YU8(7v|6lqIZ#= zIS> zBIJ3Ed_B@gL2CD4?W7PHHkU{{W`S9zuOByIiI=QFU-Qdg8;Z`nL8IXCv@3kmI|R5~ zC$Pu_FLt$U-9r@DnjhlTaCEfF8{xoX(dKNBHw9_7rdTMYrrw-=>Xr?CR~jG*x6wz7_>nc5eBorMI(a z>XZ>l2jdFP@Rlx~ySeVhSM~9zgfO}^47_I9H)DW11ufvM$6xLQ=b-tZfx!e65K{nx zgSf+C7tu`B;@KhR+US6oqY4hQj?S*>vJ#~J+1Fb=k>mw>z?3pLo_*Dq)XOaq!lN8f zm^v_-jY{CWtfzQy0S(2WFhU1?6FNFSUjlbWoI*YMFM&UCiC#`XP2pBajP1%$go?9} z9^Vu-(#g2E@Eg()^V^*L@B16w$3KDX-+I7!;NI`m_`BEJ!}<5gt@vxKUwbDkf6mU&tANk) zu32t8pGUs3leIDL7PgK0yJ2bTIrt&)-?n)kp?XdpSuFk15}H{|(U4Ncb|D|>WIjHo zW5*1FP7^23MN9x_B?%XkH>rOj6S^P;!5;^)2x60Q@MIvi^VxOH_w3r}tRAbWnd{vD z-hHj|zBLg-&cDO;7lAvKuzto&hNEG}WO}&zg>>?QnFj{-=j^rfdqRt0Y02qWig(tB*;YmRGGc^1ZwP$C0GX zIDZ3)!2r<2R;qDy8p-T|{3g4MN&JwqH>Ka{G-||g&nGb?=h2o0HcVrSFWJT1KWy`b zr+gmYi(K>n@aHeFFkK%FMPVshL;?EB*w%v7;oIsj3v^F;C%ogs@Z^FP_Zf_stzgoR zxuQo+2Xz}+RD|S}r&lv!SFnj*2w2TFeU!cq%K$P6%FlcESy!14U}CRA88o6Us^2Bm zeMZi`eflT%e>!%&Wppy$>+sLwDzpCm>6O?E|96K3{=`s(41;mBU!Js?g)A9FSMOkj zML|MBLG?FMCfB*1s^ygq1c{Ss9|@Hj=_8{G3AzT3w+IYapD>xXjUXI1puoE0E{gKU z(gl#;hQN?R_E~X!DS3JldDH-LB6y5SaR%(BWm5bR3=N~D#v=})+62m}5>@OOhC`aN znWd#sIhnt~l`W5akm$TuS?@>DF~*!C7lBGNo>?9>6Jt~5`f&c{M+p~Ew`e3}>t{1G za8-GQt>gp{JjQBuMO93x%8!LuLAey|H3Pu9=&TrUaNEgbL;Ek_#~}*^oL-EDVM;C0 zb!JD7?Eoj075Xe2`L{UKdOKClXVLt+_D~lWd?rU*SVJJU8N^3((sXJ7OSNnjkXof~ zikSmh;w&y8A6o)Jrb8i7gF0P#>nOIPV+zexvf&o2CVQQ$-DW)FlBy9Z0w{L zb%zfr(c3lXbYf#~f7hM-+qJpN4~YN$l$WPCNU0~)3SEFBlqVvV&^DHbKE;yivg1LV z@Vf(H%0qcE)n+HQ!2VLXB6bs!ql0rM`xO!~ zCke~Y&Pl;Ngv}GZG~TL+C>4A}Z9t=8*Er%0$zXdfDl3k4g8$3V;h|Ni11Mp)NqQRb z=&M_AC`O}e`fJ=8~rk0k}+)al)hmGGxaUC5iz4f)@zJ+4rwEv1NH`TW-^+`vI>}Z>G zlWz7;b`rAmBHf6F&<&#OZ{}Xv87jmi6B!Vb?V9#q|a=m|z96 z=nJzu7n<)RXe~BtI-=JqrBVG72gYTqjY6jjW$4S`_Mi*djg+Am86qEKpL)Htz27x) z)tkpPC;t_1mNN^4q~Cw{ob6|P(2tf12^O zmv3g@q-Ito?T)T*+&u9q*{KBTb3A3=_IBNFWDn!wCFg4X6I3d9g{H4%ymdonW>GsV z>PEgPL#H;tCa$_8w5J{}A~d!a(HFC_DJ zHJ$XQMj@-#JAu-j76*hn?1kKZl#`UtC2>Zl=)QMsWKanXYk}Da59Oyr0_2~mf z5dU{w%>oE;rSRaM1;Z_c>8b5GVKC@xOr+Hr!vP!OrSx*jwH;M~CH%HS3z*mBXpn+yyv7J12=RTHO4dYuLXrVa?KXjud{W~+@xOb2*P7Erc3IO7nM-I|eRE)(~+ zwEz@hP0I5!fh5e37}umy_A*}Mwig`(7b`4GH?vg2+bDo{oqE>%M@kC2(IG7}(J18& zf^iIKlC}9AB0)AeBL8;uq58z5?euNe^rDk}p5k|!gIg3|Xn3=VZMJ8OxBk1(!j4)3 zjY*iFb*l6sY|I#m;<|1hm_{vY$OwOpppj{$954ydON;{l1l0>i{4` z<-6Vxxm}dRR=#M8da;qu2s@J>3UFJ<3ZO1B<*8nvcGL1tfdibW4qFi2h7+)22@ z{wZG&U|878vy8I&QDpfWS`Jt8myP*|t$G0hym=VUYc)Drb^?>J5y!0WXDL;erzI_t zVLo}cu*43yG^;$(@Iz@QRfu`k#Tl?8nlppkeokNz>M!^+-v+!w#f{1{QCv-*b=oRh z)-k-hQ6uZj#e-7Af;Er3y z4o-UBaVbV^e*re3rc*z2EHYZ&q3+t)@QJ^A#xAmB7ty+dOTvA5n$I+UsdZu zm&BA&n9S`o<-#~xa7N5=6Cop+tv;mwg~&O%a+O(r~Ml zOB$(4HS4C=QwjfwN;g`E)Yc|NA**70;W2Yb3ON!N%8bn#}6&D+y|bGOeM;4Dm?5SXNm zGhZ~9V0#c~s2~hTM8^RCZ<$1?;rZ3Nc)UL9LYB--yaHDi5eVM7XW`g1CsPud1!5Gj zNu`}R&_wAhz(1#_PuCIp8`R_uYfO23!8aq@XkwVSlBAF-i zatrgB7XBN0Iyb=batswGFKMgLDE!N|Hn(KU-s!K$K0ZR9CAJvEH!1KIee?v?r+TF-%Y5Vd@D;){!%& z8F&JKXv%I|LyZ6YBinuRt3x1*_o$Ehy!ZQ|Ubja()5!P#qx|V9#}Tt#mTJ(m-D0S| zd}ZvfJO{J8@h>{tFUI|>o7M7O^8EZ?cM1j92qfs8=TJf20buojjJ>%hJ+dKLLD&rL zqIOjWerOh&DP-N3$nAc(QcC8t6#(n9kj6Qd8M7`WZixB;4zZ889t>$>@9^Zs3nl5l z4bY3b9pC*|rydl*lkW}s!?E$=-T-jtfVRt-pu7D3^0S|Xu>@9TN_db4{c7qd^dPi` zQF5cXMJYSBNV3Go1AK=ZGD|fIl9@=^V}t`LL+Mt+I6F!9Iw{KVqKRIy`it7*z51yI z-N&Il!-Smvq&8@b8fCbSGGkz-9|c0a@gcCbm);w+9d^2@^@_ona{1?LSIiCRVYrKG zu@ts>@2s<-c&XWJjl=QU?jad70b?R(KzFCh5~X1cA=U>00kY=jod=Mpr zQL_V~yTuFw$58hVaE}k5OX%}S{UeS-0{4Jr`twYv?@{b%kzlUo-BvHc1FIwXgBALA8_`)AfT z5uL&Layl`UfY8lxx4c5?D)uR7YS#fGGPnl~y3|qM=x&6&OAX)k%ON#FJ#$~qISlx> zb4rglf{L3}2PSd0JL>Q>KGO5?BAU4gx}vt|UFMG$qi5?%tZP*;+z&GQ9D>eUd7UA5flo{fc_CF!2pW$P8r=n$n44^W=$}` zu%%n0vYIBkvj*+l)zN1+j+?*xE2mY}nrvd>wp;nHAM1sNI&Pbvug}lOE zcI@g@0F5sDH6E`%lIJL`49szn_yRYf+(=p001r)0bx-M{^@&DzHmQkPcXt|M5(NR0 za}4{H1KwQB_v3G~Fm+~!GmW4b%RB4ro9n&`C`DzR+8QpQ#bFU@Z%Z7t!rK!F%gTGx z^TlbBL;OW!Aw4=kp^s5cMayRH$}79-mbW%CKw7k+x-MOQ8)#NbDyaNJL^81aH$@^~ zd0&QHP-af%cR)~rmnN+-6;$z(&KgL^+NwsPp2Sc@Z1CzP(gB{iIHGr%AzjV6Al^EE zkxQnxM_73;2#=g!|M7v6z1H-rgP;51jfS{;b9XJpT+IwcricGgF=^J2$j#W?x`(z6 z5bk#e!^1%XF++%XHwP*uZW{K@+#BchUi5~ZWQ>DyB2%BgiX4bQ{z#QBmQddOB=W_B z3q_OiXlLKXz)hMSJ$FIA_C3yvJo5$KsPc7g7+V^ZM3)?vXyxkXCnRFzQm&5?BPrl7 z1a#(DOx|StdtAhZW%($n9m}psbOVtM_!IAzNAp-~sW!!(qEVI@3GNv&21bMJ1mGo& zU0Y2zK&4n@XVfAa&s38+p8j?$Z1l89S8Sp3+iWP70fpd^M&rwLUUqY*(*wx~O_%~1 zqNy8O_C`f}K^9W4SxB57E-fDsi6uJh=JW6r_wYw;end~g!;-v-e4eX=ilfv4bgB3) z3$TtWlAFvg#TNNZhPD883nC2mZr1Fu=A};WG<&$Xd2$j!TKt)R89^j65e!vRvGf|6h*_j3skQf@H5pj(CCn}znAel^C zZK*OKJHQ*WVS+nkLG+HY?BM(ru!O$%XbF^Eg<8zC$GGRcIO#N8lhwOZ5sgWS{sv9t z{rFEI=mHH@0gai>3G=w}0AWf{o2{HW%d4NCEPBNx5G}U2xFsUt=^rr%%&DWI5nZUs zP(eBm1Cu%L&tIt6VV`J3V=EAswB?>^I-g$E{Rs?T9<=403G1cF+8H5NuYJ0UncJPc zkim&=_5R*9A2w%gmlWtsj_5;8LV&kZAmUe_94?+LKgJ8{8K`QjgS3bEv=1ix+ZG|r zfeB`DV^z0%t6(MPb#}l6An-cpgI{*gKH=J61aC4?qHziw&g-NGnX8RFeHVhU-dB`y z@lwpnpYceoe&}EtMITAnBC+)qEkFF!tr35{`Rkoy9A?KL;p&{9;glDmH3jIEfEWi zdsQ3vFgsILZ6DxWxaiu$TE7skXv!ebiCvls%V<~?SsZJ$HK|&zgkhzK^(d03$vbNm zXxF75M7*iOrukn)JA#Du#In|R%=x#uk=+%O3)uqJOZ2s`#qzA4EPySAo{G(0w zdwsrZai+9sc#rXP$uAiM)+0WePcx=|e!;NAzNx%~;6}(S`wL8N`H*-InZQy&%Z&b` zlDJ*44r((`HIq@ekeDuLUY}z(= zfkbTYlCXq;hEDVY5qggk&(P8u%oS@f-yj;^T4!t>)#L>hgIxTRHSl<(r^cF zIQy5vnfa*dRO6HT(5Ztq)r6H2i+iL$S{XB7b>&**mLu6~C$izV8r=HAnaN^K3U24& zzvk$dL2wSVlWr7;!!lU5mJPK^(hOQH>|y>FQh+91bS5pIdBku+W#+*V1)0B9e6@Ow zhsqs*yc6c66Aa<;_7S_=z0-*54M@qkN_>c1p?w7a!uff}HVF4r+$M}sJ1MVSGwJ1{ zKZUbO1}v-c@UY_zZ$E3l5B>5J87uM+q~!ezavWt(P|l9f#GwuEf3_e6CHQuZgEZl5M)^kWTs-H@FvW+u(8?GG%Y!M8UVj|`jk z>=fHJLlfizSQD@9CK=to^Z1aWKe3I8OoUt@Q=R_AhlK`LnLZf$@@A~kVwgb6oN0yt zs8A?Dxw7>QkUo}T3i_McSm-0D&K*x!KA5FKEhzBKqp_+8srt4pZnA;Q+uNN=C{m{k zYMD(t=c`Fo?<`E=(B^K7!q1MdPx6Uy490-)VyIcNbv@v8s3Z3_^A30`L;R;2{Dn>A znx*59scj+M`p{QLoJ(O=mG<2%?p+2z`1Jhd!tNikpzCk`BgSU4FYH(RyR9DG5r)KZ zu}EFYKDIp+aJHm)p?DOEPxhheqBJJ?6hxB09Z8IBM)5^{@{slu6W0ZIg%7rj{oV+s z7NeBT?0&B9&S$@TBwbAjPjf%dZtlz_96szWcXsWb8sF~sylTW)<89YsR$sA zSR+Ok{dhJgfu_`54nga*nqI3!`;q7)YG~SuHewCvBdh*4@Z-|PrQe-^bM~}xROtqE z4JB82-R0sPk7LF0x3|r%stWqM0c|>htblI~RP^^Mp?$+qVEmMAsRH2s;RoRcWFU&g zGS$uw1is5{jj&d9p5_&;(dR?f%<7z-8X;|SwJTNgf~*j`IQ&lAboHI9&U3D`sh!rY z>JHm9E<-u`NV-m2FHHbor?#P99!kgHk0iHQc++ucMXwqpC!J28alxIphR{YS$DPcq z*3ApIcaPo5+SPT;aZ?V5eWlSpBQL+pj>sP3xh`lj$qh12%^rTv%8dTKh;MY zef>LtT6m~tRg7u)?WiZE0OCdRkddGfP~bkyYjz1#B_M?R6!IK^u@vv=Oq337Iu0II zOUvXE?{S~))PDf4B;zZ}7dIRHSnBZr+>X%J1{uq$z{ndbk{D4|Ubj~%9kIYu020~C zN^LMXx^c+FmXPrMEQ;Aqk%1!Q=?*rNoWVcNpC^lyN91a(bmJo=)%@Us4zV5_nhP5$ zab64@2-i`WGt&cr)6C2EjH#eAGt%3e%7k7Sp)I}Q9jhZiZ~3%F%P#&C@6@rS*rYHe ze<#R*Ki4LfYhzu}keVq+$2M~$EnpP<>sCf{8b6pi7SCB;{)7eMqv}2K&ky=f-T4*D z%bIXBpscvZK8rV3yer)IFH-}FDK7E;O_|NXf-W>qNIV!toRjW+A^*`;RCrg z3rGT&!{F51Z;t4xhew~cZSIHHxr*E;axAfYf}~mjpI?a zgB?;C&;(~QgXy8Jpgy9^_h*OF5akf`vkAG+AMEP9 zrc#I}AEV0H9qTg+YVVND4q9?CKDfij2l-}qI%oU((sE{F)#6Q1v8(=NpAOya@Y3P} zM|cKwkHcgU3dj0!VHd`BpI#JxSYb$Nll73$o^vZ}^A5?w;<6~U5%v$b<(xgg6m6T@ zOVx71Vi~fxfZJBuolz5)Q&$h5){EftONxIdvmnRUq_Q-4mOlj1r)E!3H=fw^rF z=GrB027FKBTzyl|ei=6x;~_9~^k1j&|9IqB zG&k#zSLsblr_6ySY|H<^YR!86xD9FUx(VRq1*FWw_eV$iaS#v=&gu`DTc%Ucb1Nr6qSz7{9W5y&AEo+#e)f7NTZ%xF_FYtqN z%0w02+44%7ar$0wAJU92`B@Bvjq;mL9uwV~4Q_5m^nAGLUlTDr=E>w#ZW+~$MX)Qp zBW+QogScc@6IdT)pQdICI3%nPJ*TCKo2b-NgHS$1Bp6<)QL1)|`8Q;^ey|B#xzsT& zf-WjqZM(YU5(`=kOT+G~SOIJ^mF73^a;0<$m znKc9P)$v3BB1{m8afp>LuN+I-LvV4k7El>08MEg3=MqZ^*mk8-da*p%F#{wxo)L%- zn!%-;GkKaLmV^I)KzbtslH*cbjZ5?5!ZTUFT&C*Qt$6&+DjC3L9u7015(2(Nqw)&JG1aQAhIJ}-K_X#lDJ|8u?GoPsIc(i{@;OEIw0hB$>zD)~ zLu)<6bGny>D}zVX4=#aPCnrM}2B|o<2J!gasqd}JC6sfa7wN-s$hgKwUv?#pRg(fo zE%$Wg1XNeulMzrJVbUvu^%QHAeukGl3Cb{%tNbagvw6W0(4f@k=T&9 zg)tI;{@NJ=AKROX;`mkipHgJ@yi|{|I1jpx5aVedo9O+j;4c+cVD6e!MV;dMAK}g1^ zpt44W2P)8MCH_nF4MhQLgnHRgku64$QvlRqzzLb0Zw z?bbYFX#!{{9!bQI*7~!7A>CUEJp9&-tD;-wt*Z#QwBYUb26hW=~&^E zRkrN^9=4!~qzUgF+)ht%Jj5*XuZy!{XfTtsn(>*Gg{57K)ylg6Caxa3lDQ#;!m(HE z6??}x(N6A*paS$!tak`TNQ3dpSMD*4+DD!7AOPZXvHh?!>%4Ya`m>+{^w)x$FoCSGInGtSRoi7;f_=JPeP?4gx7|r( zrLVaM%;w*~19w(562$oXCH;d^40j7znns#&!!6e`HpTr->A0KE=L}h zYXR{D$Hdlfa7mS_=?Yzg6yR41I!cwNjm}Et3%X};6@kr8_ik)Lz==Y|P$veloc<;V zc-%Q8gGKEsj)&hhu}BL;@BdbOzNTqW9o@JnV{0Ami^>@4V3=HkDbm{y?FV#`U=o!M zF}n)S4(5k!9wTBj?+nb{=jVDjAIqJK^Z?)2Zzkmz#qckWc;M>m=HYolqwcM3Vh1%s zeb+bMGWlWeI_U2k3UDW0_6kjr^MqmqI?}EMKG@Dvv(BJ2Q|7x84#_^#ZrNO2$(b?@ zdR7$2IQn(;O6``72ex8Z8_t~W6k)mI;3MYotZn#cT8zyEypVH+LycAeiWc2Eb^sd= zWHeJS^{FUX)rct*F(180jabE1=cXGQbuc_tT82onFMcvRXHm7q(DzVpN8W&7A_i^1 z1H?4i-SDd#KU^1l318teEqV-!>WEB!cI_6=`9-*ca^74DcKt9(LW?@ye>biHw2m<4 z+8TK7Q|p5P_3(Fc61=HiiF42IT>v1AN@ndlD0CJK922Soszl2+c5^vA^D=$ofVF04 z_9H&GF4eV3Im^Lt0vHOitk#2T(6T$}{JtV?zWG<}60pIc$M14xjDgaH?#M08d9HAtF@^m=33w39=#QoEXA#5GQLX1*{FAB)pnJbuc2%d^m+6N&@D`cH`(GIrH2Az zlgR}U?2qU0s+fdh!me6L63r8#?{~D;-cJVoGlx)0lL!X(v0sTu#92!3)ZerL#)Fy< zs!A5{i1+YXf|!GQJlYf90r0Lh*6~3pp@^4lZ8S@=BMEJVmu;!7HmM_#DBjj2RFGzW z1kPwgD!PxS2J)5q>q6Z0?oWdyiyK`%+Q?b^J%pEBgD>u-gZ1Gk%2F-zVI-n*=>6S@ zT^TpvML^*;AaDnp!2s&W0Z-ps35Ac*z`>-B3Dx~`y4F&5Ytl#7{cG36GOcAKrPvhQGj8H9n-OVI z#SXu}crB{j#R^M3*WGGS!(Yk!Vci+6udb7qn)k+%nKRTO_>1teo{8nsMZ3R(ZPRYaWp(}3=fr~@IM0ba{>2yh<*U7zQ=oj+i; z*j@QAi_^l<-Vx`OLAXyTLyw&HBE$Ppdykf*cy@}~ZLI;^Hk+}igpf?rL2ag{;?%^j zCNU9%NSDst-pNqHak0YsZpop|z%A>tgOPP%d98gxW0n4w9e`D|nm6A~c=X9&I<$az zh}XfS{SkieVdSw?HUOV3eGC?!2MHzr5>lqNcaIVWtwL0^P)+|I5p*2n_T*##)pfe1 zrL0wm)?LxAK%Y>DF}(1y-|xH4+x{=%r* z77*lJxoV`R1aL-eh$-pce?3q>Bx1Pi^}8)eXcU1)EdP5FG3Xo!dR(rSb$UPDypH8V z`OV*2{7$61Gq(f62i_(R(kpLx*iVzanc9b?7jg04JQ}Jh-2coCDh~ zCaenoG43*-9$*4thY@9_j4f5u(QvKI(Ug9iA@O%+-cmD=r8DwBa(8P|*lHL8sAh$- zr`X7e3cgx{Pp#(7jU_;kL^(9_CSgsE%{8I2COe8{vOsCm&6?Y8oTtWiWK*r$_C#`1 zt&=ax)4#j2g}7>UQ<`3UJ5#B*w%oPsRRCmvKv(cYc3fZ?X&ZarTwJSq#1fZ+b*m`?xGG>YI zWQ-N8Fs!F@TOn=QU6k0oqP6kxZ1S^K-V@O=YF*rtvZn>~5Vkz;u7^`~eY!dh7a}1a zI7FtQ?pH>nbO17Tv#0<2!^0-OA85@ncfCsyvPn?Y;^5|C+=j%T~ANzC5Vfp_j2g>-1U(EamdILpBPBzHXKl!+iakCJbnAVai zG1=$jtvM~tB7_|m(()ig>Q%o2jdP}Ss>H!}i@?Q(a!hZ3wUVsnn(&R)4}UL(1EMlj zbv^+hnG`0y0&D^6QrR)JF(d+*U<48`;zwt$@94R(sPv%A1|Chf(1T8pZQnZBCk)3CY zh%}-|BCAl-5uA{g`%cUJz7mErD0E;)vjUt;%hX0#c~`L%#DDLW{rrvwb999S`a*~lP^ zwL{6yKWdjy(F`B?6A>e3EbkAPY|!I;JnHTm`bzICcdy^M)0Jq@mg|Qc_cK3)?NgPx zN@v~>B+KFcL~s;$@`@!A`EM&qEkTa~iN6&ucry$w^=mw~FMnokA*ArEWjt$%u8H80P9XE*d8im{_tg?X+f* zXm>OH;}QfNcpU)OKU)S}BsXD=Rnj6|v_0zD1hVFt8u5|Z1a@SW9cA;srToM0 zrcOyQfB1JL-ZY%)b$5N9lrO)aWaWgJ$x7$S-@$;(3Fau0iCt9&o)aLkNccr0a)j=& zSHh?o;r@q+tsP%ZO{9T3a1|7sf$$H&Ie4Jv`1|9snggMZA_H80e`jO=RI}`haD%R9 zRNka%Hbvi_$doazrHshf9Q$~iyVvg}O*n2N*ftX)U(Y}0GG`J9Ax;whcTDqVBJ`Zs zo)DxqJr>nJ=VrM2h!e#BIRvP(@=aRO+g9tWRE^3B@e&a-c$@{0TolH{89k|b!y#ss zM3NCl_el1-3jkcv<(i@qPO?4KoL`658w{di6P=Z`N7{2~Z!A=xH$brR{mdb2_&KSc*rhf@h^qZXqzv`KS1;IUjDupG$oo$T}e2ZD;b382DdzGr6eN`5BO`leGm?!){aT=a!jW- zDje)v1x1?=hM06|N@zCz-(y*k%~%El%fJyd3OX_M>g^OF2WFnovosP4EZI|v#%ve`C;!a^zS-D25XTNF+Og4vir>o!}4z5`|F|M$7>QD;I_}@o`he$`C|{ z7l)XOEbw161joKwMY>IumsxGZb)8M%GHhlY_KkFdQ4^|*tngsNht7k-2m)r1Lb24^ zqKQ!+-1##kxqn9kk zY#ZKt8BAd!vW&>+0NAdTpJWga7amf9DN{@7XuK}!g~p&@S#DR{54Si0UiH`CKmea# z{zEaWAxmC=c&r(F*ZM2m?(Xhx zq=p{4dk9Gd28I$)dPtEHiJ?mb0qO4M<=K0Gc)#u+abNdZ>p0JY#~Z=)q&VQJwYZ{+ zza%Ggy9Fq2nw>P*u+y3|^wErZMQ9tKXDAxoMAJt^=-gb?C%V@E3K=bV8Yy+-N9|hv z`*fsHqj+c|S)9;a;eDFKTYF#N~D0 zb=Ep~e(3=Iburuhq%Yclh!-A`wqZJdQl~!1PDR|pQ|ghTA|!X!RXyBTX-ik4(NFt?nT3MZ>_KmX$^p z&6z`J?4Ls-5Ahby{kW8hK~n#PbAK#m#UlhL13(Uk$jj)0EeshiG9Qjn*Z*DrONJ`z z1-CySbLan-PlX?7#3;Y+TPywW8@vQ1;gLLac)-y;%E+92JzLZctBx|7`Q0m9dxWx| zv^8%)vtd8$uvP!MD;-V;A1#0%2)yK5!u!j0BT0j45W8<9Q#T_I(X@?iv&l8yT??;> zl<3C)WFl|_8{R#B|4@5j=Me~e_aos<`I?!&HhJMyt}%pt%&%;3*V(F}hMuBvQ!v3! zLWM_%TTw=5uo+GmIcw=<-w+~zA(M8H7$q>{PO>@doijC?7x*~rJLs34?74piASOK| z_V>_QR#VXbB2KVaS=FWa_oEtucZ!F;(@d;-f6zb#Y_jWAUY4C|AW~7j@yFj>qmpm* zhW9Tau{+c=e}jLw{HoG*mlKvZ$%(0%Rm$exnXD0J=OE#m6xn_S>t&?!=#9k= z3P%Ga5ya+X@7zTOIsy{rvz8BL23(1QnXxkv#jA2aJqPIYD+y_fVg z`Gl-HsspZYqb}iIToc_w^JTt_8 zo4qgf9h&`@+k6zp_yt_$#U*w{$Aa(uK>4$V;?fA$Dsve5JHvWiV^HQ@?5L@~eU3Y4 z6(T1_al_=Z6O1j1-(5|CUyH5+iE=94Ro@i-C)fy>)S_0eUtGD#F;|{2lEwRrpt(bG zfhK<2hBixD^C=}GN(D07dRe5o{)zIy4d35hj+HDzo}k9@-@=dhlZ!xEA}Z#`16;fe z!%;l&PwU?7qFX8Q==&~E|Cj{A^-io8Ped^yjEzZ(2QMY8Bz{EvY0LxPtO@6K^}|TsqXYZktb|~K}|sv`kj~ah^&l9ga>NKtxFmX3E{+7OJ zJ?4f2R&!hqaWXW%!yrM5ChXbQ?k3@7qL$3F;t}UKdGPj7!nbcu(nUSqGVHAsr4=w< zit0N@E&GG2AWw>v)GYY{&E|j^S>%ooF9VM->`vBjEVMK%W>0Ok#&Z%Q9TmiYzh<^cBZ_eK;6nFSV-v*z?k?k9p z&z~I1bO0AYVhXx{zOW^c3B|J}j2qbZCu5;7XnRpFkMZWJa#FOq{f(l)n&-HCnk#(clY20gu;sA?byv^t$KgMd_=ceGu*T)p zr?|AB$Kqsjh8)lofSIePXj+xa`ZBw4SLigb`Qkn4!E1}IQ1Jl11u&!v36-nI*msE? zB1@^;_r{tIvbI%(5t>J9CJavGKPRkm8bLb;zy$rGr$$FVNpdJpSXWOPDDRxR!*Y~$ zc`xbR4vtF<^%7cDg0Mc02Cj@a8A%!^DiE7$82^+A_7;}3IGl~qpDcXmr5dZ5u#NGqKJ-E(hi8im^d^$6!v{^K8rd4`$iQpAWOn<{GP2We zhqVLO%c4li!4leq`t9qu+A@1>H|kGH*X~CFZ4~wTlbf$bgEIy-UST6a@yGlUa5jSx z(nE+iysdMm?h9M_K(}j%jO01{))Uk5IcZTvqKSuIg5n(>~;nkWp zd({X;BVF_O+$dRT6ga$cV4>Jx_ZzJGI%?v|l|ce4kI3`L{@`4hL0!uz9&2^ACfsbb zdnhHJt}Z%uNp`NyNHsZ|%!SY>yOcY0ZjU9c4D*45X^dh1&dfk9-4!p%U&Bj@rk>yX_Jl`nUdik!j^w2&zNrE39PMgCs+>H!& z7&D*$p?pvI2Uc?+$Vt)3__RJGnyD`OIzX4=T#2h^_#!ZpaWzac7x0N#y>32Gia5JQ z{7q$EL7gUAe^8b=vf-t@Cdz?wxHwh=3-T3XFL}~|a&MGvV{7wQ1d&m?P#Al?D}RJt zUQSMv`>CQcuBD^X>C<4OO_K*vb!v>z`a*X+wylkaj2vd)I!)dGAX;?&Pqn{#cs!>x4B`w#4X83#2;p2bYnoLrMU zFr6Y^`TE)mhlkHV*A7zZb?V+vE8*KDL;$n_e-$F2DNZ@!Otyo%F}(l;;?u)G=elxc zTw=UagC!?j>jD}bmp76f(K+5n)e^cK7ABOggk{0SB94)T6zC^PTYP|HXa;Z4vz)L) zn0D=`>A@F2l3=G;P3P04^su&oIXTTLO;Y1o8lmk!=A`_}hp&l`?TqVyrpCNqT&I@6 z>qPUnb^8Pz6-(|3A&s{DG2PB*Q|Q72^l0Bwq_~)?KKtZH@b%kYv*{^hfkVuaEF^r}v#VL13e1g&LOE7F#pNRr3I zee+)c2GFi~sRe607_}R?O7sm%I&#PLr-)h^v})y9pg|%(oE^})PyTO1`ZUmCFw8nD zh4n57ued$&J8T3f$6xhLAiQG1%ZCjoBD}XDVaDW9_@Vt>> zDd|dN$J}_WC*M_)NY{wkQ-iAP>#sLAEUxSK`i^#l?@(XW9$45z%IlZkX|m&n7T#*X zk*D2DlI9PR6Y)$u)9j*gbg;HrN8O0Jggq`rKMvkg-kY2FaYFgyvWQ*FE0OebOvkkQ z#x3?6qXqsc*6;VJnbGc4>wyEm7-^e@Dho}?7RjV>lT3ZcUAaxIP_(u()nFY~SV^qw zm*M$nNorL(sw?oy$YP zpdQGnYoF;%eIg=+;%5|lZbs%J5l&{R-4CSuYY@rGaw!2zzi7IJBwsFI{jELCRAJ9^Z6nV^-#g9YVuCna22$zV9H0|Kb^mxYLL) z3$Mawg@Xs3NZ;=m0(epr_{Fa3u}b=-L-ivrk=MuPd}?~XkT%WGi;t}UT#`fdIgwuIm=wy>6%KzOFM;75YE&EbstB6r1;VFFe|Kcgi=ItbfBDT^Qt? z%()H`8w+neE$=dT=Hb3s`-Sk_SOdVV?Gc!Ec0hO&O5OR(YnUIBsQMKh;;(8z+)5E-Sa zVz=H&k)Y&_s?Dw{z}_Guy%s!Xs%wp~PzO{$y8AgIYL*HP;7_fg&^iJv7W5H@V9nTsGnarR$vULk}s_r2rqxuJ`lhg99B%uS+=t^qw{g$;NSnfQ|wGPUbG6UJy@!t3v5d zsFTCurmV`=;>=a1%_m^Bi=Z2?kaPF%CRlt-!9|&U>?D&u5YNN`{PtVTu2K`Js~Ipe z?~ufhK83s%Cg(*K_GdL^`d-eaVYWZ_^}DwqEJ9AV>Tii`JK)*jsk}&SV2e7xt~y9M z`E`M@N~2cYhgP0F*TU&aG*iK}^cpFL=^!>Scu1SQC|Cs>SkX3TyvC61c_jE)}G;9$vF@J>uI>SCplqgHkg^9VJWFiWYD zPZ~AR&Wg$6?I^#USg3?Aj=310B2S*0)!Q+rloF z!yBn)sPl4y>JfI9SHFjf?-zV%f{&&LW-DqjEsJV4ug0%5FfssF)T~}of%%b23R!k+ zN?p7n5V$@CX65lr6V}`{4`fB3{8hYP=ln9{USAT755zy7!0!ra1!QR13lkb$#F+hc zCIH@0;dTa!YS-70$G~QbB-$~V>Op(!4b_@A$fr{s53ix=9yYV|mC7NRxYp)4pY1wR zpqg^(ck5aQxIo!VPL^&e~QSOc*dPE^}BM zrWAMx3w1cd$#Ae8kkTlDNya`z8Lsa66}>)+lRPS%G# z$WSlZ)9pgW-#TZkOw0LCTGYq+Rp~H)FMeq5 zVpS1pLiwPV|Fxw(k5Se^Jsu{%B9L~eC!szTZ4`UctObkU`(ZHEXOJ7svc~t zaIn>aV|>xeJUq@cd~|(gyw=>#5N5`>+}|=qf9`KOEqh__Ug}w}6MSiq&1;ywPT~TO zJ?|JZ0WJ0Jb-~Rr!671p5hForgD8|i=a?8RQS$ zDcws<9tlU}MJySvODvLs90#XD*J!#yP|}d(C@0kTe~rdM2}$dc+-ycqG5ceCOvt?Z zNzqy=;OsnEL}+)uDisn3h%Z?qDY6p8jbi<%EPDa*UD5N{ta8zoB)3nh1_#}l%y5pI zBq@~E-!{ekse!Xzzg2jD^^V?54BM0$h27Qp=f@76{rR|1-`L23()#GwBRpPq1*bPP zU$g3(h=*e+Q81W6TUiZ>Tf6!FTmlQIMk-xW7GQcJ+P-nt5J<4R{S@#t^5ODUe zkUp^`QmfjTepnLYtt~t#pEyAOr$#v`E1^;>KIkqcB3y^edk*C(Zuo~*I{!Xb!dwPG8!&5(Zef_$WUpVUVu@hW62-#MgyEn_}^O3eO|JV7)YbBD2nG>$ceEs z7{4N{Yq2iTK!j!dtCWKx!o4o6oSj>%=Y@XnozLy*Rb7qvRqy}ttIyEME^ipbD?T!j z(MLaF9YmiZSu3{zoT+xKh!6rge1F;%ARjXs#xdUG~CuPR)sK=2Dz=Rm> zFLNKL!d-PI6+w3AeNK0SQFYs0!>A3#=8N(KEUcD!gyP9aKA2a#VZa-U1Uz3I)qG)QxbSm24XT^@aO@d2-n_#xeNmmN#k@a z&AD=a?foz7=Gd;t87{pU3wQDvKTRMfS-l2zGUYS#eM3q7IKN3C^2661j4?dZvvuI& znmY5lR;zro&Ro}0cXILD&qZbvJemH(ZftwpmE**VuQ-)G+lJmfzM?~f@5sqDF^au? ze;7aUfrGGV8H{4pxTOd)VHLs?Ot)`7tj@)3bLh{-ToZiVltNB`8q30kOh5_xzy8fP zF)#j2o8JHYo8i*^BH9;D0pJbHrUcY7y366kF%pdkmj7-N6$C zBtr%&57GU%I~Gp!*;Gw-YpijcM3qAH({GVbbHA^GPZqy`Q{|dF>ke?!YG#CDaPOGg6F(DGPwRl$5NxUr50XT=9aNGRRlkyAvQOt_u6&;<1a566mc;#)7Kh> z4f1SO`?Hb|Gw)pht{mB7)#~{B5WAJ6QwCi=?yuofJSkl%3)aBFlYi^ZJpO4N&R z2@Z*N|GTB=L@qMc>q{iGm@&YrzD|!v7X)m@x4)ErXu6GhEig*dGFOvmO5|c*qyf3I zd>L?t?5{XHxGTqeVvS9y8LD#%5?*M^sFs|qzyTCtgA-f2`^A0Ck@x-O4ldC%ctuvDs(D!BiDThgmi4$|f$aKxEWOA3V(=KVA;WWKD)n zBi;shy$IXT-t>**?xmhZ<;Pnc=3@N11nM|k4COvH>;}}X$~(M(yt*zNi^#G@)rc>U zfAgp9HBaddC<^iWT%mwkqJwz+&Z?6N9oI)*iMLKa@iNij--ua&{#@ zmjTmI!5NrWrj<)sIj+5L&coW2@NCg4xTCL8sh(Vni`ic2w`71?zTDLFRp|R|nYM1P z$o7fdIeUU^zl;E4c~O2Q*Qxy<|6oB50cukeD!9=R=~W2&0jb*pc!^6HwEEap!A_dy zNCycpXQ4*GVQ%-dR_Cs@Q!}-v?84BL_Y;gHKqhlYdtBm~V;Ihu8vW5YBhk zjw+NzT#xu<-CFV(H%NW|KOttX#xYv61N&)xGyZ&p! zh4DwQ1xf*Kl!F+hka8&buP9D@m!DBnp!{i9uzSf?z+wq);zM^4x&Uy8Cvrai$ z<;3(>1MU2G6@0wJC}TUNQC)y}uNtMjm9NKV8|VEtYcB7~n$CKoIQlCi?)V2OfL0n h&D}Xa(siu=2@aNFy1#u!K|*?dKAchCw#0sg^gjX7c|ZUF delta 24273 zcmb4~LzE^>7-q|Mb*am?ZQC}wY+GO1wrzIVwr$(Cr~jEVyP0!lPA(!=xrjw(Joi5D z)F^Pu2yh$;K+bza(%WX{FbDNy+h0PWKMjNf%v>{Z-0cSg{W?-0!qFH@xm1n@MH&>W z9k6BY47gFmkMDK!gzU<$-m3o6>j@nTdq(Jf;W)-S$9&wS zCbGJOYj)LdKdpHw+{q;3%cr}*_5Ru>wx_DdF+}79{7S4Q2E&!wCjO4OHQ2s*(HYDF z`p{v0Jjn&boWWxS4+76%x;y;#+$KLSpvDVA03g&a<-Tq7<_8seb-Pr3Ux5N3dkMF> zas*_y-xkw(AZPS$+hWtMb=0mve}S>MBXsY?VaJI3*<>TffPi%2MDObVfPi^b?7|}= zA%YSCavu9S2wp|rR8K!xc1O{`eZzhhOsF!wkL_T&a(CkJ*$-#ViRpw)T539<>O2uLkb5$-GZKn zAUJ;1LqmGvrTv)!6^`X_^HYI(d7>q{>H2;~s4z43!2M1lVRb|k=%7n?OY9bs1t3&z6;I?ewD zP*?FUnI}RUkdA-va+4I^e@1wA{pdm+{d1G|Vz+Peb8>FJANP}y_w&8d;~7A9+U{79 z?}n^^4~`T0o^a?pBhUL@lliutNrW-MAClH<@EyakIAl!2hihyOkiXPp73CcUy*n1? zCn`O}J}r2EiNzoLoZr(Q-3xipzL)U^@HDx|l@W)=WpBKs$@ddN%_cd5@yOVEed%Gk zcLi<-IpyMhzF%mduQ{GRu9!P@K7vx6zAvF>U9RMdhxW)39#FiZy}zpS=e<1qU=!D~ zmi+9zO+OoGN0pBb9D4Gia0fg5&SUN)%VergV3TTHbGo&L`KmqF_8thxjLHiHbpMnY z>66N)KK#W1o1$6=YzMU^#};RGBqL+gafp>d!4VE1AQh0O2{EPvAy|x}{X|}{?(>^` zw0N%u&uC~)8siV-FL1A};f76}3)7ii5H--Y=!N~z?~GVBoZ^$J9 zwy9THx2e-1F7}eKEgsKKvdZ@cP_u5*h*ns-O6LIKgBv&`HNm;Rgg_CFJ_R3WqrpbO0N=(*Wb8G>l}FxlM75>u+T#n(Q739X z8c0fuJ!&}Ny>I%mY%#UTB|vm|^mtY8P{mcR73xKBZHcMtFt-`Ga;~ocEU2g+Z0L;C zd`kB!WB3C_EBJ*aR!Icg#?Hub31@j)q6^D%<^}~aV<5+fw}QFi8DW4KJ~4{4_U6EO z6sA^8DqIqYw*c$13q%oO?6x-1+7E*pqGf8rV+QaiNi9Jl;V5Hp?ygw}WJC834jBeV z@|EO|)DdrYKju3{Znk0o>b!D5C3JxL#1$<;){LnmgL+T0Y2ikYzhnU;F*vj|Mm{(fnt>Q`Xb58%| zFWZB=K~C-L#x)AFq(6kMlcJFV2+WH13CUowZikCyk-Gs8gndi{P6zIxAL5DGRN~E8 z617q+c!)|Ab-I(^-AclsZGXa`Vg7yT=ANgo4Qf|zbd65X$F_f3dAgt|aFfpuzqYv% zISbr$kG*bS$wpX6I^AxYd&(RQDdJ%K44i%DdcnK)+jBf!fkEpW>-l>KcyeyH1?@Uk zZhU6>i`Iy#97qWOyQIaA1iIccWqYw zvq%In5L-wm*fgc4)2%aTm@G!7nc-=!mC$)JoK)Ki|CsTQlxS+~bTZAd8hy7UO=C3~ z=6RWpV_#$g6oapyW`3Q8c$&*d7sbjILe6)dgH%=3QXZ+OY_t=yAEF+e0%b*u_yYlB z5)3(e)7Q^NtRvZ9G7o+LJ?LZJ-qPL&`NFpGP_z*`#GW6b1G7K!jt%Sdefa(X@{9t> zWeN$!OMo06ItDH^0J)ikces<#+Hq$}Q4}b(#I%n900|F@0{iiJh)_7pqQ5c;io~<| zcgI3q>>x?;6vxEc_wKXxF|W1EZpm5{jhU*Li+Z2x`D{?>j%8t)YQ)KbwT3q z=<4DEx&dzB(|rEI_AYnVua}RPm^j%0zHiw^_j@ewcW>HrlZ>u!8ZC(xKE9u?!>w@3 zu)SxloErV>;*uGynmPa}OUe*Twr#btDqVhJ z5)>`4OJYA16~rY;Gvp>x;yskG_GOX#G7ru>-R~W6(9RQu5L0g^vjg>)mY44x;(Bgu z&Yw&98&`5=&j~Z(mv8j#o4W1T5YJ}OZFCR2ZM2B&Y<4lGjO=(T=*|?hWd~tZC2sgm zU*U(7-L)0F#5ib9KsO$u5DLLd?n9*gGtm~KV<<5?L0JoWAs3(7S zHJEVf*pj>dFH^=Z)4NY-(p_!-dK4dVbgqf z`j|0>gX?j2{!kwG%-GC8&mo;Pgq609=h6~tH%3c?y4ype{pe-}uyLlQ^{F{na(#3y z90l6&Ure$iipsm)+dnWQz+6hru?6y>tUlHyX9Gekuyg2l=jtwQX-2lFUT1V8m4&7< z?k>v=rB)eEfa|hKt!dr$5fpD8Rz77uyalYgI|5IJ0>S=8>DIQ0xP+7AUC)37wnpsS zP|YG)Sg|o6(o@n>dP>3ZNLiQ=3$-GVEaedOy+)zS;iqKrY?6J<4z&vFzbcelY@|#j z6`Pi%(;Z%uLn91q@ITXW`93d7woxWsE$r%Pmn-1#fViqa`4l8VZ$op8fA_bRMYZ>R z(17bZFcf7{KlyWTiFrNgI>Hf>VFGhK5{6e2bkqPhNg4b+?q9e1`8Qrp1ke7D115e1 zheYZO*Npr3rw<`ezPYcm!a{Fg;68acNk_-raa%h`dEW;t->9EA@bQpkK(_)Az6BO1({ zRQ{^8G==#5`=h8lAzQ1~CfvtlCHNsXVIeesDrA}k+s^CiLw}WVvB(C3hxBc1l%sNpJb<`-!-E&0{j(j zI=NF3Il9ds)2D$UYl+9C0}|z>MQ0QvQ#NS5xWVg8bu=TIga} zV^fiUcH0P_ndH6g^fe#lHcDFwBJpD9ur8lnIH>#dLb~E`E-;k1@#RQ?t>yKTS`m!)yWh_F z+Lj3PDDCiYImoyUbc_g`ra0U*fMkAa905QAC~nW}o_-I7(+R!s=q&v_#3^Hj0I~;F zvC8{uoq8kwNJUWZP~jXr77?FtJ4vo$o~yqoS*E{?Wp(TGL=~3lqID@l+n5t-*9*)~ zp;2s>5mC~1*H#ESfeSC{iTTo6A}mVXNBi`dW?wPAbQWBRHp2^^vV6=f;5CXSCE$$C zQshtj)bTw7dklCSyKsVVsV~Lzo6J{?BP98{Yi6q%`h{zt%fdu~A<;o9M8&o1jil!O zwu(o#CnmBl9Pde*Tq(wshj{b%z5sOn>Ji9-Q}diH+$^||m4JRTejmS;~z#K=ui*aIU%Y=_M7|6BR?d5ktlB;= z8wh770>Pg`lpx!2?t6GuMq~`fQJ3`UvfH)r#WA0t0NS?~AUMjWb2w|XEjkVbkNwkF?0FzlkpA=5`qY$G} zHFpPj;10#t{devM!1rUO=VmtaXYc#tOBnt0TVH%S^#f77RJ{X#mHYkso$uA%_hsYf zR?#cdL+E*zJUc-f``~Tbpko}7xk;2B8Ihqqa2q;cXw9Pkm5El>p__n;CAg1vU%TLP zEu1iR;&GNVaU!uBfT@5s7QTiJoD@|K#B^uiPb~#UC`^|JRwn2+JUxEmY_fT&s%>lf za`^AvrQ;)whW($5u3J5a*)d!J>_J3i(E7^_Rrbj$8NwUc)>M*`TXr?FUM#xm`ww9eu`*{ zEnBMJ6+u|@NaG%qmV6(+Y1Jhvr_Ppp6{C2F#m_;4`*C z{aqkp9ut>Qn5|z2XDZ9U?akLBnc`lQk}Thz6EdO%NF%rdpD2s89&Wl(btXz8WXKdV z?}I_rll{$at8ZE_?DoBJ>yLY-qD5ak9%=o3sou6GJ@$ncZ?yW{s-RjEz*SHx&61!% ze$T04K%x%sYJ?FR&J4`Ze!xgXG0XGydaFc~jIu&}*UD{Ba5ARhw43OjYc3w>dlQ9= zuUF9oSW|HOcN5k_!&Q>hBW$O#uUmc>z=hC&C66RG8wcnf_1Cfov$$u)Bil?7?4!da zD2a(;Pz#j1ayv_mD|L*bCzTNTNULcDbM%V%NnoX^emsb;T6WWtyrjKQ6sduBE>!s92SSOINehK(!Cy2cin`jz#b7gKqSer=!386MOOP> z^bVsOv(bM$Ol)!GQnJvRif>&`Ys|U6ymv`+B)atwm%id}Zlb8cIL$V@f=Es#wb~;2ROGSLvQe>gDAov$O!*u7AB89rutr(1%TF#J-w|t|S zQ!-#!|Cn9ElslB~0A^>?NBOe=B zT4?)BE~NPtQbO^+w#hkYe;lc=TB<|_n`E1fGWOUh@zY9)ijHUiuWL~* zKiAvh2;#ONVwcCKu1?0Lu6w^L4@RXqnvD0%7DSVj*)yJTEx=_vSX1w`TrB$KnK0YJ zRVU*#lg5sc_P7NX#jL3W-$fa_ElO|gDr#m+qN%kGX-vG8Y*f;7g=Aj8x1DZfd@zg_ z5-poHU!4~^&YGI}h7NzP=dut2&L~g_7gETeaL!#%qKl3#P4w2hI&|AjMj(!ItdT89kR}GsBB3#`80l!e&!tzgZ z>OMouosJap>T@o81LHRcXENyb-PYn!= ze0X1;E$yBqOQQ2yk}y%9sE1PPK-D)@x~k@{n$Q$7mPG?3kx`(F3l36{lxt|1gXtpn zyQO*YIqH*-Pzgu^X%i)7KSNTGMV8ZhxllDG4HyH5;QrShUVM z-|K5at1E^zPpeWceBPAKRFqkv8mf|>GOi$WLF7U~G^CtPSXzPV0z`;-_6oKK1eWAz z@9o;pHS0qq7otS3R`r6TrH|t zWS`!Z&7#x<=sG|-CW|eaTaV7MXgl(qne8#3C_SxU3j~wQr{b1?tRi53Mu~|7^cYTj zW)Zsr*LhVG-}l4QTImo$?r|$W4QK)5)v+0)WtUojGmsXlaVtA?tcx?~NHw5lRU{i? zT$q|W6d^2kmBWh?9K0+7L|B;yT-wwsvc?)04pI=l6Q`Le zoPHPWH?xMXr(ASF0MnthS93@#L_vCh5K}-Y->u!xDc~@mH5KWt+Gr$gsxvsPEwj(7 zC(4bQE@&&?Y_40wv$gBA%pF;_+7FC6a`A0~4u*w+h>+y}PJnV|k`fkUn;;K9wPflQ zR()gvY#FmriIJV^;7HPW&>Ki>*n>cSDl39!a7L`RdgJC>V5i>f?cej7I)5}jHy0je zhF&(4Rz+8+@2!#Guh`;k@cpCA4050xVq8z-$?q2Wb{ zNm%%MK6jdUP&x;chB$IQiiT!h;_nIug3;d|!QbB^+C`Qz8H`LKCKRj{@mRHgqS+|& zNRMY5OpUFjy8YrQ89ttJ8!tVZbiI^cQDo0>wazW|+rz1_O6bZq_n&e$IA)69qW!J zpiE81De>T(;>x+qy_8UkUYVmyWR6YYI`~1z*UMy%OLDk2PBEfOl^S}N%=eu3ig}JV zgUF!hjvPy_>ejR+iy~do+#}~CO5`B`nfm&9WIshd+?07c#(3~@OglGATUR{5_|!$1LJt~!t$WC?4&4p!g21YnMR2fvjH#QNP$Kk z0T$k}kVKT}hbzsB^*HLUJxhPgWFrPjOSN``;6w-=<8WDdNu@2Y>Q@@Nwl#X9b_qWd z0xx2l#frJqz6gG|VQa6VttGn%?$rPg@UZX+dMhDH^`45p0SVWJ4*Bm1plLU~t0ghq z_Jk7HMXpyvogPc(o-k>hQPxKQ4v!b|M++_2L*#|4`)O*4`<$BW`^yoU@EIZT0=@JB z{N=kjJ-=oMkyc5Z57?PPHxz_gZPF-qUYirc8r;M&TR2Z(Z25^`On`>Rd}O#IGqL#l!A+Xos`lxnKS#e7v#4u?whf5*Ora$+?H^Akkmz(zEw4ouQc82-YMvRWSA>yw0#(Oi9>4 zvA=MMaq?2*TIt1EFp0mJfQi9o=SBxahN4hj^^8!U70gG0VMF1*aL0qO1WwQs>xQV& zYc)Cz?;`%&@cik#F9A?B2X^-PfMsvY+99YYC08Pi?|&Yai={N{m_k4JrvW6?rm(c?}Hg4z0CK(`Yr50a}?*|t&qGccnryAXIxyFx6*A~ZV$BqB6VqtM48 zS%B{Euk)W6gjm4G*Z#k?cKd5*XzvNqZxk1SBEETw8`FRT@Q7p(23s|7`fIH0$p@DT z;vSpGGco=WmC6iN#1E8L{xJ=U+%y5iCxACdX6Xu!U`^(L-^i?|x^?>luih={fb#J4 zA2D0?_QKFATND~}uBY`O)c5os4HBE*knQRi{zlvH$$$Iz1Lo!IKVVkE9w?s$Obu;2 zXGYgNr@%V`>X>!nS&9Hm3?Xn)NFWH09Sgin%+_Td=91+MZ>+M!Zwr2M3qAab`{rg5 z{D&zj?q9sdC+-m*H@nrJ`4~!{_x}MpIbs`ud*O6%4J!%AE&!_C2fw;FYUt+**w;(X z_tM3SR3ZvuXm{kWTOZI3XHq^^?&R9*($`hj&kG>za$0Pf|6`$JYbj^C-}vy zYGrXrD7u|8MLN1eRFxM%=FZBSAML~VxS9D_$lsi97Wf{s#^kiTANo3L4EOx&_1QqY+yYpcG5$^( z^s}L>Kmz6hTIsok+SZ~Z5f2X!&_!>BMXE`F!c@!-JIJp(ka8}J6BT2(6~~MqhU5je ztEeNvyZbB8>oBNu;FcJDLW9PrZj#FgD-w3nK``h85AwH;vio0U2V^$t9iotW9D&*D z#nNiJ$Wwx9thtRY+soh3ywuG$#t=Epc42jCevx65V0x2eanezHkgI}z09o_XHiNKV z$Cz_Kz1S#y&~klX+k^~#2hjhK>DwSBj9KK~!8@V88z8b>d8Q9%s||RLY7oBI{FR!HwmhSDHD*`nlpS$W^Sl6-Z$3bWa9{6j zw47o0#Oil=dr+R7PIN^g7d{-o;~Gw=A1QXp&|-6B^x~Gxe<`$@&-!W4WjXZ0dd@!9 zH|V8Q!lor8lC$D%7lYyot4|3})kDq}tIOmuWv&*ZP+xh-KmmF9s1UBr{j^Oct--|knt#?YMD8b`%eS0qU^vO03lZc<1PC|5+<511pCL_c&O5c2huAd+eXQ{!pd znohpKRk!ZVsXCfJ<>EM?Qv@#nt_fC9*}4FFaH79$T>JNsTuN7^qLgb_hcYy>|2wi^ zi+d%)$zA6td1)r@=TL995j@h04UqcFDo}l8pfkbfeN2*A` z(!KQ3TG!)qZtmlvmKU1$SGr*q)ZO1FE@N9qe;V8dID?3+zJ`NN2JXO?a9#IAF z9skZd9o!^>Pb)BghK96JMedj4N{_P`{B01KE94Znaa-9ZowdMhqhYAJ z_mt^k7gWSYa$fpPJN9N7p~Z6uMMSqAnCcb;$stIs&oJg6zgQ#j^BHjK-rppB1oX*N$i%0EYQ{M@)4C{bY9GLJ~*kt6NaB$@Ao zlVWpEgr1~Ld%_q27rS3Yi+c?5f?Lo>0*RHx8g~Ftqdp=*V;2IoQT{`6?DW9|Mrs>1 zp5cC0n3Zi5ej6;eq&a8H4DXWCej}A=*7ag>%iaxsCzA9l@Ex}Io*hJ%RmPV$@rQF} z#v&WI3`M`qov(QYwMS6H!W$jcboqY5g4OVvdTt4bM5Wyzk=KCf_`JGv@_-~;423w; zh!Fq;>7b!#5*;KpMibvgwn_XA~!s|gZxlAi@cBxs+qA6NjCc0)Pu@KtMjU`&iYCkCW9}VTaR%PV2mqPh+H?TJ5HYk(( zpAZH{(vR#G2bQ0`&``nZAbF9T^-XW}ih4g5hdbC&5X?{Nr!Ul!W|OSimb` zri5z9Sf9U2Stn&i#P2QeZt7eK;_`}yJ#+NiCA{H{0e3IVoks>fQaGiG&;+(CLFdU9 zspk)&micztZhyK`@R!+?ojIu~l_QdjM>0ZL)1(4{<^pEAz$#cmb4wmX!yO~ z=nu7w%Qz}vb<+hoJJ5(i!y90fRCXtXRU!^%)NB~VkS3PwQs&I81mzu7e{^E!80-~7 zq0y?M>X?Sd(?)2Fs&vQJlh2L6^i*}U2=!ygM#sCtpb~7V-Jc1=!~FvxJHnky~wp#d0+zYpypYP zbsa!vq^P#R?P%w*(q%uuyq9FCiB5!?ZE&;)c%<0X1saiE%a$w9n%k=fLQo0~L(gUzfJ>i|#1F11DWvVP(6+Ml-;((d;V=sR`QyC5TfC;zb3ciho? z9qHq{;8f(3HaUE$3XBm;qRodvwJ~?BxY-B4+VK%3J&Px#s5@LDd1lPIqnebC8hE#@ z+K4Nhn^);WuUSD{`xPDO67gH<`W2m8#38-NK-NFC9)?ybWguqRe*k>I{+0$2P%R_3 zYV_-HU89zqY6y0%IKH}Qnz|7XhFcO6*>?Z(8&`aUU?zZJwP0oKK`xy{C^FSqMYo7bl;-*ySdL_`O>3|XfrJk zU)!QcD9$qFC-YcmxR-V#KcubK54bSS3D%roiVgV}I;zS$0O?*0*R&G#S(tl8{;(UD zBe8$yD{WdGw7W^p(0-rtCL~XHbd-5jK``QZL3O#E5noY$dy!^$6}B8>o=vu* ze|)_RZe@;NnolPp9DLDLe()7A3RjJLH6_F}x{c2SKoGVX&i%7<*S)o&`biY7Q?%-G zul+!)sK5-31xER&-i~>a%M0<Z2ib-hgwf@Z z@IgD0A)(&Yp{-m^*VPAu&L;YKPW??0l|^TzbwW2a3QxjSUBRZQUbbvfn*h-MG-+LM zQ44#yC~9&+jT-)nECzM@D+!fSeAy3gSZjQy8R$o?_DfsQTC@&r=-&pko|S+;I$du` zw53fbM7Y)5e>u;&sRM^<#MWcvdq<3y_#RW%v6F=I%j-!X$!y3)u2_&1-nJ#>~%S7Gthbd zd{Jw@URpXWyVY0tETeS|{CgY9=3 zliQ}AD4V_$e0Z}7dIu~3-|D$2UwA*VAQ77JmBAK4{KR+H7&Bl(to8Pe{WQz3BzdP3Ge3>OdCPns$QWT ze31U_EU1;Hz>jT(8(S%GmyIO;(S(GnrLX|qqm^hO*a~<=3vh<@qP=YTyV9=*>!sbl z-NLOZ2srp{ zM6}v2lvF7x(Wet>NE@MRWf^(8v@aM5NsHI$KD*!7>~7yeRaFi|GvUt3;^q)Gl?@KG z3iIM}RikiTsSx;lo`tGM);tK+E-E&(hjR;r0)W6}+}OP=LWRS8Nwv~OkFYNY$V-2h z|JA??p}kGqYm3gIxohm72ricil?9<%XMwr~@R8^1WfQc7m30O!q>@iac6&p{%SEQ! z5{V4fBfshf*6)Eo!J+jZfBf4HVK!#1`j!|GA4W(sXoDOKzo*TQyFhCUQUXtzduWi_ z4cOp9#uYi{=p#iE2UaFLOd`AuDa*-g0k2V}q(7BFg*fWt#3WLURQ7{{1qC~1Q2{=C z1hR!{4@G!6u5V%CEFj4>aHReT0A4>wcR-h$@DJ~?^2512Sj;}Syx{KJ8?--?5UVR^ zs{h0GIK8wmPHCTaBC59fiRAb=F^1%F2w;`RSW<70SSLC|(-d4uU*E^_)mlx9Z~sEY zv0m|z)8`-8eJ(kSXz)WeRo0#@Sr4)O%T>N4SuJPjGB3L*`BJeT%FLtJpWoKT?$lE9 z8Pk<4t(#nl*8ltG(2{rd_f>VporEepY#fb1J9~VKpys!P7Dx*nV8|h8sxYfK?PaO8v2odTIbJ=Fj z)`qrMt*;IyM%BJ5U?D!BK_3lB2j~U@9{Ods~v0VHlYO6+VUj8lW%@`K>Rgh$Rg13B;{SjTQ@h{VO@L z|5`(k;|7G=3cIUSZD-W_G}$L4`|9759MtufO5%lTbbYcM{1tsy+s_Ud@J$BtxuV+l zfrZy2{&|aFh={=JT=Z9)x>0{#E^z@aXG-D$hi4+-%X3n1-a{54FDhjA*Gy`xlOTE? z3`${?zNm1LGGBW5DeNjTiS@z% zE5aE7kx6XZ*YhmjuZmi%J9-tGiND$vVE^hj#1;v$N^iGY_QxAhN|DXy2=QG z&)73$>UJa>h3^6^ht%pYPn0_p$ZqB3|B`mmC7xEgYECv?WBg6&=X zC6nG=Kc1!9P7|s#NT>|k!4SLVMn>*rHXcjJ6kN&NKn)zQSoh@5qksM4n>vj|zK$Wp zZ|h!_d{bHeoE5gLI?2p@6 z0akswZY4}>UG@Lh`!=C1047M9U>k%g!tWyyuomKFPi`13Wd22f0e9ku;w+peE=rsx z&$M-ch3JPIU&MRM z<^iUeaCXY|B^nqqBKJDMFVX*M_kO1VcLevMgJ?rp6W)>u?S5t0KZ6+19zbMQ^uVNs z$HdvKbkD3QW`4*OS%mA7ufn}|3;gZ)O0pZ{T8BujTHypS5l`{rz-Qb-0j zADXP@RofmTD4;0mb*Sb{l;e>h)7#i+`M-?t=tLjGbG})j&i~GfWyxhB+|?-n=wp4x z+CPcA%f7NvbhnzN_wtH>eq_k4B;x-ak*{+DZ>fg--ZdI%RZf>gZ)CgXVJl$rHi-=# zlMuTW!)%tSlt{TOJr$&LaIqu`!!FC#Sq??)1$;7nIQvkR_llEn+#Dzb8?qdDva9K= zp+?I96P-E{eMJ^f5VdRRQCCg{a7eETn~}zwU>wh|ZEisW7GD~VB!bQj8MKKRm0D-m zv8RtAk3%{%xuD@>SIi;BS zTTvO8nLDebX+sl#{$Z)!*3}Y~j)oIxZ+8-nUYsM^4F#9)2|43=N9v3IV|XI;+IvvQ z(5!<<&J5a&CnHkTRki~OT~S*)>&}@}f=Ee?R6$9&I@YbZy#L4n*GoguTr?joSYGkh zwTj4l63)!rqpmzoG-#@kaBtS4N@sg zK-ZdM>R>?x;S)=NkQff5^(vTzWri)ENajxBVC^^8(K!x)CK>|EBS?V#`C89Gp<>D- zac-^I`li871<)k(c*Hw>&i;v}HV1mc>zoTr41CdYXye3cK-F4NW>~(BuyQpuHI@ok zcoO#oujQE7HnB3g;V(QU@VuGPFgjzl8%6L!i@^9L4fK(wA5KUUtC4s-j1XU(Ik=0s z?p37ZSG^wVkaMHIIicaYw=7Qw)2$IG>K8tVMk_ z`S*8O?9m$ULnu+n7ONr(Qe)>*dXtc54V!w;*$hQ(z+xbiv^97ZNot>>slQds>(15# z5Sw!hw=)NPhMNlrr}Y)Q?RBBvG2|<9BK4pNFKTSJ6_4o2V%rzg4YpcuwNpuB5@@lM zZA3a+^5*3PtCHh^Fl-4z4W87QtS76i@A}koq#UYl%aEz(?+>;1h~Fjg@WW+KbEUUM? zEgpW%pPIMk<)jn}@LFoRB6RMnL)oDz z;ikHEU0#_bo88@waCoi&!EnVCc?_)b!j^B%?zdPk}?;!j%&T zu8<;@@lyVcuNY6K%TPYIOFsU*fYflU4qo86Y06OU4B8-zP+I+fqjMWrRaD5IKu!$` zi~OSC1av7aiZ_aUV~ecqS?Ye0FNPzX!Vc8W%lN>&t@b?p_*F!tL+Rgp#;CHr^3N(9 z(`PUL1Z}*q)qPF($-qF;1wX6^S0ZK#i_CL4?IQuFqei;@M zS@TiF5T}7bMn))g=LD|F&`AGCJ)0VXW36lzNXM^v`m7&ud2XE+`VDJAQ>3XaVZ_^` zcJgK-47G{`GHl@8Lo|Mj7K($*6@#?glYdO;W*j*j(j(CyRy^bh1FpumF#+b`xYld)SEEGt`N#%?rGPHt;F50?L^j<>ti`D$f&=v`ezTOgsYMo702*H-KCF*QaF>TR}WU~=#2r? zk#B}2Gw-O7nkC@MlZX>RjHY7|X~d5_0u&zM*FJS2e!f%WaiHfMe}GKW%O2Vhcq&L@4#=UP}szwrJwg)%@sJ!2bT z#Z~omkkGSN=KBU^w0gQl!x`7BAaO+#5K+w!)c`?4nX@o#wWg7%x6^&&xRqgUCKD%k>;YHb z8ygXg_~SKn@K-gDdNv*`*%pR^RJIxrn_> zK_NM?uVBo*dp6D>AhvT^!2j`9z}F8qcm3aKKmG-*ztbu;~Z-rbPuCKWlj+V`P*EkdVd<2zR>30_>0#TVl`; z;=I*ez6aKtwf*7}98~qkYxC%@B%`kL2mBM0fMw7Mh>?7-fIRv*Z7)J}U>EbZD1`_i z2-0WBCLO{--x$kFr@jRh8oVn28#NOXavIm*)%mY4>m>ZH_FyN4&gh2Al{X!1HtL{d` z8BjN6orM1}!c|eaw9(4odFiVOFyvi<8DU%ghqGQdukAi~59n!1fOEqW8oZAHXrI^- zH#uApXeMztRT{x52v0&3TLo!G#JZf$PGdw_*P0{mXuLANZ{~e`2O&@=*bG#1^%$@r z1g3bm;~{ndGCWdiW<}k2`U-;c`V)d^Y4MDz_W7gAMH9i}2(WWJCK!nmYjl8FYd-m5 zNQOd@8a;f_Cqe!RAUp*K({r)SMBvi>5C%9IPTj=?hjLL)Abh@JNI?`fk~$ZL5iI*( z0Zg){D2T#|0^D$&U4gw)ni-!H4!s z(Tv!?aG-A*FoH@#C8Sifm5Ogm#}{~-Nf`1 z?eGgIgacKZLXU!xcMu$b|76+kSooE*(U^L~>2>*q{ivgj}}<*xch*T+u*& z#z9=ZfMcROp!|wL0PPslb1nN?0v%{+(tKul4vLe<_BtfVJ1P|>FuASsn^N*q*_*x( z5**i4j#MAE7Y=3mQ5|KXU($h4QiVSqwd7J+m?ZPyF@z?|j`KbiO@x#WFHV#t0h_~lV?y#daiL@742ns5gmUC+=TrLR%b05 zD~5nAj}35V-9tTt!#wKG*oc+9`i@k*#&FyFDS94@%EId5vAn9<;~WTBvfw#5Gc%2V4RUq%|3Vs0^qbdBuO)71iL5B+{ipg%x`hg^j2YiCTeB` zk5M4w@TW8vE4bh~Ate%ZyXtuzR61)rSw!OBp3m`n#J;tV(y()UkQEo~d zl2~qU=fi1r07bj>lQm?hkyCRcCB+g3G$v;rJ2DTY!91k-m=TaxiWay?&hjk^F8uR!r*wBSARPk?-Q6&NAf3X{IY>#@0MgwlB~sGeEy4hz;(&y}&lvT|-pk}xd%g%dA@)YE0n3P)*Ao3W?bE7_mh&!6!Hd<5Qck|j~F%5is zrE=-skaphYCUUtEXQK`$u|Dv(zRrI(>J;~vc@&b1u>3+|O4+(?feOa!=2dZIv*Eg3 zx(M>ukXVc@Xm$kDo1RZL4{U=!>s=nU+@FBM#}+W;xL&F03$G0cp`|{qpB(4!)?v*{ zGfZBc$w<%gX*!Luy`r+B;eR#vc9E@j-3Z`o4 zb75pv9A;qRo~A7B=Eh_w$w}6c?3auqNR@Wf^K7y~O{FalU8*XwQ)v!wKMHWIN;!q) z{YlUWzm!*bH6S+UlaDkEUG4PCS($aAjN#H;$`ZFxUa^hty7VjzpjGZc1Fy$0&idBvGkNK%HJ$U5m zC2ZAT)5;X6Gq62naePL02Jj`ASA2H^9mb-NPHsnp<9G5TU!vC<9DraYIb{;g$V< zw{9>EnN$hb#Qf{?@AKb$;UNV$#hhq%0DF?u1RW+~8Y z+Cn6c6A#o#JZf#5(LsNI4{+|Np_m!bx!P<*nA@;X*9g=IS2_Un3eSk= z7`x02SNH(*J65t(^2IsXXlqG~{=*J*)&rs)pp7{y;OY%tJ>v_C-^9d_w!umdPD&qv3rGCvyul{6V}=x$9Bp=_kAphM6ieeqIZ*g zPVWZxU+zuMhcOclKhN0rotO|};iR_M)}`_TSuaYd->DAhEB|DCLNv^Aa3b)e3b8og z&=RF_=#BMxwy#4_y3TT1gkSC--hT{q#z_aC&zoX~k)oa~cbc3m9moyAgqD1pjyW%2 zC1s7lm(CyqaR#?dR`;)}A!T{`;LitO3~w#ZjiBoaYS2$o&I*QQTN=unJG_Z zA%s@nq(OBa-{AMdy}VZ+t8JCi&FYW=??4fmihZpzxBhVJpiX+Ejldca?T0;M5&t2; z*h4$4KIuHg|G>BfZk;Yjm)=??qGQlGUf8cJDEdoxM77?+CU0i8p`St}sRON4dFbaI zZPTNQz4Z?7ACS@8H%?+7HG52!n=&6uHSgmFha~(_c`KFRWh3VdhV5`u4#AH0ZH3!s z`BF)>o8xvOlA$}~3=>qhw9mNlw2Z6>eZ{#Z`}R^RucteljFjY0I?#)h?FinVR=|D* zG;)pJ3J9!vA-R~rgBHmM0hfU~4cEHFfAbT@eGUI2Ad=b4lOF`8T2<0$1IEirA{QaL z{*F3;cRZ^wHW@iK*onF}n&$=#s9_0Jpc%k>XEaM)?cme%n+rSU)yb;LYP!_{d7OgIZN5OLHPAJ#@h^T z$+Q!vycv$16tKA0@Hl=$U+(V@l`%tMpO4NOxqHU*+h(3+H4eo7JuTy0UR5oOOj86Z z!)C#hCMf!8PVO$oY3$u=F!m}ixjX_%gdv|=&!oppQ_=W)3{!C^ZwrVxIW8uMs6fs< zKu@3UPC!qem6#siJ)GSTTl{%|GDy8T1<@DToJ823As&7|?0vt3zj+!_>yOYIZzi1q z9^#X|pBF$-y#H`-Igc#3?XwJ_C%A@Uwd{Om?4vmzm@TguekVF+g{H+Z1kZn5& zg<<88+!xbdvwc4@=uw`{Ja?01kH#fY@8yjzYDxBi^%F3?O z#RS%0*gM93#W1ZxXq9x{PkrY`gC-Z2w{LJI=B?hFZ^}0J3wZfh9DIgufAfw4B-V@^ z1`_XCCK+DGIaNn0Ex22oS=JmhHIxV{-Ecq<3@;XI9m(|e{2oj^a{*Id4jW@0L`jOv zA2X#oVQ~bpRRs6j@S<->No`_hazcwI?Q`W4y{$_cKArp>+~%Fs=QM}%y>V>x+4Eiv znD1K&otbHUVo)T$yn~iKm|Yli!uJ;44bIRVn1q(nMnBNeN(o=h9OG=Fin(;ld)rIF z@?sn-jDOVJo~Jsj=%U+fjLT%s2nvT6Gf#Et9bW9Wmrq?u6W5kQIQ7x@_Y0|6=h6;nLR$YpEjaEzUf{tTUA9>U^Cj5_z^XTr?wkzg+p60yE> zjFfQ|NuZs`^2>F7^kD(aSkZma&9pY{D00}zvt9lE5{Kkl@QqPxfFnu8?Iw-SQvYyZ zNqCn0gngQD)?aA;vwCqTl*ur?{GPtD#2s<1BK0_ z+9y+k48SSc;wR;rJTFxYp$W+kY2Z{6z0fzgv^N&L1ZcZCaBh53oKnt>T}tO0tgN+U zQGI{CtTi`@+p(`D(Lx7c`JNw3;cg?+TV7IpbGQ|Y+D}|g)?6i!!jv=Nhs8matIQDrxPYJACgIhleI1hP zoT@i6{__}8T?Bg>v(5rL8f=cp#{|gi-*WT3t}o!<{MYR?a#dsQO84LPEy!f6NNWQy45Wi;02jqbwRW#pc0~v zDk;?9)B`8S1wt_VCEvv3>F}-Q3^xB8()zzD2~rjNoD&ezS*YUi#p_jw@2CxJJ5d~TJ(}ma>x9t-8 z;z`rQ+9KUITSl{kEbqj2DdVl1M0lEo^b* z{y{6hG8vDL5m7-?K3zLdmxM9!8tS{xOy&I*Cn4+nao-!Csj(v-RX5{z?R(4X5Iwn5 z{{b_q0JViBCT?5D8{Pu`aCls>p&jD;?7L;cTrYg8ZWP^MRtWc<5UYMFsIAQ$G&M#G${#C}fVs^*nXX z{No0SY(*S7J+&iSs?rTkD}5h?bW%k6+%D+c{H~@zYh9bi55*4;{;i}x#>#@h#ezq) z!@5&{7>>Od?9E=@i$K(hw4mLn1d{!TS}pMwT#5bjwln$1kxcEJ|1dn^C^Yyrxyh&p zwpbP(cP3m3|6#kv(iQN$yx{3{5OFm9C#)_Pfj{vr7neQZ!ACeIyQv7J8(zE=7j-Ln zTyIVi*drmStTt(hKGp08D(eS~^x92Kg86LU60FY#V3Em5L90$^4K8a(CaVG2{cAmJ za~wHqB9T3VkR&UT`$P|!MD@g8RQpLAtV!YzWMV2)S{hJ8?Q}Wikk1+THfY+)TEZt( zN?)Z62nf5GJL5mfwVeyC+jC__^Zb}srld-%vRxY?TcAe-bEAl;OQIK|DVtLxkgpMp z(%S{Un;(ftVdns-A-clNB(4->tz%(mVz;F^`*)gKS&YCe$~`cq7{Nck&9*)z6wu{0 zNj1xLgo7pmUrxW!NW~Ir6&CaQ9#oOswXi=q#~I=`p7ev#-JD1rI3{QcXueI+5b<}rsmyVOC@BnAO_4C8sbXV z`PhbMFHthkMA}HMTYalfs^|2I789J&*a#%K%SchN&)^hfplx=Q4tsQ!ugRl+Fpg3) z_(MEoBl_`wJF^03>(6kV|5!U*f`VwGRAb+aLATsh4rjJCySPsBOVV{+TfaB-J7dxx?{`N0Ox#3EgY8#UIf3-yMG*;WRl5r|j%gPV?VEed65C-`w z@-G~ApV@bCt*9D+nmgC`{jitPghC^GC6sYi zvfC;AXSTxhjai{JEI#`*a|1FG$)N_6GQ3d8Ji)Knf*g@PNRacr!l^b-p9tHMgP~}< z#{~<=rH5OnM(YBli5Q)DUV-QxdP;0S`vA`7njgDmvRE_a)NDW! zUxq5-x)JLq0?^sn_+dD!&=b(OPc1%yB%2CU7dbvl{MA^+ud*x1%CY!oN(i zg@m9m!IrN{*LnX4K;-g+0abDoyzFd-#c_xZC4Qz+I54}5%he#okyMQEQDeWr5E!HR z*Yj8U=1|{snEt*&$jl(gGye~XS~EzdZJ+{4yad+T<5Fz!NF8fn`V9*=v`2qv@p_k_ zc`xc9u7H|I6vImPm#~|qMDEo95anWW;)lLk5f(YIzZVY1IafSe5G6Q_r(}YoOYVTk z!f^e84iXlCc8l>MTmU9rwkUFu6b*ls)!$#@21Ob!y!9|{6seuwC2>qkE+jLk>{cVj zLwYqGhdI#-7t59K=xY2tPp0>0Lbyjlte}bu6(S6frGFw)?IhV8zME)cm-L+>KYY}h z-_o%bRLi#shMw9lgdcaO$=)39yBV3rw9K18LY+fvq9^e~K|J2_!TC|@s#y*^>ixpf zFfUU&oYJ$^4&3z{7FdQUEk>??Ul;6yDF_MzGs+b*h1_`wvX)IlI6ojny2n@8E&l+i zgDwNgZOp{*^csNS5K>K>HydMiCMADl_p1W$=+s%@qxCBKuv>PT;T}r)sUSLylK6b& zSlJgE%Q$w6tXyY&tD_!vq)ep7J`)2bBl+B3M_^YQX#8}ge9%;rNcWpnk1jE3Bs7%_ zE1hkrIoVf`);XL+*f>=RrN%Zn$m%?<^hXhSqp62w>s`#KleI@y6>CM9pdMHDy!cu$ zq|S=A&Mz`sJ%ma`t19}ny5LNk3(bX!fMR@@>5DV=OBt%s$}YG&8S0#BLY>F3YJ$ZX zw@?~Q(&;K@`@FLd7%gQyrI9wR7;03Lpuk?;;4YcZ=*=dduP-`0>gdy4{_5XttW4XC z;xbne+cIl!O~P?`zYXzk!z%5U*5YGMoDEKw$9aT?y5gFY-L9`)x&(v+gD*C&%emo; zc%ufc<6jS)G70j;Sc)nAH)Ph0?nK4NUeoNae9XR0$@3Z06y;kI2bTlB4nUn~S_`%W z9G1`vn1Sz^dv|A?-)5HG#&O!(|sezpcSnYJ=X1lJzY&Yy|9 z!(GuEu>HG99&dw^(X+2|lQAWc2c$$E*e&j~qQnW7F0Wavnh!r>iBlbkPE1{Wwe(aKE0~yo3z{2aD$?8HY7?cZ2<6Vd;FD}xpHx@0X7;7-XzRox-VA+{<3@v zN<^Xbd-e!XYIyxv(TV(l1cff=Wm0nd{IC~hC$DkIFoF{MK|eaNQmk}aZ%ARtg^HKV zGIzncMO(<<#uHN`TBG{j?{r_%#VL@_RIc@Yp1}eQ!irZ8u$J2a^3n&2&&) z(s1%k@4k6@&HlEF@T*yf#_myEi)xnEKi4F7UKK$Sak4a}= zhII~`$?HYM?ppfqtYfc9&()12}C<1fmK;*KtX!{ zLckPFe!7MBZ)a0Wx2*nYn%I;^$YL+7{%TsQGd#)uZ7k7WV8d3$t7VtMAOr7=sehhW z7Gmn|9r6%hWJM~DK`NA8r43y`ZLjJ^T~VWwGf!u=C^MQY0yyFF5nkUXu&ztA%X0{| zN*mpt7)d_+ z?1KTn-=6F{*xOPHo0?bbtcvGYE) z&m)2LtS{ygE=jSP&SRQ;Npx3FUE%6fbkb!t5fWCygaq2$lQ6`fnvtb5RcSaGzo;Qmcg7O_Qb!Ac>y z#+tU$)qA=7fMI!>#I?FlavbSoeBBtfzT-9U%7ZEMIEbra5RKhtq;TSjs7^bz*?+mP$ywiRsbj=OZ-BNz$h0g z;RKmKLcMm&dUAK$idJ}n7^mmRvQoPvM;H2^Dcq^>jI>rSd3_pzQl`zf5KVkXNr_#Qx!vi_;rHm;1-_*>4SWCGzDXXbYY})qi=>#^pg{TW71~E~`hv9n&mNDH z>g;E}OM$AMoD0{6Z*nDrFgWpEtKL^7099)lN_Q%x4OPGqWtJu-1>ovo=mPhBx2G{q zKB9bSyj!ZHoKA@*bg=O3J>o!Xra9+8`=o>JSrIoH`7J>!5@SMkJBNSEfk)Y{&9#mC z0|RhD)Rv@Gm;9M%xhhV2{CeCbUrri7s2B>8{R@;LJ#?_01UH)yqoyWKuBHu~;7cKRN$36%#rOvWk zK2>*tbQ&aHWTEg z(pMcE5=fR=TFYRFw2wbIER-$UqMkqZ>N&zXE|i2K7`y@K%AK#7HaTZoc8PheD2sL+ zw*;K|-OA}M-lHvDX>+QnwaGe>v7ZGyoUSAKzUT5_p}U~q{9I4?#U**}0(vU>_IIH9 zcu!}nb53yak^~gR$gX>d{-EtSqV;ECNF@9*YQh<2_evhx^={Y8?DnytGdN21B{Ika zLDJq-=vlc#ot*aI)DXyDh^E&tq8@9hT-;sl%Tgiol39lH1)8wb^^ zm3&nK&@V4%r;LHCJ{NjGB0NQ!>2<$Xe#~M%IqBn{7YK|3PlZ=j@1gAq7H5C`W9tt zc#e2c)Gz<-A-7i>TarQks0zIz($b4*`Y8I=^^yqNJ(tI$sXq}+?q*%218cN>K5x=y z`szRRmx?b)Ie7*r*IIgU-rqL~DUqhEeSj$@0jLwJyHc~&d1YoLVJ!WI2h1H=NCEI( zEDMF4l10K;l9fyS?tMn+su|n6!Bs;&4U349T6vnloP-$)W7LS=PtYa1LR9J@c`%VwbyFijqVV9Q0te=ha zk!2J2kt`Ft;hoT4*!^XdwhYrYi7XKfQD%%0xh-~x!4<rscb@ z**}&!jAW7x>w5uno-d?KLDt`Ns@CsK!dMqG8Q2N^9QkxDkH>8a%efdy59nL&q=Q5$E!oK-}pwoppLFyT(kcEvy(C<@|ysr zw_P|z=R@X;1H1@0=2BdDG?WQxxD_`T)z$ggU>~OH!}Rr146aCaT@6s}ae~-#<|{Qv zc#I#G5AMN*`y8~Z!vxFO79PvJw{uW)HVI&B&clY!XJd~>gwBc&o-Vqy&eNlELd8ne z6UzuyJp$#71vp{U81|Iu~-Th_7N-w+woqeAa+r_r$9S7$I0g;S-RMWOEaax00* z<+YzTJZyzNosn)9r4&N1YXW;=JpDi0%R-^}XE)|&YAea*kX4;C$oDlDU*9_Q$rP?Gh!|m8Z{yobe!1eQ>Bj5LwfUEgy|D7VCQW)s%$@Uos z^uyFkm+^(~oOOEk!RbF00KxkAy1pSNEr8YC#mJ-RlOoHSBLv>Du*ksrZ=R{&Mn`1y_~Y~mZw zroVjKOTC@Nd&1U^mp?`hr{_GggpI6+oK+o;SE_qXR+|8f8qy%^6HZf6@1TNnrZevWtwq~^spvPAzC@4=)F{pSib5Jo+{uiNt${GLw diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index cbafb066d783eb1ef7b7e451aa1f15a0c505839f..5e2a79d48c7d7727d182c47ff0f8bc92b36a22cf 100644 GIT binary patch delta 8529 zcmV-XA+FxyQQ}ds4gvx?Hj@wntOB@~lfD9Ie@A^abDk}cd3WW~doF2fv1S5-mN93z zdyN*Da{ojure-_KMeN-37n+v{cNG!S{((;R1 z$J9-ZzK9`^qb?LstEq|PI0G%QQw^b-x19^uDH4t7yqvt`&5?B`MRL$wEIwi2!a{`M zIgWP?IE;6VIQ{#{%dASvQf0kKucNOs3cBV1Rnl0&0y!jC!XRmg;-W8=qBzm?f9;0J z&MRbLfy@jO=x*K-lUkgH>!+*# zygi-1`_I+s%`X6vd+buO;?nXSx>#&_Jj4K_z@sY!9LSa(0Es2ObxZii__G+2Uzdmg z6WTWaVq%!s6BuA3aO4aj`reX;6jTNC%x??sp}j^%jNN<~^~Kt6Yui=`e^>D^ws{H# zt8Kq*=056;)XaSj7kn>QKSq zZx28M)+Bmh0gpH7-jZ7qc7uXdO-elMF%Qr=m?LH`xn2=^oA>x14_J^v@XA8a1`gcV z6k4yr+@&k2J$-)_@YXYL<1w`AfCVDRg^a_21YlyJ2dQ%%9F(WFd=p zF)8PDVH?P=i)jEWypZh!qa^qZT;H9@w-jPBz?`}O-6MCyC-<*40uDwd0%sTJ9ci4{ z@bArSH18qYwbzj#e>?zP1PEmyzBqU#3~z?)4aXGTQ*7Omi{Ak>8NSDCBalZdG=dzL z-ea=pf(zm?1T7!B03HK;ON2Eu$F9c!6e0&%93Sif-}stSK7H{S2?&GE!*S0xw?ua5 z7+@w=Gmf;2Yy|z6K78zfwTI?wn~&%#bXQ>QNaM|f<`RMAf3kD(L4U3J;-uD|H}okA zMP!3KK{z2rEYT{Y345cWhB?Hx2k<qkBs(7?|1Ayz{`C zFl?tMXpRZ?mV)+L3-AT2qv27$I=aLj69iH(CMyf3ltn1rdd2o&1h%|g$YIyA3KL_< z*G@6KBs?ySf9KR)N#=nwx@G{{b|4q;2>nVpvkMs!S&ra$Vs8Mvhu9X|IZDJnGujn$ zeqKTM4lys%LX8`ozPB*f+tsLfY{&eLF0EJ3Tn3nZauApcRAV;Hi{SR&;g$pj`v8Muy4mV2z zK_f(nnC4bDzib9R)RVdhJb$DK1yn7PPmRrzZX9THgd2ytL$XaF-xBT95bZ=0B9~ku zvJ-);fZ7DvD!}f9Zq?|wg!?pvd#nk{QLItJ!Zri5F{0hMs8ylf66})@?4+j&D@d%+ zCvuNmaWdU3(W>D#K)7nayC7ce5LyC$8UpU?j?Uw&^v|7;uL^1d#15+hy9?6Qpx+Yi z(-7`JlYR@dJ7sIBrjGZP6>^kycYm&ia|;rz;D4JYV=zc*&h};&XchV6@T8a z#&It%_9aFLO-B!QzzwlU`S6AMOrk-* zWs&`1kwIQu&lIr+J{W4M?jN;;gYvw;fRozs5nRNj?Sx#joE;qg!t4keGt z&mb|8eGVDCaUt>M3aTl??mA#PZhyBRgUoN!4H{pZi-0KyA-6039u}!c-DkIz2RXwj zu6e6P$`p?{Pl&1;!T7_8YkD~+W)YNmg8MW=UCPuC}ua*<8_ z!TIo4O8@xq_ru@6{`+6|=$HRv&WG`-=YIOfTl0^HU*5i-4u0|8(GNEd?tjP2`(OW$ z&1*+MJ<}SqKb-6vX@amMi+(`t)Vte4-bn!J?=ka&h}Ib&PpS5r>6MjzrIhqG#gIK4KD!JL>m_gO2eLK4hQC z;@82|03G87uaL*!%Hb{gy?=hkc=zC7j+Fm%ivIeI8YiQ{W5?iQfO6)(8M3nVpks(w z;}hdw#P0oj`gUoOe=O1e{`|A?Sn(nhf2dW8N%*LiiO)&SMyVc&IW5_vULcoP$jY%> za%IXO7F*>q%~KRs4KZc#R-AAuEU+Z4T7_LZM&;7$xv^L;SxbyHF@GMR*bEPzwpJH$ zNE*EnR><-tV_9Vp2i>gJNFlu&^`RJcR%fUimAeZ|hGf3ous|t%BA%$<9jS>cxLF~> z#0=a+n~U|#!wX`%2zp4!{Av4PXVm5ywRuKQDw#&To+fTW^5`Y@SlkCy?~qFaOJC_Q zYIL@QgrMAHBn|X}f`^$;R0=59k7!2``soceusf;agRe;4e ziq-9?^Y|(&#&by*wGq3d3nffHucr>esvT5Ckb5sJ^J>^!kQrSQ>+JMexk7mx(~pQj3x6_r1=&*FAiO?VsqVG?TKlQB zpPq;P6i{L<8F?kI?5WlF8qF(Ge-N&_bK1FvUqwi8g&vf$>SzTHhfq>$oCt|9hiW$i zS2aN6H;j~anE1ZSi(=udx~;zWJ1WOxoRB<+rW}v@Df|KK>7gd%Ke7=UNIBL4(^=E$ zv;08H@_$Z()E3vM|z1C(A-?Plu?svOyfM`^-5mwSkKsc`q-823`22USH;bhkbN}|VybL$ zE)iLMnuWtlTCP$$w`q&3PcQ#lS1B*fsoGb1S{=In-2K z@LKi|iZWmP`JZhE~F@z1}FR9oX@+zbC# zF@L^uEB*08{erFEs`Xnv2ftM*b0*2?r6yk?N4^>IqFb3qi4GWEp&T1%FCw5vy~Es8 zx^SMFQJ`69N)vD>K>iO*U&eShCG zikwgNwwHMZ$#cXqYkwpOZf7>1DDcsM-8WJWAjJkd9K~?Zz#tA^>>yedayjbva?{tR zja9gq7FM_rN<~%2p^PFZE^$#qBcu@Cxk1?$vDo(_7Olv+9QFIUymU-3S-j<|UrWw} z-8JQNqz!X}ohf3u=c*x~W{Ty+@qgM)g`D`vvQj1Wwv0GA!&Z1;D#BaCU%w!{STf_J z_ylyH&N%u5O$Mv5aJ&zQon4$OPpr#OZW>6Ti{;f%j_AUui@JWNJc>?#pb@%OVBNZ0 zel(Yhf)KL;NZF>87?OwRO$^#UTo=OZlA0o8$roxB&7Art`{|F@3kRu~Zhufe8sQq* zP*R)_;28K;8-^C#1@LU{RHMUD)s%E(xv~mD+FsC`dkjF4tREmsLF=8BL zuVd2h^?Ds6s4j3^(O8a(p%j{>o8fq*33$F1)OBjf^YvPclYa!E^DOX98J{{zLpg8v&8*->uN!i{T*7>r`RUBQP739vuz7%NO-ztUU9X`L+H z-O7;dFD%H5r%EtUS$_hFT}RZkU;}Qhr?{xb@*g z2&0nxo>6MoKa>c+*^C&E525&2q7~$=Ab%o3{xpSJT3>vtOUmeyp(eR15TZ7M9+KIb zC>u^xMWZAj?SFn>kosU#N+mstc7M>Sm1m(=TAAL;bYG^AG}%#7hD{N)&**A}Sys@L zm*+7km1Mbp$6VL%`}>;J-!a5+Xhysz2i5KxTJipD#Cw)+wu|7dQu8NM;iqnhbh{^M z^;Ze~b*M?&N(<>Mm(VIlg*K<@|f@NB*)oQK%)LLUr zmSZHJuUyp3#w!9;Q+J`JwqFsb=G0hXbgd?9HQ6)NWT@STwHj^THQJFTbyy*)x7~_G zfNE(pDT&UoZwwF~^x+gnBm#-QvzmLvADX7q9r`!h_>0TTD2Gd4 zsNxx4o{#tGd2-yar^$5o=k|0)-RuPqWtc4vpscNwV-m%|ZEFpsP}_N#=++2oji6S; zOL;TUWa9cfIW?EaI>&Iajm!yw=;V&P82c6G0)g}8x5qpspN85XSl?^Z~Fyq9S_xgk3$mkeXF1^PVx6`6$mvDycw4bMA zOc`{UyoC2e+?Lxh-VrPLz!zh|9ghA>c~PbRr88>VkRg|2C0~d~n;`K@}s_ zautb^Fvyb^RUtdb94!q>EA| zkoN3o;82rL9p|wtL@ap(vz?Uk;ppU4**jGfAakWt?aNivxEIT$0IA?m4gwo{4A41n z{5QUA3d9kke1FcR3lQgX2oUtZA}JNfp$246?zJtM*SnQ5Pj*|&o`3P|Sd&C57w!XM z?;advGA^W{eZF>~Pa1W(R|8Sa#|*DfH)vAK#v0f;44wEu4Sv>-CXcgqcrFl=r|!sq z3pjqls(^v>j#1&f-0mN*T6#1_P`9y`46rv z`JUekj=@@Hdfv=5(InVN68Q+-yZ6-nr!tUEpa&-9&2LS<b`$XI z`FW@fnHEVejr(*p5Oo}%bRQM5y9RhO{IAEnNFeCY@-%kj!_8yM7vo@Rx$|-~8s)}W zPn&TKPxD>F_4u&BH(Z1H;y9MzxCjC7IkZRehWp|O7O`IR>2R-^*qR2LPVkCv3(113J4OMFc9N9%4xu@AJ@&f8dy`uN*Nm

&d;iea}#Adf;&L4|8k8+cE9i}Ri+|Z?vl(#?6NrP(UJSPoxm*+W(5tI@G#pgMJ zrzm3e_C%*W(J9Tt$CF4NKz{;WIPB$yL(IJ(W{=P#kpgrRM?CHT;j za7}Df`W>m@pqRJEfiA=5+JU!B)9A)cCsIp*-BM@IQZx3>V zl{~H$)?0D6gizK$EUHrZ<>CA#CUaVXRyFuM97TTB?hlAbkFmst&H)4q zd>he-XR)@9Ep6N&c}!|NHaLhOE}&h3zmhQTXcyn|D%3{!^4*@*k%v9Ziv} zos!ok4006_-w}d|2iJ(XLOhQK9pi=uX+i#MUqX+)bKPt?sDDcuLXfx7h@nn8#;M8h zy*$Ma>oh$$*iAm-J`}& zg7E8^KfhvPsedyvw8ieZI9MT%!Ie{PZ}C1+9%W)u+Ay7t4ovnv8**$vpb`Ws zqrI9m-8XN`C^wbFoBG?$ZQLyqgx;ST^Y(wiq(o8g0AztbH3heplO@Dm?_KeK0Zir; z(0OA=i;5vAJ+0X`r7A%C?tqj2pjY5HZGd(6jcX2*Xd$659Ooved7m6>ySx}5LZwsB zL@ZOG;ur9n()j_hPjV&k3qQ218Z8yt{w>39!LUb@I3h8BPjs1JJ^?TW&6+7-aR_;v zTI2;^li@bb0L;BL!*=(*2z@C*_2L^%(R_e#zCsEx_aine^w3yjO?FZ$Y($Nt@w|UL znamE*eAqh}Sx4w#HbmyZF&qz$=A#)rgcGA0=xER}KEj9KDrNEe43cU5 zo-RoEYgFrh2CER3%3&KpR=xH8`iqsXLSZ*`q(7$9e=68m#`Olr@uuWJGE_-XCrifS0>T8r%R-jYjBQC8@fpH z^ERp_0jOZ(b8(*1C*r{{Py3Ag6osNqQ38#I_D1QUrH|-#t~A7ym6J}T_Cd7^K9o}n zC2`dr9#*r!pDe?^L5=f*^ndoO>-*)9{AhcEabK7D?F%NCIt@tf5K6;hyM(k3Ki!n5 zj*sO@4f|GqMz=EhNo8~^ATLMbfi6#8ii7u2NwxO)LIMxhBqg)hLhjh){zc7E$hJgvyS9F33jlLIC!e`B+L)XUEZx^Xw# z?$><~yWXRnP3b=M!4t=HJTsN8h{#2e_{acO~| z{pP;+hVzst{r!g9AJ=R02gJ-c>G>tujT2+g8}ttP{e%ACrgwZY7@YKu{%v%OU#Q3Y zXvN^EYEof87=2WNkr)4j5h`RY`lvBsFd$*|**GP?I_$W#g1VwvB!h{mUr0v``iu zixr$`R$9X;KT<=(1*3Wb6{keGK*j0$G^gUe8NnqvX(iDyE*6Bk=o)!+?V2&U5PR~? z_=r|l%Z-OkXd5SkqtWE(=(smN9*sKu)$}WLjFbLge=->z_xh8`q$eIn?y^CDbTpa_ z$D>KdIJ^4f87JdW$M_)r`fR`tSD!$CI6N}GJc_$(;LMI#^CMb)@{sir6QKlypYTEa z{NSvmpUubmO#C)ZdXELluS!I$53|$-Qvvd-_}CCxW$^X|%}_gvD}V$B2uEo07b z_ZlrQ#}8Robjq$=O6MKm48Z3vo&@VJ@ipSlhk%Rs*WlF(I-jHUmrMA?V=gASHc$es z9A?e|Ux~BR)4+$#hsTGqTmDv0?ar8_K5Imy7-YVxAc9hoikn+sI)Os|T%eJY=F`P_ zf1wIbb`W&0oI+o-`@bJM{w$Vh`=~G!#0uVsrxgIivtwq>>k{lU;UF-%DQhXH5`e1VIg1#D{G85}_eGS2l^3F7Gf6f;fz z5AL0fchu9U5xR2L3h8Bqd&?%A6e_v*6Y%@&!#l0YNDl`aRZ!Y+*A+hL+NXsv3 z9aA?s`XYuvj=E4lt)?cD;|#RKPBnyT-gYirr${uS^K$Z%H%Hc)6v;tzvG|063kwm3 z=Q!Rq;4t1b;`Hw)FS9BwOO^E|y^g-lDCn94R7qn63*?Yo34^2|ii^HfisD4mf43VZ zJFk$11u`>Cpu2fTOlooZIx%L@L*r3w`<^G;>(bFP759_AM%=pyGV~UjcWbAfxQ_wQ ziF@A&moi%^vSUEhBD;cIP9mh=l_x%GiXR>z)Iwk;pHX5V7qBG)o`Xy-wJgA=AMi=+ zf}16}CGsc)_=MpBwuDS{5zBJ0f8tZcIoe$lg2@7W#cavn-;&GUZ$3@`^Y--Y*H2gf zd3!p2_n)iNn_mDT_t>Rm#iivvbg|gz443@nWj7C*0!w>f3D(TZ1WTf zR@;8z%zZpiGxs@M@VyxNa8`@8qweZcjL%Q*r_g~C<0+O5ic#}`ep?dcf5 zqK)$TbZwg45>j~a_g|kLfAxm>EO$XL!_dZm%T?Tp|5C7OYgrAvZ56H_O}*~bp@PHT z9)JX_N%X)19&ggUCATE(1_i5{lz7-<9-wnDN6cJuy(08B@9{q#upoorm4%=U9JsM5 zv|fX`OIK2R`u-~5t!LiGV`$X@3q+6$8HWQ2z{EliQs+9z<(~>Ze<0r65=Oxsn%Kq+ za*~4#xxhut#poW|p6L08x?F`j@-UA78Bj6s=-M@rCq{O(;`DMyAqfcehJLKg30 zQqJqbHjrNz(*Ra@A=?K=N$?xEzB`d`Da2%eIduWLNA8AC?q6#J9E?l^&MwY7(m1i< z-<#WL-b1)+uOmTte*n4&5XwM&aqvnQ-VE6rjw!sS*t#VbzXND8e2>{iAdgsR1UW9f z$7In37sO)-T0V3EJO=od2y14JU5^1ML=LhzKG*}k@inJ>`rLI5_}Bw$56#y$AJJFnuE5%n#+wPvB?8H1f9K?b{#x_JNv%C^=u;Gm z$Od_Wa6*b$qE$!}_C`ewbBJvZ;CV#b97VYjKrqmhWAYpN3OxQt_m*5RFte$7=Ych0 z*iKK-924v<1?{yK;0snqgTs7vbcsDC2&7(2Ru)Vti%`1titWJ&Y(l&CdQDj zonm-Ncw8FKf2q5Y%mZh1%>cCRKrY@9`jv2I7cwHU9Kr9z-T-(Hu`RZ9l!$$1v@7KN zyn^l>VqT<$8aFt7Z(**v`6M328pb9UXC=MzduC}4$N8*%jZAuv-0jWGIH0X<%s8;7 z49paQ#nxp43CI{)In3ky>B<)X3YD5LE$gXVLD3wJA4cu^AKXL=7hKF^PX!hoZk7as zMu-qG&8=>J*$h0=le-8!f2ausR4tNEjm?s79B6Zd8;80>vP~i167ACv?N}2cms}#U z6M?IM+637u!0v=@)#$f``!s|*(FElv)~I1&n}OLF(e7N-s!(qU_Gt+ASQA!|SfNkk z9=YOVx>=%C!)<_Y)qrvi%iLoLh}lw z%rSp`_SmWVP@&$k#{RIzabE08j1ZcRCMX2m5Q|g^DyYvU+99;8vNx=9+{=r)i4jAy zaSH{28)B34;S2SdM1y|IBKyN4{k*uIDPj$JnpM#Bh0g-Va4~;Jlyo>(X9E?sJE#5D zsk|$va?9_R!{dP_97-OOpFv_G`y4WO<3i%i6;xA(-F3ip+-^Yznct=xG`=_&0aFe_ zZdd$0EK-rW&u%LZa)wh}^Hz(LDIRg25LGvV@rRYKzyjYR5YJI?}xvC{rA7_(J%kUoDbtu&;9g|x8@%Yzr1}v z9sJ_GqaSV_+>e*{zy2Sa*N%XCrZr}NIN3MU1Yt=Q{eal1cejPSlK|Am=Rf(ib7b*? z?i4RbXjY$kcX@~Eao}o6MzCGw_&kN}P7cir#+#Yt;`V=K>KOBuA`T0i9EqM0MbE^a zeZ(Xdchv6<2OZ-he8@hN#jk^_0XoJFULlXcmBU-~d;N~_?!mzvDgWmb{q-9)PDX>r zj={$O<;;CEWM%6?#}Ki`C&s^s-TU|S?b0OwSfc;^`Df#?;zcO_P^%P^@KG%jpOc)8 zQauuLTC#sfy+AIpkdhAOGpUHO-9l{FDQ7J`9!6Fa{Y*Q6j81~SXppT)oQmnOmCrA zFOz@g!jr{@S)yWL?83?mZ;FVYyOVwU5qTb5s7e=hjNjQ3xmO6OOCe4fT%h8sDN}c~ zgq_M=Y?;c~5?2LSe4|+1jyjL8vSK`!bWt0zOS(|P^z(Y^AgtP8{jXD27s=0lGGqN~+!a{H1?Uucrx9ipkpAVHPPE+hxC|fbAj&Hj8hY z3EO7EDtfk=ux%#n_TJY$(KNqr+%>1aMw zQv+p3wD|^2(lCeXr5HBJ~I1x;v+xYxq@!1Xt)mDXWfF z;BW{fwZ@5%2y>`*GjLS{G=9TKX@`G_@4LJx7S5{M>Wjalay-Tf$#ZDR@u;7|AHbd- zXhQxY8?k|uV;wM^HJv`o52P&bBuK8DU0o*e*f*;SzgW}2eww1-CGb~R+GYCe*5(dm z!TAf21s&sT?Yfm?G<9@6k;$YO)Ppn1J zO^{iTIW|rTZVh`Th5Jq{nG}DPnVHVMFB9SEOK)zfg!|Ad{X8Auqa>d6ejY z;T6iUf%YN-dej@`rqYG;)QkenLR0F|f}m#wm}VlM)%fj1SN>q}tfPP4p{5G5ocU|y zaVBgiUtIz1NfOo--))94)dMPZ392CVsDy+|!fQ&rWw6|Nb)Gig=sV^cwL;)>)H~AD)|Nam-L{CuR9D18Fg%|} zulp7+E(jCGPU^uBz!`t;o-H9JVz1W$&PY7Ef|iA>aIf0|&Y|jgp(L5+l)J?v{aSJ+?5-)FBW;))>`W2MJy#6@HB&4nj@ND~ER~ z)b%^%QFQtPjnIF!0_)c0@}s$26oi-+K*~0y#E?8hZ(`8?;kpoJm(&y)OTJL6Xy(*E z*-wAGUN}g_bc6cQ2-nCiA7WUcM8lq_gn=d%HI$1{u<2GxAp}!REri`lDTc6hv#KGE zd#fCNWaXe|23@OC+uoRPYeq}NVg;Wv#;=VL-`n`8{!o9DV{jC2-qyV*7ppv1CLC$P`y*<84$g2lFl=)9E(R%w=i=b1v0V~VHOt+gl>?xIIn4ovJch}l zQe>ox5#xU-E6-|UFU#26ejSs3uh;7sL3M%SipFwO45iR4EnmQ3-wemGCgAy6P}iv? z&(~`)P7;L9v%oiHcn(d}J%hIG$25!M{m8lC48c#ycJCyUX%B6a-`@7LHZzjqnC1xG zKd5_$TnVDF5ajf%mE@~wn5nAaFqsZ+^p52J4-|j@3jS|YWJkG43pcJMVlayNb_E|U zB*6Z-W2`WV{Yq~Yr**P)cPm4-zpx-Lo+`maWeFs99Z}PckE%!gwVldk9oG%adNw{x zC8)|CMNkVFC%sr1g=D4rs)q;yT&b&(v}GIGAcQS+>yajYUh0DPh>7^Yx{6iwucTFL z=}>?1nPwDdfbo;u)X{3lRzntP$W~Ujvf7u`6HS_fFS#&*HvW4XH-ZY#6G-pG_Zkx8 z=`pnOyp`vzJa6UsbCT!Bnw$eE!amY_knDjFpLY4`hr)CZeVD(O+Q`-4`kJPWnb%Jf#I`!ao~ z$&QjTY>J?LMprA$vVx|(JdZ)CB+LCf=DL30-`A}Ejv8|w|cvT-ag8$UEr7A-ZDZk24(lL9I=lzqnDG|w=o>8 zLVgwsxs~Cq4EJUDM3*DKCJe98bUpJno?x9?GCU1MUBF592^;0raLPc;Ddc~Y0$Zt; zlL~XS+|FVop?Ts`OfP!OmXHDHB7g{-+5GF60Jg*)z{F#an5esTA*j@lGmDmeU(EEM z4SBtRAjGif4RzKHjHi%5leKHfOBEFdl&0E_}i0Cf-no&eSb7jrx^A4R`g0Q&19s+{C`aQr`Y};UlfaxMmgB?I!X~&z;w*9v@HSYl5 z8Q5S&U35!=ubGg<`c{9l`12%&a>;jiC+A?alUwoQ^x1@r+1}xuV34wRR>)h*&#)ET zsTN%Ra5x$cf%ofbX2gfnEP{X89G%Am{SCA=r%w%m^Kj#$YDz8DMcaP)s)%8M%fFP%}_h77qJ zEAc`im3r*o+A;OjtvF37{QhId_!S{1IzABuQT@h`vX^$GSG9^C%wLJB_giKst15(O z&Pv@&B55JxgWKi~su-!3yAU)W(W*mLkz7?6ajGkI;fj(2s(v-eyy5Hz)}HdJmNdUq zpvG!6QEf_8s3w28G&{UbLZCj)A*jkWj$1JqJ&}4@#ouroUE;%qbwv?X{r;ay>$E4< z`B0O9?gu4^$eH){+N)O*HUmX>J&4#iO3##N{1!5Ed5v;?-WlMz7l!D*P{Zs_jz_ms zP9uj`D`a8Fki7vOB9}!lRL18;bP&U1*d-(aVLUo+Px*$z8jqzPyo-P z&d?+O!#HTRwZB|HU5YDBa+=nN%&a1h#Q~j4>%NEBMwVOy(bR#AFX2_9fA^=nW3!kb z{^%{L>+jgtn{{6%U6d+;v|mR9!;vPTI?iKPh*gEXmmUYx6a$3-cgr<#ybD+*!YJeq!4@(e1rYSr-W>S;XrD`N$ybP( zxED{juYrZ2?SUDZ^B-JU@;$#79D}vY^t_qrNRwbAN#rAR@7`1Qle2K4RWDXo1t{-q zO16KOE0P<6k_EecD2fu*ork2PhrKg8KCVl>ZM4MYXgJZtw9BOnbF8-Y6eR%_JBz~h z<^4s;Pn2{SWxi8GJ7va}r=L<-s(eRb6Sezj#xeDMM`6?N;6BPsU9Ja7)%trmky@|M zcNICunh@+1gKY2kNrP&}_mhUYOZSgr1f_q3OR*Cu5;6sTAR%sAA5iN9ia2=G(}r~O zg}X*8dcTcbEdexUPfKvk*wGa+83Gn1zC9qna}#vx0`W=C+mKj*&8!|1DUK;o5Y9Q)3KIuLxVs{PjX82!^d67WSq2=w! zhnvTiFUG;ta_8k}G{}v!o;Kqep60uT>+xZMZ@32Y#c?dbaS;OEb8rQ>Jh?ACIm(mS z986R2FgJiG+#7AdgCUH@7%+s`4EMzmEMmRr)8Sq-u{8}go!}MU7Lo;5cZ>oU?IbDj z9YS;LdhB(F_a?Ult{GiHhB+T^!c8w`iOp`yoIe(Gj&qZE9i}QrxuHuTDQ|zClLpny zc}^PYF3)omBPb;XiqCTbPf^6`?TJo%qEniOCzD7XKz|}$IL-};5_EAvrot1kEl&h+ zbeqf1UqYV=L)&mm@S!K+n%JoHJ5s?xF>j9pU53rI4c*Yo)6jM?F^M&A6-e7_HA%i@ z`De-UhrQg8B<~u{w_I)}RXd~+y%}X#f2bsgnYPfY6+Q)tCSYZ|E ztFg??w12i)nVj?ENw3$7WuIT1i!yid4D`|D@MzrY7(w|E@%vE_c}{w17zMb~)b+Bp zILg+OP-VduM9%G3OodXCK_&zK)1!fy?asC6jdVh1PJAJ>hb`ddX3U^ef?i5L)V^RHt z@ehdNGQyp6Mpvuzpzag@aUJ}SHxr&1{~~tp-_y5Cll)_e{`cpf4Oy+l3)^92qVU%Z zHt(d6{HG|rrqR(t`ZizJwlo=epT) zP=A*+gdlID5ksAHj8l{0dwGf<)@gchu$z3u$NvYyizV~RK*dYmBwkz)%rLaCY-EY@ zdb0tFyiE>TGdB5LX6Oo0IzVU4NrkMuH|SuFAY8ji_&i zIPT_u%rJrO<{dGqI6FKsX3#_9QS8%E@6eDdx4RB)wjFH7ecQAOk(Z}%-{O6uJY~eB zv|&0O9hmHUHssiTKqUxNMte1Bx^Lc=L2fFEH}$uh+qhdK2)#cw=I#H2Nr|G~0muS> zY6@;GCrgOC-n-)e0+`Gxp!3F#78OHKdRnt>N>zaN-2o^4L9f7Z+5qeB8`lhzXd$65 zJj_i{^FBG&c6l*Agi5ELiCCsW#V_DDrSk)1pX5s77k+42HCigP{ac3Jf?>y#I3h8B zAL%l|d;(w$nl)3v;t=vSwa5#;Cc|x<0hoJhhVAZq5&BYs>cuyjqWJ*fe1#NX?ni7` z=%KO5n(U-h*oYcO<9YvhGMOEq`LK5|vX0QfY>3Q*V>lih%||nM2q#9z_;h_4hIIt~ z(b1q|e1s3dRm$S`86?yAJzbFS*QnNi4OSs6mBTiIta|JF_cN!{F>K3hdgZ8N&gu-r z^s!0gJN_=a+yp0-y%--l(8O#bf`U%(A-3U+V}9H*-cOUiuaSq`dydPom`w5CPo8*C zHC!bETq;^6M6yil1m30Y=;vc8(=eKWQuO1MYuR2CyoFAh=&}|ir@uuWJGE_-XCrif zStil3E|FHQ!8tN*=pxC_+o+ZVpn{Fh#d%7fhzG+w?KARI6pA)Q2{ano8>NSqKBC*X z(hyTtPCAv^2h}e4P);qB#8rEESj_@|vJCqMHO>pt|Jk#y@0UaJqwNXCJzeIvFPL2F zG$6S{C=HA464E;SbW@@_K9(mn>{}Us-OA`EmC>z$yc~`Dx;%L)4&Gy8Z7(1rD8;AC zzKqPVWQSQlN?_c^+#CLpXBGcRgk~uRO8N_R+$f2ec~Y~um1N~3#h$w)D+~~Oao(oq zWHjqJKF7nPX!_FpOjjxi}75|oY9-cq8fDZr&`TSj=4k{jAxgQ(Y)GOJGpJE_q1WVtPx+|xZ_?~yK{ z%_r<_Ls@lktQiKEvbPLp>myn#6|C)t%GTj&AM+<*>M+yxw$3GkOnMTU32pBfipr4{b%m}9Uw~^HJHKV@{MMCtT9JJ@8c*_* z1STwhWV3#ppAmH9ZnoX8`yh6`Wq&oJw2U*`M@KCH<+@ZCdC?f#eSEu*Z};($67*wI zzDJ`B{c+TVB&lsD%Kd zd@S+}1z$x|4_sfJHCP&;Et2u{ywKMqA4p<<_y?WUyQC)Su0JkYud&tr^eH^sl`=N$ znP*Bw;0w;q3cgWkca<2^&JNNFFjk(c756D)ouK)|IzEPjgYjg1crfZujt?gDIXpO? zqj_&W>5UOWv7(TE$hEFKWWU$*V_b}*ac|O}!QR2RKeG--hw$)VX7(lraNawd_szk7 z7#&(sx$6RmH_*G|(gH#I&3*3;=P6P8`wh20uGi!bh?#NH^GmWDC&r*R=pFR?2mQfK z@AzaeIO!ez+vpg-P>=c3zN8*Ac5<#CeMI3*t6Tw7c4)R3^$I>*NPzuu$5>$!`<0Fx ziq_}Zn5(&E;zMdEUu*q&4lX3-Z^oQ|x+}=Z+JX!nFuX#llJ=5GYTQP$XS?g8fhKiC z%El>!Y#aS1`j=5~X`w7S7ArW>th9zxex!zm3r6(>Do%-Vfr``hX->s`GlENU(n_LZ zTr3E6(KYhu+BIWxA@<~(@e!@AmKzV7&^As6N2AHn(Q$8lJQ{WQtLazh7$^OI!DKQz z?)4{=Nl!eE++~CQ=x8(fQ4Ptze-D_cTgA*4nw{NW_DZi-cfUsAtO5KxURr(W@Ju+0Jkj4znCXO~L*Zpfi z5yIKz<|R?1xVW1tes71v{)sza2x>!CH-%`Y<4_IG*0}Wz;?{pMz26+DTLYrY1Jk^P zR?<2wfw+_naf&IaL|C#On3?kr)%f!G1FXzcjWlbdxob%C=(GeLP)1z(6XLlr#0eQ< z)AZN0{ED!laoErFys`K{FW(dh)Bs>BjC30Sz?>ehmasAy+Ao8F1XXQp!|c+E1G8fK ztB##@?A*23`L2Ie96K-AfEz|kO;VhnnSU6fYc7g_`hKOjmo#S-`(jQvlD;WoZh=$C zRI1__{>7!5b)|tndLV9~ee=bTaRFB#c_8d*`q{#%f@k~*yvxy8Qk%HA*KJ$6;QEpH zvx_T`ARe@i>VmV&uQK}UL-?0=?h`|Ak}&){HKusQGAV!Hc~)ndxth#LV9;~KeHD-J zA+9;|c{xVseN<*3t8|eW+lmCy%+zkGdvbDE?4~~P2e^(filMhkQLZN&Xo&Jt{cj55 zXGN^>{Z;{qZNT@hC&5=k?;3jVT1-1_7ss?qqDIm%nuRpQ;5%BxgKWg1Kv!K9n8KzO zgruVOjKY7?l-G^G?;`F>@*25Hn@w(<5q1qRTFgOQeN(ekd^hq6IZXPI+r$ldFS`2; z=x|ginX<+d$(GvMX{%FWYl7P#0v6Z1VSdyfl8Ojrw&}6R zq-R3srYC);!h^F)0ed5x?Usk&-7vGmZj+@2FavELm6O;7Re#QLv2p2D`5E^-s`A>i z*uqkjW}q}uO5$FH>n8?M&W^@~HbZPhA0Stw8y~C-kJm&GQT?O2F3zYU4MA>7j7`2Y;Io4v-KzA!5>NWt~ymjp0QMpn(f7P<)H#*OGtc|Z)!k*6JjA=Y5p+ja0T{yrL z9niEUCkfO4!+#nkHF8HwoR;?M6}{Fh&C+|uP37j&pEg7P@)zk_RFkxtuK`11Hxd03G1BGUr>sh?lDz#%?TY*Dm_*TJs zxBT1LEq_{asQDVyYA?x52+Lq%pL2J?Zoi~~m+tt>u(Dv+U(%sR#Jxmn1XHUZ5rH$gk6NvhP6#u9POA?~jwk*_37uJI@Ei{ zSgqY}Lc2R9=XR!ZV){CV8C~%b*dzY;{KVhNkotMDXkk2SdBM_nN%Ka33+A*W#fC7g zfevnhR;vRjdU3&v7s5-V$j!*|`c)EcQ@heyIyJ4{bGYTzCrS^>WhUR$_-$!|YNu02 zN3Cwjv7Bi8?keA`N>Q%*YOm>sZ2mb`aKRjSTJq_t!tEMI$fi!?j5>x-TvGgR>TyeU zi0bn=^>dd>uL;YU#Rk8B@6>H|PS){>r=9&$)sF3-$sd4;z31|oDct0eZR?CWNa*e* zFr`J2;0_L=I(^E9QIuc`5yufAB{+!AT5F&Sr3YZZg$EAmj}bxtiN3*TA%f1h12)=w z98r6OD08S5QTO2muA;WSd*BoM9-LV(DqM{a<{*MT=~^QqM~$q1;;iSeP5q&W`YC1o zlrn;vMK z-BN`-F4=bejZpRPfPykq$r)XtQkEvaW@InTeWm*^R==dHzRT@;sCTD$GP#}nF8~1l{}9Ey)meG~058psBme*a delta 1800 zcmV+j2lx1v6_^#U`~rWpkF-pAAo2$zeHR=<91>F%z>s~k*XP__gbrfjVsH?Rm;n|@ zSSb$u1~I;5&M z2;ppU^OC4hT-;3+zqi9-|HK_I1hpZnn?khHai|7oYux$-fs@ntpU;Hfoa}C zD`_2;KwQd(IK`AyA}m=C%*^?RYJ7S80aj+JMw&I!+%=@xeOCexC?hWY3GrMQ;)D#b zY5Hqgenr^OIP7P7-dOyfmv0ILY5=emM!F3EU`~%$OIR5U?U%tof~q#QVRmW7fmt#A zRmaXccJ5m2JUV|Vj-3~5zzrj&CMnL(%s&j#H5WxdeZNxNOPVu^eKDsSN#7JPx4!885@-sNa4sZCtm>$WXjaQ#U9 z*~Jw|5D!{Mb-~%?R~h~FA^b}__lco5Nf>^f8dJPtnG}EUJgYO!TutUAFz7krzKTcq z5Z9dfyd0zRJ}NViRl3NGZAF4;W@G(`ES{x=2j zvm(~`eyf1QHsJf$li;hNcMZLFEvB8cieuU(Q6p&>%|e=D@EtAUK{n!0psOwlOkqbTc9W$AFatg8l#|#6ReyH3*tqnn{ET}ZRe9}M zY+)%%Gf)~SC2_C9^%DarXGh~gn<2KM50I=I0E*xNr z4rp4FlZ5I2VSf#i8o8q-PD}gsie77$X6ZfSrgoTQFmCsM5*tY8Q97gJafft{Rlg(e z!SjFcr@#GR%su}{-y6Vq(CH3nCm0_G0d(6B-DnN4&h`vlWzxCXe5QxNgmRrAt6q@S z3)0t&nL8ziI1@aV58qWWbK891jV_I04I)}S$wz&<^Hm}655AM}* zu8woN7U#MpC(N}&zZ9@~dMF`YONMA}HMK$nOc>f^5X^|3PWRNYa4YP08)D(kr+Mns zM~xzC6!C^o#8JsX^h`utWx<>fueN4R!trX$=A{2$0vhVY^VXT_HUNe~ZGL^2p^kiY z@dKT}sO6{1}R^U(>zE!Z^ zE&q0Q%fBl*)O-zUwU=Zjgk>^FX_-D;$9*(f~i#yi6FYy zoLWZ%edY?O5z#+XA_+=SHoRdT86!pR0TDESMXDHox_Z-cnRzg|9gJoZ)HgRJXy3bp0&JSX}qL=d7}k$T9RTz znASiCH$kh_0TjKs;Kd8!B~s*OWO@B63Ad?TX)T?aR_{67^6C?%2jw!8Z)*IuG(ok~ zsiUJ-r{q{pw0(D#Z&sx!SADhD^g}lP94okB4m>USbXDPYjU!}Jr*TFd!zV5&{x|iw zB|Aj*d7S#WOQqL@<;-G#gWq@RwmK*4_{7uBeyM85_Rr)Gz{K8j`OFk=%JMja${ z_Y#=WqDXKD2T`3qWy2^+FolTY2#^vSL}#rv(1p?iFyO)i2ldB@p#MbQV6+fHXWRiB z?LCgDJwlW@REwzl@B&v+Ti-qKiG2^wtQQroMhJ5dL7#N35s{;RMpkjwbJ(W-P(=Nd zvVKY#LCtQ-X_nZQ4Pj*yyLs7JI*lC}v z97nfQA&*M7oqr=#{X3wb3{`SQSE!Vw$*&pNOLJf8{)^Qw>8kH?`(C1eDe59M0s)8! qKrDJnCy@vk3!m*9y-;J{Tr%qT9iB{XC;tlo0RR8R&iLF}dH?`hK5zd3 diff --git a/cli/client_retr.go b/cli/client_retr.go index b8119d816..15e823b39 100644 --- a/cli/client_retr.go +++ b/cli/client_retr.go @@ -403,7 +403,7 @@ var clientRetrieveCatCmd = &cli.Command{ func pathToSel(psel string, sub builder.SelectorSpec) (lapi.Selector, error) { rs, err := textselector.SelectorSpecFromPath(textselector.Expression(psel), sub) if err != nil { - return "", xerrors.Errorf("failed to parse path-selector '%s': %w", err) + return "", xerrors.Errorf("failed to parse path-selector: %w", err) } var b bytes.Buffer From 9ea229ed5a61b69aa75a17eecd0244f9ec979657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 22 Nov 2021 13:04:12 +0100 Subject: [PATCH 25/33] retrieval: fix defult ask --- node/modules/storageminer.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 1a2dfc19f..f4d00606f 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -32,6 +32,7 @@ import ( "github.com/filecoin-project/go-jsonrpc/auth" "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/go-storedcounter" "github.com/ipfs/go-cid" @@ -683,6 +684,9 @@ func RetrievalProvider( dagStore *dagstore.Wrapper, ) (retrievalmarket.RetrievalProvider, error) { opt := retrievalimpl.DealDeciderOpt(retrievalimpl.DealDecider(userFilter)) + + retrievalmarket.DefaultPricePerByte = big.Zero() // todo: for whatever reason this is a global var in markets + return retrievalimpl.NewProvider( address.Address(maddr), adapter, From c6ac582c9997f39008783d4abab12a414d82c4cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 22 Nov 2021 13:18:56 +0100 Subject: [PATCH 26/33] retrieval: update cli docs --- documentation/en/cli-lotus.md | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index d279cd00d..f1f5692f9 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -551,7 +551,7 @@ OPTIONS: --datamodel-path-selector value a rudimentary (DM-level-only) text-path selector, allowing for sub-selection within a deal --from value address to send transactions from --miner value miner address for retrieval, if not present it'll use local discovery - --maxPrice value maximum price the client is willing to consider (default: 0.01 FIL) + --maxPrice value maximum price the client is willing to consider (default: 0 FIL) --pieceCid value require data to be retrieved from a specific Piece CID --allow-local (default: false) --help, -h show help (default: false) @@ -570,12 +570,14 @@ CATEGORY: RETRIEVAL OPTIONS: - --from value address to send transactions from - --miner value miner address for retrieval, if not present it'll use local discovery - --maxPrice value maximum price the client is willing to consider (default: 0.01 FIL) - --pieceCid value require data to be retrieved from a specific Piece CID - --allow-local (default: false) - --help, -h show help (default: false) + --ipld list IPLD datamodel links (default: false) + --datamodel-path value a rudimentary (DM-level-only) text-path selector + --from value address to send transactions from + --miner value miner address for retrieval, if not present it'll use local discovery + --maxPrice value maximum price the client is willing to consider (default: 0 FIL) + --pieceCid value require data to be retrieved from a specific Piece CID + --allow-local (default: false) + --help, -h show help (default: false) ``` @@ -591,12 +593,15 @@ CATEGORY: RETRIEVAL OPTIONS: - --from value address to send transactions from - --miner value miner address for retrieval, if not present it'll use local discovery - --maxPrice value maximum price the client is willing to consider (default: 0.01 FIL) - --pieceCid value require data to be retrieved from a specific Piece CID - --allow-local (default: false) - --help, -h show help (default: false) + --ipld list IPLD datamodel links (default: false) + --depth value list links recursively up to the specified depth (default: 1) + --datamodel-path value a rudimentary (DM-level-only) text-path selector + --from value address to send transactions from + --miner value miner address for retrieval, if not present it'll use local discovery + --maxPrice value maximum price the client is willing to consider (default: 0 FIL) + --pieceCid value require data to be retrieved from a specific Piece CID + --allow-local (default: false) + --help, -h show help (default: false) ``` From 25e89d3a7ae27b14b992915bf3a796bd955a5411 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 22 Nov 2021 13:29:09 +0100 Subject: [PATCH 27/33] retrieval: require unixfs exports to be aligned on block boundary --- node/impl/client/client.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index abfd835cf..d8622fea1 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -965,7 +965,7 @@ func (a *API) ClientExportInto(ctx context.Context, exportRef api.ExportRef, car } dserv := merkledag.NewDAGService(blockservice.New(retrievalBs, offline.Exchange(retrievalBs))) - roots, err := parseDagSpec(ctx, exportRef.Root, exportRef.DAGs, dserv) + roots, err := parseDagSpec(ctx, exportRef.Root, exportRef.DAGs, dserv, !car) if err != nil { return xerrors.Errorf("parsing dag spec: %w", err) } @@ -1012,7 +1012,7 @@ type dagSpec struct { selector ipld.Node } -func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds format.DAGService) ([]dagSpec, error) { +func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds format.DAGService, rootOnNodeBoundary bool) ([]dagSpec, error) { if len(dsp) == 0 { return []dagSpec{ { @@ -1051,6 +1051,9 @@ func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds forma rsn, func(p traversal.Progress, n ipld.Node, r traversal.VisitReason) error { if r == traversal.VisitReason_SelectionMatch { + if rootOnNodeBoundary && p.LastBlock.Path.String() != p.Path.String() { + return xerrors.Errorf("unsupported selection path '%s' does not correspond to a block boundary (a.k.a. CID link)", p.Path.String()) + } if p.LastBlock.Link == nil { // this is likely the root node that we've matched here From bdac11ade78da7371dc54eba26f5160c1016222a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 22 Nov 2021 14:18:50 +0100 Subject: [PATCH 28/33] retrieval: Update some API comments --- api/api_full.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 8c720588b..06aaff99c 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -933,13 +933,14 @@ type MarketDeal struct { } type RetrievalOrder struct { - // TODO: make this less unixfs specific Root cid.Cid Piece *cid.Cid DataSelector *Selector - Size uint64 - Total types.BigInt + // todo: Size/Total are only used for calculating price per byte; we should let users just pass that + Size uint64 + Total types.BigInt + UnsealPrice types.BigInt PaymentInterval uint64 PaymentIntervalIncrease uint64 From 407c2ed1142a5d8b940c6cb15ecfe2907b7890e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 23 Nov 2021 17:42:43 +0100 Subject: [PATCH 29/33] retrieval: Drop the RootSelector hack --- api/types.go | 10 ++-------- cli/client_retr.go | 24 ++++++++++-------------- node/impl/client/client.go | 23 +++++++++++++---------- 3 files changed, 25 insertions(+), 32 deletions(-) diff --git a/api/types.go b/api/types.go index 330263f8a..acb18f7ac 100644 --- a/api/types.go +++ b/api/types.go @@ -210,14 +210,9 @@ type RestrievalRes struct { type Selector string type DagSpec struct { - // RootSelector specifies root node - // - when using textselector, the path specifies the root node - // - if nil then RootSelector is inferred from DataSelector - // - must match a single node - RootSelector *Selector - // DataSelector matches data to be retrieved // - when using textselector, the path specifies subtree + // - the matched graph must have a single root DataSelector *Selector } @@ -226,9 +221,8 @@ type ExportRef struct { // DAGs array specifies a list of DAGs to export // - If exporting into a car file, defines car roots - // - If exporting into unixfs files, only one DAG is supported, DataSelector is ignored + // - If exporting into unixfs files, only one DAG is supported, DataSelector is only used to find the root node // - When not specified defaults to a single DAG: - // - Root - the root node: `{".": {}}` // - Data - the entire DAG: `{"R":{"l":{"none":{}},":>":{"a":{">":{"@":{}}}}}}` DAGs []DagSpec diff --git a/cli/client_retr.go b/cli/client_retr.go index 15e823b39..6a23dd14a 100644 --- a/cli/client_retr.go +++ b/cli/client_retr.go @@ -451,39 +451,35 @@ var clientRetrieveLsCmd = &cli.Command{ ctx := ReqContext(cctx) afmt := NewAppFmt(cctx.App) - rootSelector := lapi.Selector(`{".": {}}`) - dataSelector := lapi.Selector(fmt.Sprintf(`{"a":{">":{"R":{"l":{"depth":%d},":>":{"a":{">":{"|":[{"@":{}},{".":{}}]}}}}}}}`, cctx.Int("depth"))) + dataSelector := lapi.Selector(fmt.Sprintf(`{"R":{"l":{"depth":%d},":>":{"a":{">":{"|":[{"@":{}},{".":{}}]}}}}}`, cctx.Int("depth"))) if cctx.IsSet("datamodel-path") { - rootSelector, err = pathToSel(cctx.String("datamodel-path"), nil) - if err != nil { - return err - } - ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) - dataSelector, err = pathToSel(cctx.String("datamodel-path"), ssb.ExploreAll( - ssb.ExploreRecursive(selector.RecursionLimitDepth(int64(cctx.Int("depth"))), ssb.ExploreAll(ssb.ExploreUnion(ssb.Matcher(), ssb.ExploreRecursiveEdge()))), - )) + dataSelector, err = pathToSel(cctx.String("datamodel-path"), + ssb.ExploreUnion( + ssb.Matcher(), + ssb.ExploreAll( + ssb.ExploreRecursive(selector.RecursionLimitDepth(int64(cctx.Int("depth"))), ssb.ExploreAll(ssb.ExploreUnion(ssb.Matcher(), ssb.ExploreRecursiveEdge()))), + ))) if err != nil { - return err + return xerrors.Errorf("parsing datamodel path: %w", err) } } eref, err := retrieve(ctx, cctx, fapi, &dataSelector, afmt.Printf) if err != nil { - return err + return xerrors.Errorf("retrieve: %w", err) } fmt.Println() // separate retrieval events from results eref.DAGs = append(eref.DAGs, lapi.DagSpec{ - RootSelector: &rootSelector, DataSelector: &dataSelector, }) rc, err := ClientExportStream(ainfo.Addr, ainfo.AuthHeader(), *eref, true) if err != nil { - return err + return xerrors.Errorf("export: %w", err) } defer rc.Close() // nolint diff --git a/node/impl/client/client.go b/node/impl/client/client.go index d8622fea1..c2611348c 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "errors" "fmt" "io" "os" @@ -1024,26 +1025,26 @@ func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds forma out := make([]dagSpec, len(dsp)) for i, spec := range dsp { - if spec.RootSelector == nil { - spec.RootSelector = spec.DataSelector - } - if spec.RootSelector != nil { + // if a selector is specified, find it's root node + if spec.DataSelector != nil { var rsn ipld.Node - if strings.HasPrefix(string(*spec.RootSelector), "{") { + if strings.HasPrefix(string(*spec.DataSelector), "{") { var err error - rsn, err = selectorparse.ParseJSONSelector(string(*spec.RootSelector)) + rsn, err = selectorparse.ParseJSONSelector(string(*spec.DataSelector)) if err != nil { - return nil, xerrors.Errorf("failed to parse json-selector '%s': %w", *spec.RootSelector, err) + return nil, xerrors.Errorf("failed to parse json-selector '%s': %w", *spec.DataSelector, err) } } else { - selspec, _ := textselector.SelectorSpecFromPath(textselector.Expression(*spec.RootSelector), nil) //nolint:errcheck + selspec, _ := textselector.SelectorSpecFromPath(textselector.Expression(*spec.DataSelector), nil) //nolint:errcheck rsn = selspec.Node() } var newRoot cid.Cid + var errHalt = errors.New("halt walk") + if err := utils.TraverseDag( ctx, ds, @@ -1061,7 +1062,7 @@ func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds forma // todo: is the n ipld.Node above the node we want as the (sub)root? // todo: how to go from ipld.Node to a cid? newRoot = root - return nil + return errHalt } cidLnk, castOK := p.LastBlock.Link.(cidlink.Link) @@ -1070,10 +1071,12 @@ func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds forma } newRoot = cidLnk.Cid + + return errHalt } return nil }, - ); err != nil { + ); err != nil && err != errHalt { return nil, xerrors.Errorf("error while locating partial retrieval sub-root: %w", err) } From 83f65a673bc60b9a430756d9f5abb0df3ab7ceae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 23 Nov 2021 17:45:56 +0100 Subject: [PATCH 30/33] retrieval: Docsgen --- build/openrpc/full.json.gz | Bin 25677 -> 25675 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index 3477fdc45dbf338ee327f4d3c43adb5d2a97299a..605bb8cd4e322444e11787d2e88a810d09586b88 100644 GIT binary patch delta 21232 zcmV)wK$O4D$N|g90kD%1f0E!YCTR>KvQ3yrb^=Ob&9pJ?w`MYR_C>wLjCnGj6hG)G zlfxAKH~gf!W;{WW`A=p75nWY(DeI2doUQLyxf4rf_8t>gCiZK`*rVzuE{QMF7asmQ zM8W*aXoP~rFgNO?_Tod*U<;_58it7khzW{;Ad|-!EXtgHivR=;e+oFnMDqK7=B7hJ zD+A9cLBQANAkVFX;YL8@TJHPeiGLa$$F0LVCCPp&xml8Vx#DoEK^e~I_da2A9$=?M z!l_S~prB>7b7W98JAgyANSb%3F;b~Hsf8h`44R$8`4kZ@IrpuK+GmnlEK})}I01sJ zi8k2WhPGM{j(+cVf5MRXDlHb&=7%mEyso-^Atm<*M2=xwFA!C^3hHQ3Eul{2O140S zRSkBd!p6jz02tlz7!ed`o_>6Ip%mGoN2z$+9WZ$+%112*SPS2|C#MR%sd7)(n?l;L zPsxb3W)N`;2Dyf#qKg%vRJ$GenP#b7Wa3ZLkPdzzjr|;8eOf%qO7=y01#aR^PqBZohXNAo=bJ%ayaey}k2N{_kpg`#}EhKRuaa zSHg~H8AdaFJb{EDZH+x8m}4mIy6qew`@IXqd`i@pZP|pNW$;t2huFh7wJ79!wFW~b zQ&t~A*6r6gWvQu96L!ijks6o!{Ra)5i5zBbbyjYOe=mFzsgG@^CYl1tPsCEF*kFy* z5N6iUkDih`%JUb!d6W(dz;FbR%kl4vP<43YqY^IlS*yQsN{M^ZnT`6SzS)j^0s6=XLQ_}uroR? zZS!~Ee{D;!$${HvI4%EjEA6+eM8vgCwqu&%Hz^;OW?PDRxY?#{aB7(aKbx`c-`)Ah z;mA$Eh}|6BZjQ{%E?i@E9C-;}#Vq`6x`=Y>M$^6{t)OYQ<}zNZE=~AJ1|v7IiWxGC zGP#^Gd?$$7h)pF(zbM8x&AIAr?cHW=T+t$De<|aWYNN{h6k@{n_nw1i>bn@5o>Z&l z#;5Dvb+5JcDJIv@{-g;>2L}q7wlqQDP`5kRDm7OtyKJ+!+vRPW*r5EtcIGwRe|y`t z7ll7UCOx-WZoQ~_2O+UD?R6CuJ5S`Iy%zVM7%pcMByZ~`ZY{^BCt`6+Pe*UA{U5EN ze|KXIy*4m*fvF6t0%OZvbUIzaV_H>SaMt`BJbR%EULdxZt*D|Mo*>0UE@CLUXFOFd z)AF1-6i6|5((FSnHW8$E+1?j+qtyiAJ6H0f9O??SG6mxlz41CQsXG_JZTlj(+O zA$xnZH{8xJ<7s7s>yaM_6J_?;-9(3d_g z*9r?A0qA*gH~}7)r6YM=Zt27Ur6Vu#jw~;=S~p31F!0a@QfvEsLI(pI>+89SdK~udRU6v@Q}%ijYVs6&<&Pm~v&ADRfbk&O_a3e{^Ik z%jdO823sbRn)nSZBZ#asfpsRZn89KOiy5plfpsRZ&IHz(;ORIMY?vzay7rd8fc?r^ zx{NZUTc}cp)N3zMe0yQil-9(PE{xB074$p@d?5K2q@*?`1FKz zp!-yu45ePISnoN!s5o=zSKeV3e=Dl8(r;+KN>$^MhJjP7WNhxnep7p&RV?5*SnV=< zJ87h9d&ecrO&&?3wHrOta?Aamp?}eqPtjFv1__cvRd$dLSV3m&B^)9De#MsgIY;aq zV)wn4$_qENtZ{)Hgl|pN4{HT@#M+Qmbe^Ua;N2}E+z{6rQe3OZsmP7(e`DA#r34D5 z)=CGA)wnV~VE*YTho0cV&1ra?x|@w?QilNF)u^>LLYJ{m!Zu!6|4w#BsQibd`f=ccyn0a^W z%j>I2C};DjuBjt$ZN45Ge~7NzKy)=+xM2`F9ZK#V!bwE2B$B8ONw7K@&>?|D$ts4G8;f66Q3>)I+wf`Fr;ndfdu}k00Yf@N^J~&6SA%=%6+I zOi0Ik(xgxDuJ|ku1g`KDF%GA`D6zA>Bi${qpho|^M1Oy#Et2&|%@`S?GL#zC$c)#p zMjI5+5_9NTGcqy1dw>}N0{KJV22`WyOzb3+vbtoM+csO~5_@O2HiPYj;7I}RAa!?1OZSqyyQJT) zwp`hOOf8vZWI@i+)aMLv3Ig@srt-F_ylpD)dtrow+O)yifB&Ox*2$=P70Zj=OTGjy z3WUxp7=(Ic6jLDTphHDBMe|}jp?k;bJW?kJPO@{;k}F^4 zL9WwhjJnwbXuUH`ZrHE8JHLK}Z1Su6wYMvO_0kq#dH&p?`B#0bg8lb#7=$C@8#~g} zC!e!+VQUwzfAIWh{)ryIoLp*;%E*qQ5i0-Mggds*8)x=mV2nf0LvGp=nUXlKhOhV^ zaFnapHBraoFhGFu(76F_DD%$ZR9|2uI@?@F6su62f9)N;@m(sw7T1OL&>sRfq7jcPmxwM4Z9(!RfQrP3V9ct`piWhCLq<>%DnU#KAuJ|oY zjIiqif9U*Vnc_~!<->K)@AFkagnHdtUTMyq*2-pBHd#;PzkWGJjB>?U?XW`Re&SupIyab%EncyB z#o`rmA6&FN@PfaBk0_OesiFbs$7Lo7AM?KHL|o}F-uSl@JM&+$R7+4 zf9fJIRL^Kh4g~2ADJF0WTusn5nIb-+Vs*DJ5e01|cD0sYk9_ z*W~QW)$f=8cXW9C+wbT9cXW99?*GmYuRa1qZgD_~JUzID0fs{l0fxB1I*d+{OvBLE zv5z3XCZac)njxWbHbDe9(DOu(nQ>w!e<{!&7DLEX-Od1#O`ZP$J9~e-;u87?$lfuI zr;wxf2pz-b+q=+0r4}NllWpm+5z;GDX6TYWnd;pY_Ae3t4b7i}XX?oj9-orhpk#&q zYp$QC+iY|Jy=;(+T{erPA^nqqF=Swb5OAR1o$isDVMEPBOUlS~D2?Rb2XvbLfBl&f zNBac*A-AN$>mMLH!&A(A2XEyMy^!LU=K-4Ha2ja?k{`-!+FC<8=>3~`+yA*dnmFXY zCg{Ka^{-yPSDIyeIP_gKz6}O=&!G&>_+-w;ukLQ{{J%$JcQ$hV*`MC-PNwvR&xY>Z z4ZPc9y`~;RYSc}!Tx#4cs_OYOf5EHi32p3rr03qGAN{e-TzS-Y{Lz~t7t19!!~_QO zcf_Ht;MC$HGPJ*!inEhaKWGc+tkz!(#LWyfgVuf}b?(HgO-cB+0~hF`FL91F8?lv+ z!Y1?!48R|-2@lBL;O6ys5Rw47|GXaEk-hO>uLc8mK=%gxZ?fy}V*l=Le-hj}lUvmL z`TgFU719ExVj|<02KZ-tMe#n?6r}7_+$B!Q(U?~*0aKzj8UU3dJkij_2$w`1mLrqx zmOvymjQR`3Wt3!rNXV4lED--x4O^-Yadbt#bxlq=7(whYfJbp5^?*}=?vNAm4uj-a zEMu{Z#WL%RWh^c*#|07Ye|sEga82RCZpC%&tbB_Y%5nffw^E;ny)5pCv}Qza_qwrb zB=oGiN?kxl029Xx1-}EBfIrDlp7e-^e*)xFXTrLJ9&4L8fBtcWJgHSG*Om1p zG#A;m6?j$=z8^_9s0gl*Vzyc_`T(2X_bF|qfinC;|F}fNz4Vb&VZhJVG!k_QL7li7 zRSmU~?LI|QR&AE&QtE<1NEqU=S}cwG7g(NFO1JyP4P z;bM+z-ct}Sdq&Wae@9iGlUqh_8NFrn>&58TR594A2sZRd-a0*W-6QBh@yVO&1o;M< zWRNJcfMu*v;F-Z~gFFQ_rmiSUMH5Y+WPpp6fC5d>b(y+^4>(oFWwHo>4yWD4PFrVN z&SSSct&Q=a3>#EATR(AM>jzH4v8KY{RYjy2#kD>let{SYe{NB6Ihjd=R13zDUyP48 z3SF$?XrpW#)$ZO!lE#Ruh;(pDL&5fv6E%*YLt>)@9tDWM4)~|&@JAo!+AU} z^2z(n4zFb3f6lQ6o73LcINRBOQxT%3xLjLvf~1LuQufj`tVG?*4boKM%h08nMZ`O# z$AY|C{zUo2I#X-4J*(|)s&U`4`)iiFqO72z6We@GL3sb4Uy=0YLYK&5k-e0zna zz6UuLc(uK+E6=RVB$#@A-=EWqcp?@<;D0CdV`+%9&t6ts?{-O5{;6iy~2J=ZE-rHV{-Q<=x(H zUvx$=e;J?CORoEoc_Nzm3uLMI(Zd`~7bjaN4<*&={QQE3#OP4JEECOI&Y+7;=aUlATpxOS=hsDEgXqe{r2 zryWQAlnrYEc!3;*eYIP=X+ABPyJ_+0?DB9t4$xSj>w_Pf>+SY6?=1Z;If$sEXpT** zf4`@}jpXQ_+(arvnwYjD$WZL9E!t&ysCtPU|0eseDHKgNn&KsoQ;BaDi8L>Rk`3Nu z0&@#y{l#&Ckvw(u!>*gW-l>QhD;fI-Bzg2H8PRqJ z45lBCVWNm(M?8p_NDNFFc4y(bC;PM0S9S&z`R)C_vJ!m{1PphyzIG zz@gKr+%n3s7&D+kN^d>Qv*yg#Us(hZsfF8cnh$+dHoJ>Uu3YEe8=t+M)0s3Fe{J{r zz0ZixXmFE$bSV>mCng;l+@L^wu9S&Ml~Kt#277~7Z(k4I4EEm$NsMk2g>Ehg(Aw2r zz1^YIob=AaV?6ZGdm5K#GuOwL0#X$})J z)vG|_x9|G&;1Z7sa-NXSg(?^X1V%%CY7sUJ!YVMz9DH)2tfFH7nRArbP z97u)Ao1&?x(OuxX@`*H(syZ8vW&z*qVZ1$zw}$HlDf+p#`PEt0|#}OP{GO= zD`TvTdCW4#+Lf$bsZQ3vs)|c10vGqDI2&BIg0+RDFzR-@bqQt{=-* zE;-~OTPW)@J=Ti(vgVu1{QL+Pi2E#gJ_^JlX27Fk>_87V;xz-5e*maq0F9G2b{F8R zWWudbw?h4K3H3jqhuvgiaxNq*m!W0cqhzC16l<~9TJ1U^%Nu)K2mxxg*xm&}YVwm7 zK4Nt%!iQY&%JU3OTQ92DiuwjscqsMtzx=kuSKlxpzF!qjI2JhPh={Gu7!OVQvVShO;m30_gPO45osugjYdH?VZRe}ptCi{zl}&=LR7D%1xV z;8v9FOvH(qFxt{$U8R;@v}$b*0n?TKX1gk4wD_PO5I?45go~k=j8=PGy=29wazD#u zCd(~^ka3IM+`r8eS+ft4o>e;xJcGcY{v6QJbGZd{mLIhh#BfJ=+-OJC41WxmjpOL_Tbrz*&?-nt6Aw`F3of3~^xF~o%L@0sv2*>E{4+it_i zB0?9)^tfl4x{RPZ4v736WhG-ias-xj0Yk(^!PYhTEWV!1XqJjD0H$y*!w?J+fSgXT zvju)MYRwLw8xKE6O~|Zw{=K@HQn92RS`zyGE8`Q!&=36_WwIf=m$@Q1H zgSuc1?c+4mg=na7y*G8dUl5t0wWGaRNCw&aea2yE&G<9yJKnc@yNX{Vk&6_&PJU~? zB?@XLA%h?#O!jStSw}Bpo+=+ijB+PoZp9!+2mPK7-jb8qlWIFFe-aD&8A7aU%azd7xZH3cozntF3bX5w6i`eig3~9+@x^~i!>F=SO`6D z4j>a?NgA^R`|eBZ#QJg($9PHuscVu!Qd4<5e~Mx&imd_J8jzR9O#5}w zZ7z`~mSQWgr0N>Fs7br=$fJ<&@3nVVQ(fMJw^}?l1N*TiD)Lu@z{89KIx4x9LIZ$_ zBVU@Sv$L1gSO7B)~n7!Hw)b!J9OJHue#TD z(Q_`ByKQ$_e{tyczo!(hrS=s+ZKK}nd*&l2^XFXKy=5? z-z$0js^rBe+0o5>^VyjRR0Qf%n#5lrXlm@ zKrj=^kP8l7S7PWO@!&b-fWsT4nEHr%9-Rs0P4VIba2;-M4;*QrV0avdBoj|jU%yqC z5~FuQ1r%**Qoxjrv&Ca-A|w0hi7s$WG9`lY?g*MGGkuVKKARvuLF&e&+WCT%XyfTB z46q3ee?1ork?M_w9A%!iEA#qSq=P?BMs`iTU z{SsJS(bpxEk4IdFX9V%M75B6_uW(BeUh2Q417c(5%V98`rh6@X-YS3PtbHdBSmKZ^|$f!7+10F!apd)4*oSt-o0ptF1ZOcLtPE-&yQ~!x# zw#Q)0dj#0c0w)WcEO4^GX%m6dUR7+`MD}0Zk&m!VklI2M-N2~IlVs&xeuT%TgiANk zVxiYbyKlg^@54DqOQ146tggXXGFw&tf23k0Vh9UyTSQ?)_|{LybPJh)X+z2@x3LNNpkrg&SgdF{HGDIDD&C^Bo)hv>>NP0z9&~>J^c!r-AHfE%5KY)Yn zI?w6rN4d-1lw5HZFv4tV3))oM&<$26wK}QQNv%%0i8|@4s`%jykC$+0Ztl(8ttr1_ zZ{<=~83fj#Y3vQgB{Z1ppTSIzW)O^X67nt5lj;_1Fd9zL(zT8{)0we+k6ooll*K zxlWTV)JmO^WOt{>M6=L5TAilEk^jWe?}f%OgPw=Dp`H?7woC5cDa9eDZ74?OkO4oyPS;AsV}LtF<)W}V zRuIm+z|Pu6FRs-$G8=kjf1hDzduMxRC1EeD3evp_vNm&S0lo$J7T{Zezli|G^o=+9ud${3h4jBB0LTF1oFsje}?(Q4d9Fb8UW@a!k?(&f1TSrM#f+!hqTrd zhOH|LeOnXT{{!OZDw9`R&Z(;}$1uNre<7N?O3oV%*rx8eK4V(0*)}{g8KDuYjL3>O zWRxi_b(pHMr3uzK#`?knzr0tDNE5-EVA14Z2EShR2MqRaif&O#R^3eJ>{_|9~+X$X^Z$IK%|_D8SV1=1MN(FyH|B)R|;oAmXMUTj^+}f1~M1fSoj}w%v~)+pS61 zpcyVE<4b3P+|X-n_{uf2oEj(E)D0R+u~gSJ;w;r}TWOXZqmd$)r!OtrZq+;BA`Y>~ zl<6z0%vq(&a>zv-GPSS^^O${Sax!bt6d*_)TOY{t<$mvX=?rnAeIfMY zhR0g@zM85pe`&HzG^{+$tIG$vuJT||6Uls;Fy(^xcnmR)B zdo^+BFVK|UB9;HK6{Jr$uui#&(sXqO-*baC&rg)0e zauumF3g|e<=PdI^fZ!>-LF%T!1d=fVl=RbRIKkqaUK0V&RSP0-8URAMY(tDd`VmY5#3s~pQ#p$~Izs_+!Eg>JLDvL&R5XJ5 zB-6c~f03tZ(^_Y0AN%cSo72c*NsA>dmb6&1EtY&$6Po{W2K{C78l~#5IK6w>h1d>) zDW(R{8RX1PN^`4mnx@i#`-#z16jcy)g}E^+7>aXD2jV zf&wIiND<1x9B}~295{43MTAFjsp7=@TOA>yhx75P`S_VvCgSXJP8?m~;73niUKL{M z2u(0?#ovOvAQU8}#4mP5&%>dI+2j)&f3x1f&hDU}dvWAZ=SIEy+3(3v(@7iZUn6`?Du#nC&=ob}2QPV!hN%$ThxRU8Hc+WW(EvFtX=x?o19nj|z&AGn; zMIvjjIHqKTgDG+kr(9bbbj;Vne}k_O^GifHmVJ@X^ZLF24pEq{`v(o32?{O{auG_k z+8nj15jE8@v%_gxyPfqRJzp*BW~Vmc!jc0J4o+S1a)i@(C(+?kAE9agt;_XWxIycCTdl7T@T*tWhBuY&nn`~vxCf!5~Z5x_u zU0KOqMUY6jvp%4fY~%wnWg$l4AyWYQl#J*?X7d46Fq$?PS&e0JT>FPPC|f&@^tdqe zDvU<|D@7h3if9WGfC`!6rBSk48qdR>mGqlBOlhuRPjM}0elv!dc;3T;Wjxq~e zbHxAdjt8)zbKIfBY0$i@6Ubp}Z*X%T^)f%ePncXHF6)_i^j+a8I-}0bCwSMe0&bl% zrKKDwRdQ3ztf=H>U#^jo8y{LQb#QfGN^jDuY4X! zYT;bT2kqwh-uQRTx3#9UL@9VvaiLryUoDthDFmu+_vfyv-BldB#uVFAS9|KZrgpw{ zWg~AZZXT+Qf7R~lVXAHe2lrI1#k{2ECSn^)fx+#=r^G>C)P4QjOCx265*|>E$`gg> z1P91btv1c!nQ5^x_)hwLa=AEnx9hHxX|?UpElLlgO`p_nVoZGDi@IAf#-OX4&=$qv zoFf^8Cg+5~GbuY4aEpYh8uXFrpfLl<4BRXh&89Mxe_C^6*{5!1G08KLjGCdd1ZQ*= z{VcDym4(!Z4FFoJPkKqkm!#+lsZVSlh~zw5@b|7V`jkRck|5t+kh2j<_|Yd|grNPTjq1jU$dH$ho=n5pm(L zt?{e0f2ppqY@9!SNU3LvUb7^X16lRGT4LA2m@SO8FxJ9Y3u7&ewJ_Gg*!zRA4?vZRAk{_PEnNkXj*2gCPC==C{`t_9Tf+8R-FJuBpBlvn?v1tu9 z?e{JxG~grXd2K?3r7E|)mC+P*T6ApDu|>yE5gl)8+`8RYb-|`In`|E-Ak~1m1N67= zf8}S#W*~K89D!29U|Z~}M`pdHk+j!W*zodZ{nnAkTLQFiz1 z;)q2dBEsPmFDJlN6DxE7GQByOp=K2qDD%U$o?rI3&K}oUNMj+5g*0mmX{_hh>O8+} z2mu>Hz!;3bu86XE$stFFjw6KiZ>V`lNZ(xj<_E!7*uO-gYO`R!xqep^>bE$$dgr)2 zRjNe>_yhqz9D3LRH)tNs2B0JPW{htU0jDR?^K0@XQG0GZ7*X5NZtoXT^pn8}FMr>g zInzcT?uakGsR-MucCzPv!UP5HVd(MZAy!J-sN|1Bye0Eu%bdy;?D5<87qXSbd@<8! z{fUDt(}F4Hhh@STvt&ig5qcgVl)OZDzF0#*zvdIj0d<@(05i-dm;jH`n;~>=01}gw zW1n2+`R*;22eK?H8xymO6{Q8ROnMabir zs^X|qH7u!aMZ6kIzAc6#nKBJk+9F?Qb&Ly5Z34*{v8`)zO_G+O?|T@j#($$`Ih8*Z z-~NRG9|l6jW1`X)xSF7A5*5WvCP`AQxPX$Eocb>{``3jSz(nC}DAdrGm|PRp_Eu*? zA^W{e{q2;-tIV|7^p^SLmnnPhAJ7YtHs^kT>UsA8K2DA_|@Ibo&WcU?9N8cKl{_$-N}^R z@Y&G4yMcFmtg&UdT^`HSxLZ`!^Jl_e(-YdXCv&@QBa3D99orfW+oaJ{>#uROXtCdm zxzM7Xr2+`11XGdKWnY>wx=(C;jNKwrX&V&%6B~=ho&f)pW4&?t9Eq|a1%p4DgLkD_Zl#`%YgAE};(uRpR)P(#fO1k2Tc>p5PQ`zp| z^|NHqB~3eHGvd%)b&pC!6ePDtowno(RjZWT_aR9_xaMD;VYAa^lOtnhI4|6^oFP`c zHT@vBrWD?qk!W;|uKKjd+#>Uq$UHrNu{3I8)rtj|;pbJ-%YUSBBnDeeeAxgKahEW` z!lo*#n1WsnrX~ZUT5OwWrkd`87#lW8%aMn2jk}FeU(=dp*s^IG5y)&>FNH^28e6mb z$1-Efj5}k-TGMX0i#MBzZ`l2}!FEO34`o-EhIUtaueLmz&NQ(g6}LRxNYZjoq5n;N zi0{heCL39Nynm7IvF2R5y!qy*)cFyboP=aZOrKHn2?9_(o=qr2>ZpS0BM&e{e1?!Z zgWxl&4^dMX22(!o@TX{xYe*}P-LefA(0Z9IB-G_a!kfTGJ7q( zhzreBL6Q>3;UV6-qBn@BTStp=NWQB&|KV)gEkH=gxT-JZSJ0RCBu018;lN4C0HzAj z5s1DFfPcK;9Jh5%t_14P?i8UA1CY#T6BMA>dm^BmI@AN03mMXvTpTk%i8mJ#0z`{Z zd>Ix_e%K;DUb=Dvhu?4C3;tw- zGJi7Nvh3wc$oE7cdX~kbTO1fKLxJ>!9e;V~4i7y996Fr}1|xla(@tU;4>ZAj z%9&Zp`wIpL1R()XG(f}gMArD@6PO9^rn#XX&|B;x7fj^ULg}UUb%F?RpywgCpZRoP zhCT1#n!E&O6uQ~^iiSR*fncNZof8Jqfz1(L6-*Fx0Uk+48`Tm`MQtapZ(2pv-fX!Z z+Y%GTrnF$0VvUpev((TD~J+)mY4<;-gbI_-LgqSMmnsx5VT zs`XkB`r`n)NDKm!xtd17Gu+Se$Q$r-1 zSicCq7M z8%H+lhG^XotsA0sL$uIit)Pc>L$q#))(z3RAzEg9<(dV%>S$Idom*o@3;oOZ<{zpZbxJ7|7D>Dvx$N*%C^_X>Q49=~E6tte!4`5wo&uHjvqj8!@+Y-2>-8kKH{?0Z#_XHm0Khkv?A1HzWFE@`1j za81GE6u$i&PV>R-i?R+2IWmF{`tj31#3dwk=TVXys9ALTB0Z3EH1*YCrz>A%-71uT z=mUZ0;^W=`IoQY2KU$vQNAWqOaFdg$d3%R4Xzc_AXHh6vJDZko%@ z6k{sPWA>eal<s>3`FR4G~QH71`4y&yNXru1M%5`QHb`%kRq>Nq>{5_Hy|HUNpIj9Cg{9 z3X>Xy!A0Y^|FlUhVt(4q_!p+!BPF97sylN}RR^)Cxq*jR-38Vc&9ALK#>=YB%wHRN zA$3D^nKyOWYd#=;0f|ee1TpqLpwlA+9mxgT38J#bDm9#7?sK|zLZ=XuOC;E?^!tP3 zSn0{!@>*bq6MyBC?oC8k8Lqgbn)4*U?MK)&1Z4$|Rlwzb_j)bI)Yjr0C ztH0pNVEJkegdW)y2WzO?EXa}NETyY6^ZJqLjO!{H-`3?XDlpnB?A)NXnT)DhdnB?^ zQ+uDnF`1xcq+>*Qxp-99jjeRF($PvsD;=$LOiRbT?SHzQP-mF&Wx^*W_00nnef!=t z$7#|_?NQg#Sxz&fSrDEmMN`^%8J+yd%zWq;1#|6Bfk`^tfDIM&0156EB@8Vq(e(xM z&Mp$KDYx0Fi`OdlOcWxaJ->-ssLV*G3w(mSJdLQt*;z0o*;0H|Z@`Y?NDp-kDT)=_ ziV>xTX@7Y9o}tU5^o9YMDm0zE4tib^I%9_U1VAtq6D1?OP(TBD{2uYKK46^szId4l zrWVHz>ME;T`kNUV{b0$BYV}$>XCgs8h-<1uW4F!01-r9y~pXuWIPXbPA#Cht^Tba>zw?*#-P@{l5H~GQn4MSHM5P;0E!?#i|u2 z%zreo96QW3wU8NRO4o42Ou6R#Fh4#6rkJHn**rpDvcY%E3Fu7f1!q($4BC0LoKZ%2 z?NZ;{RxMlhY1!uj_F2EN^KVTITWaO%tITTcwsGf-nrUj*ih6v~cbfP1>+UqGuX=)@ z*HXJqHDIn@r?T!O^$nS~?8+?LSW=-a3xAKswLsBMQx<93n2czkLQDZTq#@U;whWT7 zS{7}YUaLG&$ycmP6O}1tmL+O;f3QCg&&5C}gP!~wBL|fxnQ!YmpTs{o*x4TJ%7{=h z0*fS8&&sif5#j0chA28r`q>#ABCnVo{*uFhD?9UnGzj;5?}(dypx5+yfE{#>0)PEd zN`PV<8vQ~Mm3#cNuQr#MpiAU%8kBT@1{wd|cg0uEk>No$Nk978VZy8QcWh8Cn)}AL z5(?APw1wm=nS{{NU%w>2YI@N&yX}x#oJL{S;zikXsAY?+X&dQmuj_91D`9gE1CAZ+L(X`Z1-BIMPfw~Y#y8fPQ@wU~LB4E2#)i%`MSs;AbM_Lw z(40FB8>wb_WiB(X_Er=v(h7hEm*;D=`WKCLd8@!bqypc}YHXEstE5{c{a%&y4S|Di z>jG3rhq^D@NeH=NH4xE)!G*#oq|uvh%N?+1yX8#7_0hep{ke7 zfeD5z5}XE+8a*=oO64PtvVRauPe*~-WOZ+@;`ouVjl^WZgZQ>Z6LczjB@A&SWLUjro4jkkpk#nBq0hdiNad%qw7Qv&D+K!q57 zVUfvS!3mu&eTMnO4d9Fb8UW@aV*Ttaidcjq(hI+0BIB0^-+cQ?`6j>Gsfz+3xAqBw zu7^q6o7vemmQ~84hJVc1XWVVA0v45O$u-ksGi8?vVd?wVaTs(m25wwpM1z!9vSgo@ zD_X9&P`8TKScxu`dbL{J=>#)Bj?Ar;tw;{t-7w^+l6*pvVZS*jHbL>Qx-FB5K$7H zt;&cQ69h)0;5@dQWiFRkHg4It2^%l0@?x&Tk`e~<`KrIOqUaiH$}R_WK@sG}E;T%w zGfPo@IaaAGoN;4jj8c@Wa|Fz`*^)`>v6-?+NmI!YLe>Gr@<7W2jhqg%OStH3Sa@&Z zd-7#QvOXSPX@78&eexO2iVwxM4ybn+Mg8a<2!I?_Kb}A`Mp@U3bA*CZw|CIn**zKj zM(NiN|J?oa+kgIji$4Aj_do0(vf%gsI&!|=eLQ-9x%-j5qaUvBf=_3+zx@v%_4+*- zyGAVUq%TZdIyIkKonLmG4lAgm+%vJ{Ofww$t(efH#Xe!SYN3#+0zVd|JHEm@BALnq|KEfk_3Ct35ZVo*BacjkTm_Q((z5z?X&6iRB|JA@h-=^2A36 zElU4k=2p_n?E0p${F* z=h1xW$BOdU?k9zQ=2gu7OWwr;ZU_bL%@d-j4}U>nu8NL_l{43%{pKD3RiDJvRiUA0 zLu#F^nKMKO0a1&wJ+0DZPgU;+pY7L1d=fG#-rURB=aWX7ps)}VzQ0!ixxRgm8#O~* z2@SP}tZ}iH7%MZ>Ca@|+U;3=oPaa1R_+#W0oxMI$%%s-G%IUhZAg3 zJbxWdzq<$5T9mRkG&2PpdS2_g(mFM7UHxnsx-XDJZ;^mTEt_}(r`xYh%)T?RxTGo^ z{ODc4+0mRM{aap+{qGQZn9mR0zmlgi>3;@?UWJ#Y#gBI+n|2!TrZ(o0Tz--1^8`u0{hlr_J}a7Buk(N^Dpz|TP%}4O zp9qNcD7yl#>dT%4*pmQz5@1gP>`8z%7TJ@42ghb_Yh$4)upn>zR9@F;0?9HhEq`g- zkZo+4JEG=#@7l^y7@M)ZBS)75Jty>J_Ba8PB4a;*956aX00}lIQ51}CWBDk+)V(I@ zjG5`mF9YOBn@Q9-rliA6^N$N@S?1cnc9QI}qU|K}&IF?#he3b{&%2+BwJU=d#m`QO zi|!P@`{JVnIv#tJq2$*DdLAO<M8sRG$b zar?@Nu;S$zr8h(9++>M3L5>F%(#z1()LS(fqLGKHnjfR0nQviIBQv?i6-p+;ER2R+ z0VYM&3NgC0mtJZ{PM{TEx_RAb2^rTctXT7!*%kfdyifPH7Y6i-PPmr3G=H8n@*p=5 zlkV@~7Gh?srpP8m;o{qfL44?FZzNj8Vs9iqJR99v7|BT4N)A1bD&pJo(p*bT7%Cj% zt+dgm!Ztso)F6I#2`%N^Reh#bytFvhCPw!>bdLFY6BXNb$}Gm&|Z*@Ui^5iU;Xnp*6| ze6HD-Cf2$$e5=o2l^?A7TREnRzZ;seqDG7@M{~2XXNIkAnw4X)Fn_+%@&U_Rgereg zV$P_z$+27n^9P7sUW{69j@V0MmgmeHu>@dS;8YnuM%d#hNLfvoI-`P~7mE-WAb^SE zg)VZR>Y+|?*_p|T?G@?t7Twzz>^26wjlphXus;Aa+N+DD_8IXR4Q@V*cdd|7s(}(< z^c*~M6@yQeTZm{D1AR0@R2lU*b&=p#Y)%~H0!|wt zkuu(29B$!*l@>9;DRLe4Qp;ZN$;Z9!U)6=((R1_%3Ru~_$|g%+w_%=C82UK#p>zYL z+d>zw*%=ibwB8=c2p6Oh&QX}zD0*|;>US`W_c`{vwCY$$X@3qWE7nR;0jeywnxllB zz1C+5Ys!ya*M*t5K+sjG@>;Q?xcZWAq=!0%9$ze^<1V}N9ZiqIz_y%Fkd3+(U5SNa zFuyRsM46{*3b4Gefq!SR{EH@)%sUf{ag@+VXNjdQ>hQLa&Z^loFKkRx;|#Bw{DvA` zeezHBZ8rP9=YRg2y4VQv;?TS;&lhzKVW(vH4tBSWu`3w;DY>Q2h1>uJoL+R)C7aiT z8}WGAcOd^^NhV9g7#yQMfQ*GxOvV6nFoP_%{Hl}0p+}vYV#pu`H@B|IH8~vtsDFbS zG#7ZpSNVC}zC3k=s&7a1p?ejDMl>WYpaeJ*h)MsNh<_fYh)<{sX4s3aeF8|->;eYo z%DGPi->9t_WaJmF+S3FPT1k#-Bc!4z_5B8!X1 zQn2dbiGODf^gI-R86=gQ=RX@z!-j2DD{8Av?Y*hJDIr#I&oNL9tC64ag%HB(ZX z+S;OKQJdDL_KI3XjW^Hxdp^8h-+$tq`<&~(FBCoypId|i8Y&4TWjh=R0>fkD1}_*e zk%f=f->|lN&uy(tVza^D&-5BiH5uLUZrC(UpbQ>|G<)kz?Y6V)!ke@~M3K1=Fa1ki zz33ckmK909u>>+?ck2c3JrcZnpKi^5jl_4Fqj~*r`e+wP|3g#j(=OmHxB<>R^_2-+ z_R1dC>&e_Zt_}OuHI=5+>t`rM-j_{>2&ryTZ{>Eu86v$jhG87tru+HkyaBCef5!GkFY1dYRf9pR?-ur z6N+pXpBXT83uzLqgS!z`s8b-=?&E7MZKH*x=@Pof{X8W^m>>&t3b4m%4yoL#lH%u**I3x1$^5JzRIP8h1A}aI#Lj z;K%U)&$jG=S-8t@a=ma=NJx$_T&8^yYp)No0(O!PglCT*E#GFSB?+cehMBRpv||Aw zyk%SWq)yq3cyzL1#~Td9Sf+mS!B^(uQ|}D43!&5o?Dngk$KqoGyr3Z|?nIWD_zx$* zcz&ZnOT0`or|dVEX)u0&`^rOsm-Bmnf!*8wkpKPmFC?SrGoXqe@`npJrQ9yoc75Pt z>*nKsi%dy-L)=S$f~Q?tHX*a6o%P_QjM>a5X*nL?VxLdp^!Y3Hpt&?*XrEcYa%>O6 zL+0*ySmIE}=BWI7P*EDi2zmPB==*Bb4r!|_`e-ImJKbP>xPCA;W4Hlx@TgZ6M^1cy zK7&kzwR)M*kjwAr4Z$f&o_u`D`f2!<`x$sL)l+h&$Q?Vnyn@HsEN?qp(MoJ%n@UxC;}4+ z9r{7RAK1dAM;3Nv>$Uv!V`M{>Kd8FDw6XCC?M$ZpA|e)00>m2~&8{QEZUfE^@dkL{ z9O+tYK`f@qQ>}b}o&wIK6&v3_=rH!|g{zFm&kSs^HG_Y4$X00ePg{+4-)N2wh*Jf_ zLFeb;AIHg6Nb5m$KMIB5#u+M70YL54O?DN5vG&ILlU%O>*j3mEg-4nF(g9Pi3v2Md zzqlb&N&W#H(D^&MKCEexHS9)CHyStv-%agjYbB{h_~OhGU-ny*RsPm(sh%&CB~%#O z(0ow>4<6(n$WyrzQ#vnx%xgK_N3|k?*ov50a@Yx@gx09>%%A@~bwg zt!=*k1ubK|Xe+UgJ^r80#-7K9!ZqH&6fov&0G7!th-+ zXrS1ivM&{IgkD0#xr@FB@YQaUw{}nRK0SKO1Um^Xb^y!XH>r618dr&Nz)tl#TNWG< zuAxINAL?^0_I%VaO~2q}?rwuKs1L#(8WhAKPkW1gGCzIGYm<@VL2kF*#(Sz9VX(%* zM$Aq(vhpdUxn8aBaI(15%8&}UYS#v&rinW4B)KS9HzUG&9=>OGl{ zb;LB|!mJ)(c~6uXdrF%ZeQ5DVbP*WeZe4^8RzNbDIpJ`IxZ#I~q%Fa_zIT-@$?rjB z2o2s36+Q|`c7Gmqq^V4IkK2ptV#k|{80GpQ%?bW|o(b;uGQ@uIX9wy%GqjO}yt)jtz-FNBkJ;wH@}HzEVmBH(eFF8aTlAX{erTP| zgQgF$+e35O%W~s?cdLZls)c$*r=XNuA$g65pCYGU1+@ZhFW}Hb!pyUrL}G>0O&Tg+ z(i$HY8EV8+PZut`>YXMMImTU?{=u`~_5jYtq+JDJUJ%#&!&7v}fb+6}HvwNoy!`mA zD0z7sj$rZSke!P<7>xP@IAwtnQ)1U#tFU-H254eoG+vnyl$($$=rIM#^G}hYs)$JX z1MqhfhG0ZF94 z-R9qE!z>J#PZx*Fd}pvcJ8dktK{YbG({?=ZBL55y0v+-s`tLhyx4`I{jEF2|CekEB z&@(mE|F$WJ6LcKOC>PnDStO2eTDc0^dkLqIE8@gHMu)?K&h#UbvIu+cTGHB#(8!Kk zZ2|Q+q$k^Ab_;-5Eoz++99=>4d#TkEGv!@!;7 zAK91l$0=L`DcFE3_@>p#BvW}}@B*;1&#x<`+zEK>Im_V1H(*aIr5TpgY1e~3c z6XL_sQtOq(*VL)8MVwwU_7AEAwL;Ca-8IDj6%yWKQWm%%w<7u%#SkH0YdWg=S=POZ zVK?ZT&GL+!)Wvg} zeL3@GsY_j5yDRmKa=87?kA2+U+rMfTesNHr>g&t<@Zn&eZhc*ea zjT}xYoVU9%v~(y8#}!hhwyLHvW(S#j7PFuC9C*+|Pn_SeDK9p@`AD$>LJUp9y#XcKUP@-3YW0qo9xwc;#I(cVCk&}oHo<(0>fw;z{UqlFM#Wi_PYP~vyC8q7 zPa|RR29hN9`q``<(U`uP!!EE*3QRo2eFaQbaT6ZBT9I&mOXF~Ef~cM z#1mAYdi2#l0s`U;UZaeVRv;~j0K)M%XEqMGhIu>@P2;us{GGKx@+pDF;Cc(F$FSz~O}DHMx5 zBQXqyz9Jd00F-OZ?%E+rVPj{~_(C`1N^MjY=gtvd`FCywhCj!CY!{5!8+F+vp7bL z%FK%GEYdvl>s~6?){f%QSbDtOno^R@Y68j~L037K`M-?cGE$WVe^Q`mQH%jgFv&NE zoxXDP@{f+%*Qs?gTSU*myVJ?dk2Wo*Uklas;|e+_Jdt#Be>zIT=^a9g)H_5@5bqdQ zK3GKCJC%vNKD!WHY$POO;s)IB>=bl5rjLjy%#|kvMy#*a*AJJMe6RA+1Qd|j$C(hY zB=PT#rJ6Rkr+jT~m3fxLK?9v`)X()4&A&ar|5)BT`u4>ihvh>NILipJ>v4+FH1jgv zP5huFCE|NZd*Lxn6xAc_J?iyp@b#rI-_~i`Ug+*GMY`fxH@Lf)kO##oh)hsWOkiU% zU!?r4?CeyZ-;Di9+bWfgXYPK9@enFb}<}F6kz?w{=d?bHaoHG=|}8& zR&WzS5K7$yj+_iyXVz{3IxuZ`sUBNPIpixdDc%~NJ8RM8dwn`4xNse(MzS@n_uAHX z-t(5?H2x!T;xwIbKNio2vb-b|CUyk83Wv8%&wlYe z;kuu0oZcI8@bl#d{OGu#;_Y!%NfW|}QTlx1blj%%q%_;dmUwogh zqGc9j25vKpl_H=Dc3wwRC+*|0YWbRM?e8q~`6o47$~$fKTxww>gqb-%(*GNfa}K}3 zHRYPOPsQeg6zQA-iHpt zV*-~|8%Rr2-Qn2vxQbH?>{W^HXM9PBD$GT-&uqQ9UZel|ghdcU~4(2s*Q)P&k99abmiI}YN* zPYHG~n3(6J^oL8{xgG7a|Ab92os4)23CFKYsTGya?83Fc4TG46O`-@@#ghxE* z(pw=JG{whIHol2~oz4sXGpJy#Nt@*h2ZNP1zxF@lBT+oi#1}jH;I(<32Jy8~i-@1B zqNJ&QMUNqvKEJp(-#wB%CIdxA?g&8Yc@jX#9vxQA)29%Q@4L|#UMHkJ@Ek$kDlmPs zz*L)3z8F`Ep|7~LwXCM9e>f7Hj)nVyri&t08 zmBSIkMju@HfZ7p2l>IBl4kA5|Yyw;Up_<=;;2W+HqC~)~{gs%t;7=M3nE9$PdgjV4jC(ANjh7PY zCJcPK-{*iT@*iI^397cMG)$7;xO?^nVxN{q6&O>s*xkgp-q3X(9OIGdAPciI`VMJ< zALYm(O79N-`TxL~#9prw1r|(J4e<)?asLsD5B4;iDV>cSGC3PQ$|;r4V7f=hWnQEc zA9*Yvq&2N;V7WY-I1r_}fW{N6tSJt5H0!qOs%ht(lL$Hb?BG3r&syT=66WmPg94fO ze`D`!b{LSg@_v#Nvgw4<LXXFjaM{mRxi&PV}3VL)DFAnRI1Wu z6@PA@Ha-?cCYc}MAo?kGG($nj{)$5?tSFx92)|xhxmQS_3#Xm=KefJH(t>B&cT17+ zJe+D8Hk~UIGRNQ2Ap*kiy&GR7$Req2Cx76-r9Gx(`S)WVRCEHLPbA;|)NTHU1B$|W z3}uDliP|+^`=*O~zbIiW1ApYN_E@&dJC-}gVfkzG|9vKsVgY^%Ryxv#fs~vwv9^d@ zh3}o)3+lZSGcUf`t($^hI%-L1z5i>r>@pR-Z6w_C*ME8D``2zZmGU#9>>Z&zehwfQ z0x8MJ-Mufz?OGr=?N(Fv6jd0B!4rb>BwW07WGTUyM* zTm$xycIav;hZZ`?++hg8^OA~rlim=>&2VueqNwt0@=7bX_w(QLH-_gYcf@#jcXxZ7 Lep;&ciShmi{4xkR delta 21207 zcmV)!K#;%7$N|mB0kD%1f70MDVlrw9nB+N(#%01?vNKdcPgBg+nhVt#AoU(JX4-sG z{Nkrf4pa2s@SE$J@dQQoL76HUZ zs}KpT3_PO*0bd`KJhvW*8-bx~x$lc7{%Lfew+=6zB>So4>PhD1isP{cUCg%Zm zT4bd9gb4~-Ry#)qRkH&)REwl}hZ-Z5+M8Nfs>-0*Ih;=s;gWOTs<3`0xyLelspVe?a6IwsjLxm8+nR2GtVkM80MV zbX?V7H#%-ioC$!@9gh(~f#&JQhZjoIExMeF$K3&wr=on+Vt}=9u6uH-(3>j%b-gL1 zq5G7KXln)$w_uQrI4ZhW0ZO&op(ARR+C^soX&Tbe5#(tAe+QTd5Gez4>w8>LxD^*c z&#E;Wq1Kdo=vMBT^cy2qA5fn%=yjS6?UF;TF9yZ=Z_139ms}?3I{49x{u2C7NQHpj zVpkyPB-63b{e=iS_|c1;5U1Y z<2;3#HT0vWq>ge8MlZQ|TB2N~Hf)pg#dyE1)XyTnZ;fiS(lN8l%IH8OzqA`nUuvy4 zBQvw6%aD4q!pq*KQmPf2POa>cx5*l9FwCSoN4J|JGqVfVSRF@R!kIA(KbtP1oVwAp??@|X+O4^a*Q!esev-k+O{`*u z%%V&#rwrc-qBdev3DPf$@lEr-dRu$>SsPchf5=(N_@vsXGCzfw@cq5#;F`Z%Ih0V?rxoEG&{U?UY*#yb!yop=O@#%?J+|m`&n`;M2 ze{1O7SVOOkoLyilgR02cau=OWmvE_8l^2{fKL^iVsDc-WEoLjKXon|AF_DWHitZUt zl~1)iXAT8Y%$+p*kc&;kY2sUL51u>8#}O}+;xSG7)&2XJtKQ`y!Q#l|xhajSuF7P( zVOq%EUhNgQGt79}-rH6aDYR3b2H@NGe{>XZ<4{5$)qNJL(2#v>r7ys2`px?n89KO>r7ys39K`LbtZT^&IB8#3caqq>fu6p`YojKEXXi8_Fghje^`Z~y{IIXFH& zVIAl`6(>Wf7c16#4lgRs9Qu`af0)IJs;u-Iny*sTxTIm=)G8U9yRqNY-e(mHIF4q! z%-&8KsoLIg33HQ2(rE2Q&$Qffzh~%QwB=KDRhvPAq)?R|qytuv8G8vw$iH8)Wq!^P zJBQePuch+B4J~V2AP3=FQ}x4I0Uoh7WEGvKX$5$9iwHNw^@bGJDsn2ce`EU~w@WF3 zf~mFA0b@0;j1QQ9ddi_ExNvhC9;fbRBbwA9z_+zg3uLIDngB><5HQzDmEW26kKEF6 zg3_sIW!jrmh_@yI-g3RyfW@VvycGv&niMBbV1ycBDwJD3aEb$_qLx%4yN#fd`wC{> z-TLzSYBJ8*gsW@n$XlDxf5!%*>oyQw4Hs@0gieQ&yN7TRQ7nlhszVa2P6l+y$Y%MZ z5Qt62f~X3#rkl(L(8$E6i-XQ3?*$BL8+jmse=5L0GoMl$!mlGVZA19oUkJZPjBa0T z*Ita2eDjdqw1l-$^(s7@q-@;HW83H8*$c*L0LMsY=ektC4%y9Oe?%K~+~H-5)L2YH z{KXU`Se4;1K`uzf0Y(9x=B{8!(AphVeH6J4i=cJwAa?1cj;i%pM0zzP%|de};y*fQ zO+ORTF`rfG6TB-v%L9QcJVlJdsV_?GZ0|^S3oNM7KQGbWpJ|J1{!ueVhNujsMl~|y zHLTGF1+>H*de)4Le@)33FtQqEzy8yfT|2dxiZ9TV-lC;$0j8U%l41H%-IhTz&UAC# zM4inMxab~WhJZl+(6<5AC^{26$)v0-%09;h;9Jf3WudXq%QYs$RwNV)v3S zfr|p6^9lx`9vQ_Hh&t#{(M{337*FWlu{w{`8HAJc-E^cAxb*>KDJ@s$aie0?-6^}- z*?f@e^ckaWHbYwP43iu7>+a64A0eCks($V5%3r;-1z4UxcWC}q->P8$eH;eii1@~i zG>fv+S-Y^ce+!p-el-6?4`5C%wMS)SN6`qCe{I4YTPKz@zO}SzMv1-JphS_xyb9)8 z6*wg<9F4Fe6WfKvZRqDAFcO_@t|N+7e<;p&UcIUf@)Iq_uMoGlN+wNI zZRvQ;>LUlmkBxV>Xu8`itcf+uLZU_-5=iN^J-_STEI#JOBrLT!(|Fc&H8aor9z_B6$dHdxZXFYnAsKQ34N zmL^8nfAs-$ezHt)C*<TTLacJY>tDcWuisAMNjyVjexD{tO)1sZQ^e{c26m<}FdY&o!zw6Q9{k@V0l=*UoR z06bE3jBtQnLgKz0(O``D%LI%NKu04F6BPta;C+orBtW4d2P_;i^miy1W&oWVOvVXrWRI5!1=Gbl3>#l_@iHNuNyh?h5;ti2sJ>&%rbGz=Ak8JsDlv;oNvWj1ZCAszJoO}y>@Tpmpv z@?R75-~akouiq=pvOSYJ2`qnJQ;#7v>ZVvOHSQKw_57LO)%1imc0ST`Z_O21EO_7V`5*uOygZVq+P*-qj@evu?-%G{WNvR*S1$0*HuLa^}hMGZZzmht4;?<@k zeA|HwbkUbM$C{1UN=IQ6`UM8y57>kUWN&cudOQe8fZTsxkM79c_^*FggMmAsdjtMA z+4Xm^e|I+tZk@?3>iztFZ_Wy70aG!N@k;~zv%R8tA8QIyb}H@?r{rkNE0=&NQ5y|_ zN)euD=wgIRq7KWE$#zR1k{U++h2kax8zAu~^1pnRUi878jV~f(Z9L4m7x?@L;#%x^`B+MGR#*fS_Bc z&%<68cSKqk=W7~?x`d!kT#c%R+Q@dFqA9C3%X2Ap!5}0I@mMXE#(j*u4)5w%ec0;5Rv%tx zeK^k+4fp6Lc=vxEsqNNqF-JA;DTtRnBk0JZD$mI+qqmISGWzvm^lPdZ>{SFC`Xq0i z9=h%k^q~0UO?85N15GkWlv%(s)+q4I;I=`Yf*Mm-l%=AHCQvfK#Y#YdCg{3MUBU;P zs^c14>6_hy60Xk9zr|_Z5?48WQD9u%Olvf(|9}D40c& zsI-6cL;O1%2r8BGZg00QIwP2j&*>%C{m48K&HM$jRQ%{+j;4!~EtH3nYIS~oK|^A6 zs9zS1Nk4!=0OxU}C9`Ay^Vd5Oma zDj9NrJR^?R)|g-uCek)AS|rGtR!ro~{c(SyvUzBjQMD+3+Oece0|q-w6g`@A)OL3* zUILwlk;?+ZDzC@XdlMA(5kXh2(UQ!>rl`e3jv(T^J$QLXcz_eM@)eUD7)kAl^v|yd zk0)HaRDRSyG{{jUWYE)&qkhVUwE(<84#K|Lt=%-AmdxF>cyxAoI35RREYS7A56yq| zcKe!lmVTEUMAT6<$EMZa)8IyObWd&~l_5<`+Yw|a_SP2dvOHA1M2>%xeb^L=rW;N1 zlEh#H7loAa^F`A)7RZiJ9tEAo1IGeR^<-#{{`r7kLhzefw_WfSBo{#sM*It(gmAeh4-9 z&$5&Z#b3Y=*+i-`Ob!mDLgh`-RMhA$@Ll;t8c9{14M($pZ}u?W9>#y$!}z9$@hM@D zQazNkX%C`fzE!f{7rK?!ZSf6sIW4${w)_t3DyIh(A#}5%rz$j2GjXk3StzyZLJLYE zUM7^xCAzLqE7(i%LwhNojC{dO)lx!paz7;ZDf#b9=3D7)rFTVqrnW5i3CHFu7yA`p z-eP{sH8zB6Sj@l1SaW}`B8YVs_54nF5Hj8-NKtt^_hBW9J6o~2C_N}syHP@$sR2pd zWewwcjD~@OI!vfwWsH?ER>nML8Ds5A)~-}1>t9vHr4{n!c!He7L#&mU&oogZ;dPPo zfO4un#{dNedsbx)tF=E_mg6hNi6-)oVq4gDO0f`ubmfTjHy4m=NEuiYFWk zoO49P*6%=;=URUt)%YcNQ2YgjNmhwughhU6v5qX-y zh%I-u+;wfZ>+7o6&hd`FK)weZ)B$(RwPbn(za)Fj^?G0IwK!N4_L?WQMujSH)*0p# z5U2*Gh;VxhZ}pKE zP)tUvJ+5A|;#0Ywe(5c=|>tF zAI;w*bOAZ0;`!E~-#ZUNf{rQmTskAYgZ=IJzXaEPxkZ6ak}ZcIYBqG8t7_^;2^h-` z)AR~jK|bmS@x89xNKJ%$$*h$SocjbuSKcyT=*fSjynM4$735fNU4`A-GBH`(-1-<| z!uR(~c$sXtoRw|2VPp}Z3uJoSvrJt^&>aUv{*JPeF&{Ys%esIe;-X;dntT>tPi8br zMHc{5IG14vh6q4Tr`Xv7zcZvJF72+43F8R5-FXW=3e%!m2nE{X$E%4-u(#VfB*B{Q z?%jW?2qSWtDNNdKbL+}gdz%+RhP1qBqWigr`z6VU7BGQ`qzKL=*j@pc2;+CRXSoe( zVIA|W9nsnmt<&WC%iKX-u!i<=8tOtcRJh)oI^Hjc%+T7=-Yg`8?EOCDFtld;8TK9T z+r3@IFOtYbid`qaHQy2iHItA*kP;^QHp3UJqn9yHl@B6Dxsx!rVvwVLPX}+w$?TKx zVk>`>do{ydOhP+%*?q$+fmX6w$!aC5O_OHRq}?yx*{chBv2?r(gHRV{fiBuv9~MQp zXmf7TIEF=<3S}&Wo;L@O39uxMS%Q7{C3a$cxrk#trGeBn$snnzygx5?5GCFmgkpEi zCWwGovi+Kf9x%ela~XgEBERs&#Cc7w$ya|-DQS%%E+E@>zi7yuVb25O5HSXzFF0m^ zAxB_@&@~YZ(FK&mCqd2xVj@1a;=0{AR<>{edJ<5t8P>9>$)cuZ&cbXgk+P%edV{6a zTCLV<`LE@_R%=~nW$0C1RF)0`vKYl;>A)Op5`twZyKsZpD_MHhAmEA>#a0wsQEY!j zu@%MEfNTxOOJk<}y685S$P-Jk6$>PUm&@I@yR3h?ZI?M=PA-Zqx@~${f8DOJuulkFU@S>GW{9)o zhN-?*2RJQUzhj~Adsso)rZ?|Q&RXsDI(xlt)z|CnjpVbPH*cz9;4nNsblpSW59n{yna>kY+qmRe^!WwS*wj@i0;ODCqEDAl*3{`VgP@F;0pGf zr3k8jg*xU3=oZtEd2=9`31!FyhpsCzbdY%PoN~b74N^>fL_Lqrgz~0%@d3CFx3>q5 zG*B=+jzf}(r>L*rs!NH{JD~!KwlpbV%EsB^F*T8q{q#f^xF(qrL3wus&6Jrw$UdJ< z5T77*V^ZyWK}xjobQK2Jgob~f3x-JOqyd5qxvHJby$o5OoOz9@EoNd~-^-c_)x|5O z>paJ=9u>9wYZRQMERb@!+flPn%TMn>FCEXfMV@SHsj-%tPAxUf<4Zqcg#UJ@D&~;F zmdB7dsHpQ80(@0_#rS>+EU)P663WLTE<^IQC|=Xz$bZ%W!z)VKKzM%((k)20Abo`( z-NIfAdoAp>u-C%gwy<}%DoC9IW9JIvw8g!;`m(^c_-t<(w5zG=67D7g&Ewr#tQ%3H zp(GO{6`)nSUGgqT0Cg+pv8k04_Z9f77q@p!kP-e zvW%BQa~Fr$+*tsCt#E%|*X|0Xz6v#Hug*E)WnxQ=pj%{A9L@m`AYsrEvkgv9I>CT( zf4R10p$I1`2%4$?L^0cAu;o1h>}G+J1x^+?S>UvZz-g~4Hf!jT`;M@1%oTDXBnI2Zx;4GP~Dt~`cF%mI^g}5!EupxZw zr(?Q>Ou)1u<&zxj(sx@YqJzjBC%6RSC}1jtoH7Jn6V(&+w!l>?6t0chKS9T=teGN0j6fOm5^~FhE<~BF@5TIWX~e4UmIC_ za4hGSvDs%@=+I=JGtZYtc8J?}y@Yu_GlGr*mzif;^_B6&*s3Z303WUU?&JA$GKnB%}AisKv3Fa7jUKmC2y|RDLu(Q3hy|a?A7gh!7UIkg3Ikf=a z0(=YbEx_MIfd8f{ct6AA4~QS$LhQkzhmI6>YlZep+G%o{!Lu1@o`YvuuwReaOFFN~ z0hG}Yn&SELaW2RLR6$~t5l9-;X&{dX=qLsBe_;`x27Cf}m1`-L{j08lyzhad=Y%Jv?dG?7=>PP)5-0N zmgZoFJr58HJW9qWxF*o|5e&ct$s10V$69|LyC_XtZm*Q3Xe$<3ZF)9+J5j0&9p$)D z%Hd*#qKTgJSd@>A&-S&uy|Y~_?fyXRGN`9(FCpvX@xamwgCb*vYC=3tGnes%q_kQ9 zr!*uoS?K%IGo=LFs#2JwPXy#I2L&8r0(=x;>UMJ_mvIl$&EYPYR4%Z|}V zk;~JUmTkA{9dHqc*kj7{l~v}f(q%d1A`Y2a*oAq_zB4(QJ5_!RkttlAnOzDHq>il* zba?$MIN1@ z0J&f|2b7>|0zE1k!F-bGUeABXQ?+TWGqsQXcC^iDWU-{hk`_x^EZG)IzN!h$e>sEx zGI@-h`Peun3Zwj7pH))k7_Tm zW=U$ze9BD=E7y)%oACW1ti$U}?IQ>pZ&AKQ`4;6{l)s56f4?T!xj=u3O9VVms^2fh zZ;R(U4v?N1ei@ptNAfj+I?Hf;r+y*L!e}mr^A@mMz-|HiV+QON0$T`dA+UwOn+Snl z*Mzed0BadH=v4hxxJ-0}e7p92&W_sLcOp$o>^i|_(Dwy`-PKr;MLxhufPu8% zb5f|$UFQs`g0yn*b~Ql(l0l>hFN!4mJ|u2)L{`#<`G$LOJb{EDFI#`{2Fw)i#P(J`47&}bJ)-n06^#;A#Y~`_ zuG>54>4X_N#G_Bpv`jDQlmUXjJn{Z*t0SE`Du6Q}ZX}pu@ z@Trf`wEtG>f7v%)K6K|}781J^aXL#l)3z8ib5+2ttfnoqHs_V=~MFbW9oTgh^=Iwu57L$IoRDg#_mE%rW>v?W9yb0 z*|2joTXjg#ttYZTeDZvOU80l?WZOE%E(ioQOr8Qjz>qh3smcs78LQQkM4v-0z;SC^ z;FJR_&km+4k^}_$;HnQGIHoRgz_Zx|I}@M-RV06b@*MH@r4OAO-xqGf-TznFx zCXh|GvSyQRB8IjNO|`D9WUnGfq}*8_P)j!Q0hzK8qwtU^0DVeEbRo0(fGQYG8;q>R zvN*2&!yJ^Y9Y=ax82WKUGps}UVnbBod;PK*s*k~X9wv6dKko~fF9UHm4KBW_2ptx& zH!*)#7YMrZ*33R`p^EEA;0`=1@cPBPJZ`-PJ?4>V8#bRF#mb?iMPm<$#>yCKQ8ul+ z>{?THvtJQiMsBk+%vxefQL}`-$Yg*`nTlwuHVxp?>_k($Gz%%uytZhqKB_$WYr2eI zS6rW#pZ7DGox{06_U#sPQh!-J>Dza0AiIC`ks}l(-L8?Ml#tP#zLXi-VztTYL2E{B zQ4h*2u@!KVTn0y(1+F>be|N_N*w8ud(BU*_-qi`@u(db1xsQ68AK)iUE)kdYOg#Fo z@D!a<=jIcD4s(X*XLhI!%9c z-4>haY3y6+Dr%TUHW;JUpU}J?Nhba zzVJoeEg56b)lF!N;&9H93__E0!r+;doeQ`{LRAg=$aK({fn)}57K>(68A^Yxxv}h1 zH?x@JnMg*>&{={rx{7|5*W1cMYQzQrt<@*Jq+)U?4Xju_R;~ATGl4)NQ-+G|k=IT& zb&;yAoQpHcp;Fp*YdUVoJFRWS+E%P>5sCB3A zUbe;&#}nk-T>6N(aM;%PRoZ`4*H|{rA3vnjGexgilFEUs`d%%uYhlb5##$I_VXTF* z7RFi_YhmpD!Pp0(s0^~|(030J4{u#@=%6n2LmEiQ_?63JP}$6~`184fm_`Uwc36%s zmw=VgFl;af3neX-v{2GQNed+(C6p|U(;Iq4n4}-3oNIJPr23N_IgJ zkd_xR0i+RpJ+Roc2AlSKmlGQB5%jz^A;MCXTi(iO3OX%1w&>WRnm({d9!|NQ{O{eVz$!T zN^dK@AGP$}*oY{*`*m@|q7V_`aEg}`;Hrt0xqq47oXk+OiVKwa;ablxdt7Ia>nx

NXCv1Lx>3ikNz`wQ91 zV!oK^v;M?EmTAEh^TRS>j9IcG<_J9x5K3O6J726JpkMO|CF&2 zHvoxA%CS!_^L%&5Ix`<+Wv(MqF>jrp*K*U&4u|BTD?5JS!1yh99* zhBwUI{I(+2o@718u*Ch-h<}TkRb2ap+GL*)V{nWrxb}-va!Z}X-i&c?oj5V3X*Xca zvfm)oP*ri%sT!74w<2DRCEpf9kxZF}Ds7Q3v^vIxrZ$0OjM&ySxh6@=(DyxzRO3;z zoXVeyZ~uS7fDZ$q;xSQa3tUamHHnI1CX*zoR$M^IOHTcln*HlS3}B*gHWX^;OH8hb zYJ00Qp^*LFrv7$H<5gzbY9~*MlmvLLZ-=^a}BdheO{*_Z*XTC?bE>*fQKMk7a7yEvoAIGvTl4 z32oYwxm~xB#WMPiZHW^zz4{FXW|RLRZQ!cGTA^RojUst zaYu4`jv#+S?-4pubcrWlZc*@#aXf`$?WA5vqy{DeiS8Lam73U5$UOHckudIGU$Jjrk$}FapG?Ec$eyCUs} zva3r&yDPm{TOLhknplvETOMvCX}PD+|E4~~cV%*ujVwOiNcUKCE?wSy^Hb{l2u)5x zG9;$YsQCl|s2F4+zxHD9}(=(@}4r99zhSvAH8jL(C0%gIP7@_BW%5Btp4a`8($B}ArUj+6HR|l zvsz2sYS=U=vQv?eNQq1wIH3nQCSxI)y_R0Yg=VTCNr~g|5N}=48${Htqs2HR-&LLe zaJKChAf#km)tB-s=u3MNqdVwu;3Q=LQ-$aVL|+C#UT}`vx+Yfwb!c~rP>2CY=CcV3 zQ0zSsP);4{0nCLA=}Rt-8KA_Q3kiP#qQxk_3=6073wP&XVb5_i>6)7v!E~HaGY!M$XUAck7@3-#-f3iWDKbdY>_VOj>s_GeTu+a)6it#*wca z`G#mj0|aiTYOHeRwF8}Yy+eP|Y3Xy-mO4GvdMyb3aR6N;1_8-jO{3r$Zs-ZQp(Yfm z)k^~*KS=0#q9p)^9(8UQ5IY{ZLO>}(%!fGeQ2ZO;8v-aY^!=^u8Hig zY=YDany(q{OR-|J6Lfz~Lc+27T=7uF4i|$^q)OM)WMZNnBqv`if@u1#Nz&m^&2h+3 zAi2rs)OHl03Uds->`r$i3xH~7v}fyRgd83>cTGw@#!H34%3NOmlqCKszL(&Gf(|{W zlCK3|1_@{SD~V;{q_~hIs-MgWQ}(GLlFe~?O%~3#9QmgTqLP0ROhd-O1m0?XNrM16 z91x*pF}0?~kX;k$Si(T3p1Y6!I!7KHto%lTJ$|ysPaVf8s=McJ6~6|KyG0G4=)YT9 z#>7N}( zXx$JkGhRDpY~6nlYtH^FLD8mp2kccO+?2}a36fv;7P}}ork)2m3ZQpHL*lmdTdZrb z?!4_Bx-N3hF>=r){zt`YP}$u}586aG`Q{g%wAIm8<780QS94E;1ruX(D0 z6KS=b!aLx^1px$%(*Ta+!>r(%IFJC$Scs&v8HYFeoX>wn&;tzrqoPdQqClCI8HYS% z0J6k-%(^uO=hngsEy9D=*`aC;!uOQo*V^p5vDuYv@WO;v6f(MekK9>_VG`s%RLl`pbx6-q$#fxvU|ac_Ve>|^O4Ezj_y_?%L>$w}0_ zy+awab^#jegiU5Otxi^UASfS01ZgWb&E;l_F%{-9`_4d0R*S^+X~c#Iru~ZSX_9A3 z&2}llly1S~?pRSE{lzHRx_~on4U_>+m55p(fYN_m*9JG0DO?&$qTI*~wWbVFX(@zA zQ3{EnYf%*WVkwIQ7zK1{FMn;q6{C$6sJVUH&J(H4$O=Pg>4;Ju0#>7hbpQ`*Dp6in zB=nN}?*roH_vMVFNmP5e`~feT+(nMM>`sMA4Z`4}aom5}q!uwh?PmN7Q|^(H(GAs| zxu<`sgILtuz(cI=0_%(B*H$0nWz}ZpuMNGBx*@vEo4V{Z9}vHQ#HCY$7<(Vk=@EjC zgj1+%;B80{5yZcy7yMpdmn64|J!y-(qoOwcmYF(SNNJgV!)RytbgXr-f- zj#fIRrQ_aqT~4So%=j|l6O;Ppfr`F;Z<^yY>819lYw0Yfnb9l=Pn4o5ZM=+5eq?`U zK6HzMx%Q{PBpq(Rh6;Ls1b2%PhL)A+`ht097m3%D+w9cEYZZGY3X#yB-$X4`W~9>v zK0#idMpWYLEEtk(DL$$ufjoYX_*fq>PJMr0yvzhsi(?0Ml~peN%?u6lbxCjTB^_JyTPil+ z*F6e>T2fVOQ+TFTX$sEdCgdPeC@ z=%1dnWJJ35a&A$L8I$g{&Q!iMQ%Xi=;42kV6d)d8bgOR4HFRV;h0ygw>nLS8 z~jJ8tl!xAw+PxN}C$G&O5QJwE9>&3pTGcbe5#JwecGsa>ZUFjucrS$C59hRj=bWtMF$snC{% zN8?(cXs0QQG;K^qG*BU?036bgYgJnY$yhCmwoI>8o~Yz2)}@Kclrn$I61BTO*dK`J zVjz@3PyUUOgG!UkxAmP*;-4JsY!7y2L?{`7MG~uL<=DfB@bq~@6dfl0>p@_;o{@GWXOH9xuaySi2x<7-A z|L(iuE9c1YAe*Eg{p^1*;Z^!OHmDZOedAjRh3RS9Lh_YNLTKr)UlLz6y=a@=c1SHw zqp)l7qHH?UvPIUkjr2k@p|*U@w+P^t+OF~JM=^fNBTPm+q`@*Q?ir&o7Pg>Kt<*ML zD)rjN!n4aIfntq4)!5w>+M@?X9=*@OxaF%+61;OseFIu#%cuMGNh(YOzJgo zh3ovJvW0i+3-7Ab+nRz6L0+ZO+a3ORAcc5idGMRMyYOl`=)GL39W-FUsb=beTZ;Fm zCsh~Y8|%!eUc0*>Up63PLuZVoraB6v%E5wnOA!&iWX@FK!eNkHCp|P z#=5*!;2%8|9+`fn@)1W_h^42az-+R* zH&=1|$k;|=vS4zt3ue{YFH7-RU39g{KIO(}kabR}LD_%AawgYJ7iT=nfN1>nnsh1+I`qAGm@?+ns~Kc7t}*ZvRjBfIP}nRPIYBXEr%)8lOsGnCEO52}(s$DrHeaX6!TWwpIa)O10#g z>9Luz%Y?A>{p&akIvE2uE-|7($}3s2Psmq7MF7qCSUTYVZq=Dsl_7L+CMxMKMg!y~% zhZ*ZuUDxpKLc1;5cRWT@-mNK%88?V1iOyDK#Ec07BT;Z3+s!hUODr3=Y}|y67gl*O z*I|E234{53)!$iBbd5D-mxH>X2y$bW8XnD=rKr9ft5g=wxG^(EDN5Ek0%qH6$t3mI zOj)F)spJSD>wsc;pyh!^PKVhgT=X?8yf^VZ`7$F}ACIpzxXC{GjAq4$Vp|8)JB*@! z^bQ0-j;bF|AQ_{q>%}=j!KvFj=3t9%>l(yIe@}!S&&jWN!r&G)o)G5s}-~7O&g2>gLn^ez? zVSvV3(z7YBWEtSgLg~cvk(H47$WecJ;-iBWrGGwAoK?Bv3{gVrxcE)}NcU~-c(@y{ zRng=H%zKf>P6Y}8qz}z?i3{X=&_T!0hYsfRXukAgMR{!ZlR`i9D(3zr@8SVBgaY^G z3DMMtpfFcO$HU5*YtVjk4}hvqV(O~U(6b@6&eqHsqJw~_#n_%!>9VJ)_k(}W_G=?P z37Hgc?q%%rNh3{AScnPV->ZOJ-@eC;E|UY{stQtM-7_qg`DxW#3e`*MfF3AQMn4yWJUgKI5H*&CXf0uDW|bzNzlnzycg zwhY}D$f37LK%<{G--M#xOzKg-hBV-5EpydL=jpa;c5|)ipk)PtnJCc7*I}LbK8}mpm zzsU4?f+XL5PnQ>;6-})41WlsX^Nq{{GuqOfbB)}Sr z>`B0bW3#ulvCtG)kT-rRuWK}cWSN$hv~9>Xw#*$-bG>(Mkb8TQd zNp@M$c9MB#f>DpdAV7rY-A~2Zl|hW+XQ#wPcM9Kq@lgUDk3Gsz@@oP;50P>5L|V;{ zLrzDd_}N$K{3km}u8PX0ziPKYv4vEDY^At;X)#tCu4_5uH98<;L4NX~5 zBgU4axmnpW!&W!V$}w0NUupS(~R#NtR_sIQ9;j(MFY9{?Kd)x}c#jQETOH=o73R>&yTKnXB<4xYJ+!KcbCL^P?gHGrnraC0D&rM?Ec zq)E!Cu;PI}njxx;`kT5)@GCYa4srpfjgUwg?=OE2xA4JAix}V(xsH0NWv}<-<6if# z>cZ~mIr;+ytn6N8lclfQFi$EBeVqAFx&hN|p^Mk-jEW9gZ;xbz3sMQ^D9mgWy*Y06 zJDA4%9Q$2bbu6Sbhm;j-rKkW^mRrqH!p>gnvxGI}N3ZL`Ok5!7s#JNcSW#SkNjK6% zokD+)FBZ~qm)-e}rbl65TTUp*M%{|8#6mHcUl?Gb%u_W5SYFt`zcX3>MUzVAor%Rb zN@%3B#8MY^c-u&4)ohv##=9B4^zY^)CDu_Mb|z7Bx-g6gLCEFr-5(O)(kT83s>!Ff(WgoNYsSB*wcRj z7J^Tg!wh(o-XIr*z9JisB9h{6z! zkakWKLqje|EuyVYYSOSoQG4GY5Je3cw5!&H$$X5f%nIy%@TK z4)^A|tG`3UWPFR%K+qj>LXH7w5|`~ibdh* zHZ&L(3^#6V3!HMrBZDb&!Eg>Bh;1By5C|yYfM;qWL}*P1Y} zbIp!raPSFaldY`6*v;o}YXmEWlPNw_#FN>VM!{AjE9(wKeqv+MSft5IhlVSf4xj%% zpwmo@eJa7DDENn9FB;o^fb0xUF_(AiKS1^#p=0PLFV6!s#o<&x_#~xd?_j4K-#gdL zp!aX$ZU5);XyTCnnxOyw*S`c{F1wvSlR612e=!LuKe6$JBR|f&JIocD-R}1;q$n#V z-~(j8%V_v-f65=nl*tA@DGuD-cZfTZ({ltFdXLbNqDwsaa*Kj@jN_@~y{Q-K#v~$8 zx`!l>u5dK@7ibicsAU-uMvX#6(|ISw6Q<-UfJ!N_OTv6fyc4)CR(F)V?e%*nVRSgR zf4?37rxwrz2IC|L#br1YFi8Jy*kIF;4q=lMLrq9Y!h+(8c>p5PQ)!xWB1?%p8Yyeq z8JiKOZsRr3YWqEpI&H}nI%6rh??d9mGn#*SW*GOPEO=D@0X};XFr-+4{^rK~l(Ir(a4q9e^5wff4 zNVFJhTrHBB?G{%ejT)?{L~t2?-rtSW`$S@})x?(#FcEhN6D(}1vWh9_)yUGBf>ABD zO*B(YcR`E|n^{V;P_A*eG3sla)2IYnHjj|2KxWf=DLlG-xI@c~Ei>+t8EZ|uf8j3P zY$m>GM>B8hLUCRq7&sHbR$86T#MN6I(8xOiaEQ00&#nwg(g*$?7*guVK-O%0M(IuH zFFd-5sp#(XCOxpUh0OXO6$)S!q4JaP!ToVZ5PY_4& zEVlSkK95UoeaaoOq^$SX0W+5s| z=CTEf=Q(Qi$m5P&1J^`0O@U-z_T0yw``B|Id+yW4)uyZnU!AMu_RjWB2ay-{9w=}? zd76SqMwDbJNWC&TH|zSqM$6pxM+d6l338MTL@~V1)^b;@tl#^n@6@yj*7q^KkF!`$ zX>B%1eTj|KxUx&7*?~i}f2x+h5cP<4yw}@xA$`@2v($yUma1O1l@-Km%G~8ldNR=e zvK6{i0}F#TyaryNDN^ZFEMLeqR#u*VItYkVZ;+_-R1^K#AWjc05AV+py?nP9kx*qh z6Ik?xT;XXFE%+S_ym>oOud5n`r`4T4xds!#X<@zT5ee+{INZerCUs7ZX1 z3}Iahpk%y_q)$Xm`r=UR3@t;FEK1C?1uk@55_%rMB=L`q0uPF2aN4_D*&k!~X;EM+ zB?qT)E?uXG2tdaP1IUqRSH`&)jk7)o87jvdfKxC=Jg#_6Q1VU+Rx38G*nC1_vo1Eu zQjt-9l&6zQXPfFgf4NjHjZ`|oN+i9DnWfS}F{GDeS+orKA{QYb<&L2!DJ!#(~OuOvVBndh-Yewu9F-4Vs5UEi|zZF4uHh z1DgwVn&ag9@h%9ET1R!96_ThBPj~uKUgWUvBsBq(RmGd=>y`yn-MS`M6U+qr7O$jd zI0%p$Cm3oM{*Z7yMF0yrM$kckQx!Uq0f)LB22Qm2f6yEs7M4kgt9G%32?BhNy+7o& z28~?aWA76t>7PeoKzQ==j5;^z*Qt(^hZl&U;1-==HVlI(m`OwQlXTrwZse(IQ9l}G zi>iA1TGTAgT8l=aulclv-mAJO_Yp=o_Anw`JPKPT0xD~1$(XxR>(_^UbcTY(qOXL8 z?hZ04e@18!tFe@>v=zvO90CU>cjlC^istE>#4a`116WGEKwf_$6d=Nbd2|;pZivI7 zhoa`LNz&4jv>#ak+u%qczwL_02_+>AzW(-#KKz>U-Tk^O3lT@SKx52!Fu$NwFU%!a zZ?3Thc^C5h*pScWI;=~)knN#`a#1?k1E{WEZA-$r7| zf216ML4a=&O9i@t#06udLcK`tnk3P2C>dcF5eESm`Fyfv1L@d6IyR8bng`OcF=5st zCQK2yDfzWB&#t}QmAH1wLz1gmcRyrrnu|Q@=&~2`Mr7AB_ zzn+6<>hsp>RH^pLb*k&$b+3_9m8G!8e-57RT0&I@@F>m#C-C$X1_DYzp<=mFqBI>T zv>(~8RN}M%K}Jb;<|T{B)+{@dZXiFm@nZw~PbN{zhNVw`vr`o~E81p0#iW?m%9!EE z>c)q6Dv4qp!;$Tkvz!*pCZf)O-KdTQYhe+yVS>dt@k`|P!4yFfCx8?jhmv17e?Ye= z5IgY(z!{u_Q9!2vbL(>V5N6`fp;vT))elo_I#x|Q?~(8D-KvO0bK-o0L2Epr>n?;7 zwR>7O(#3{O=&9e$QJ2N7EYU*E4j#?{n-d4QTi4`wrfo~;jt5AF?vds~3nexdN<2Bv zu~&6*cTSx0^ywyiBTD_B=Niw!f3p|y`$fu!EY!_uPKu;zrlI;vAALJvpbW+8R9v4D z$cdF)0E9rRqt(Myh}sLbZl*058!JjHV+pPDOL90uH+!Ydezn_E8mf4#3XxIsbo zBG@vi?ow;v$%QF+^Q!7TmDpq9aEkenKnm@6UQ)jbFTGG8UY7a9#tU#$s=+R>QRzjp#TtoSZhc#0T@Q(s*!f0xVZ5|2%*zo&sH zPM{~v={oWZB3vggjPf)ec`O;PP9BUL|0esep@_~jT6qEdvFHcQ4qNi#E!p5rCNQ^P znq0h>Yocu$J^7%WT+cC*cF1fg>My9y71;$WNpHBvF#QD~(17SfM6@J45jgFL&v7dg zwA%t;v*t&qS|~RTe>RY1we6~x2)Jg%mk8iC4YZ9K5(9=OE&&o8cYj;5Y?GxBg~_JL zV!(FnJkx+d#5z&*XwFgFM6in)dujUUEZ}+4%eAe7ZEcrlyKnZZE{e~I^E;Ws;O5Zt z+8%wR8z{A8*6Qx*+FdB`Iclmk_e{rr7>toT^U(Ua1Dy%#e}gH!5&H*o;8Vu%(36ou zq(zfc<>}3UAD~-IL*~u1&K64>JD=P-eO-02HvV4AEz+e==;CI&TdF3fbV;`%sXD1E zU8zGE0SCJ<@Luh8=lKY|`4PV)o%Ol`wU@Xx_Beg1;Sr-aurgG}=^5jWE9H43+pr@xvX zWJyV%^bhqjeZ-%+;vU80i@X6Q2)c13ze%TvW9@FPf0u2{^oOe8aPdof^*X0T~#=`KRlByEbqQ! zBffOBId&^N)uT`Cp%xBl>B`39tx{i3x3;XhoTF@!;L@GC@*fplR3kW~Jw~!-n zgJh_H)P`C@Ks@^Gl>MYtj#tTR6Thn~9rNE~G82QGcA)^5dDAp=en~gBtRFzd>DGn_ z*S(5Vfogkvgl*yGM`#*4&A@aNH`$TJC{5xbe-0sfEw>M4^+Ei!`GYmg6t>o@idz(F zU7b#s5Xh?X0*_+NjN9mSs{NJfbMX)sZ9q*+fE*;lT0s!wis*2|@j_R=Ny%<~{aaRM zKxQV>F_XI9pH}wSc&HD@W%es>FR0CPhFM!4BdV60iN1Yr%uqyQHSDC&Vhe_%>8)cW zf07ChGtOj4uar3mEhW9n7Ie>dDe zcubl^(*O2-IOjsRzHoxM&*|1>5|wq4cojuEG$m`&>%--zdS-Y`(i=()gn^K$&?3b^ z`J1|-YNMp8?`uY6K_Bi?02kI*QM)fGVXolUT@Y=UJNRBNwQU3}7;Nv<`bLRQ8pVBR z&)0MdP1u{Z80nhHfthFyx%#mgf2WHAgLljDk};X`y@-xH`-{SHTLxHWGfY;W2}{x5 z-RdU!6-~z}cpuPd`uAr_9Gx!x4;j8tMF;o**%_W<6_nruWbY9=hJNz$JU~+%PW6LN zGNOO)K*bS|pP(#YaA_L(;m~){_%;~eJ%=(hiIK_#~V**)1IQaH|a-zY@^$K`a8CjBa2he z7e=DjU*l@gVqcxqsyXM*aN?@nn=jNb#P<`}&i!5jn+$_yJRCPkeHzPyEDvhQgVJ}J z(`$ciq5g&jR3B{b);d*3e^-E(^Qz9ZQl$i_&(r3xAlJc44-SfhS!77eQx{#pOdMrk zbZ($xY3sOQE^?@=yssgcAb6t2ws<7z{B*UOSkb`YO~!w`4`4bOmb_6-r$`wzLbZ2Isb5@NDZVke zbxp2PS1(+Om`r9jfw$4s?Wf{9w|-{P+Qd-S`f2?kyW-w^dqO4q`Tqg{0RR8()J^AK G3Izb*ul1Gy From 78949d6c05c879b8b4463e99c9736c49d77ad1aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 23 Nov 2021 18:32:56 +0100 Subject: [PATCH 31/33] retrieval: Fix deadlock in ClientGetRetrievalUpdates --- node/impl/client/client.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index c2611348c..0b030b23d 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -1157,7 +1157,10 @@ func (a *API) ClientGetRetrievalUpdates(ctx context.Context) (<-chan api.Retriev unsub := a.Retrieval.SubscribeToEvents(func(evt rm.ClientEvent, deal rm.ClientDealState) { update := a.newRetrievalInfo(ctx, deal) update.Event = &evt - updates <- update + select { + case updates <- update: + case <-ctx.Done(): + } }) go func() { From 210485d800eb087da9fa9fff8ba91841a41cd2ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 23 Nov 2021 22:23:13 +0100 Subject: [PATCH 32/33] Add some description to the retrieve command --- cli/client_retr.go | 66 +++++++++++++++++++------- documentation/en/cli-lotus.md | 88 ++++++++++++++++++++++++----------- 2 files changed, 111 insertions(+), 43 deletions(-) diff --git a/cli/client_retr.go b/cli/client_retr.go index 6a23dd14a..3674ad68a 100644 --- a/cli/client_retr.go +++ b/cli/client_retr.go @@ -93,7 +93,7 @@ func retrieve(ctx context.Context, cctx *cli.Context, fapi lapi.FullNode, sel *l // no local found, so make a retrieval if eref == nil { var offer lapi.QueryOffer - minerStrAddr := cctx.String("miner") + minerStrAddr := cctx.String("provider") if minerStrAddr == "" { // Local discovery offers, err := fapi.ClientFindData(ctx, file, pieceCid) @@ -216,8 +216,9 @@ var retrFlagsCommon = []cli.Flag{ Usage: "address to send transactions from", }, &cli.StringFlag{ - Name: "miner", - Usage: "miner address for retrieval, if not present it'll use local discovery", + Name: "provider", + Usage: "provider to use for retrieval, if not present it'll use local discovery", + Aliases: []string{"miner"}, }, &cli.StringFlag{ Name: "maxPrice", @@ -237,14 +238,47 @@ var clientRetrieveCmd = &cli.Command{ Name: "retrieve", Usage: "Retrieve data from network", ArgsUsage: "[dataCid outputPath]", + Description: `Retrieve data from the Filecoin network. + +The retrieve command will attempt to find a provider make a retrieval deal with +them. In case a provider can't be found, it can be specified with the --provider +flag. + +By default the data will be interpreted as DAG-PB UnixFSv1 File. Alternatively +a CAR file containing the raw IPLD graph can be exported by setting the --car +flag. + +Partial Retrieval: + +The --data-selector flag can be used to specify a sub-graph to fetch. The +selector can be specified as either IPLD datamodel text-path selector, or IPLD +json selector. + +In case of unixfs retrieval, the selector must point at a single root node, and +match the entire graph under that node. + +In case of CAR retrieval, the selector must have one common "sub-root" node. + +Examples: + +- Retrieve a file by CID + $ lotus client retrieve Qm... my-file.txt + +- Retrieve a file by CID from f0123 + $ lotus client retrieve --provider f0123 Qm... my-file.txt + +- Retrieve a first file from a specified directory + $ lotus client retrieve --data-selector /Links/0/Hash Qm... my-file.txt +`, Flags: append([]cli.Flag{ &cli.BoolFlag{ Name: "car", Usage: "export to a car file instead of a regular file", }, &cli.StringFlag{ - Name: "datamodel-path-selector", - Usage: "a rudimentary (DM-level-only) text-path selector, allowing for sub-selection within a deal", + Name: "data-selector", + Aliases: []string{"data-selector-selector"}, + Usage: "IPLD datamodel text-path selector, or IPLD json selector", }, }, retrFlagsCommon...), Action: func(cctx *cli.Context) error { @@ -261,7 +295,7 @@ var clientRetrieveCmd = &cli.Command{ afmt := NewAppFmt(cctx.App) var s *lapi.Selector - if sel := lapi.Selector(cctx.String("datamodel-path-selector")); sel != "" { + if sel := lapi.Selector(cctx.String("data-selector")); sel != "" { s = &sel } @@ -350,8 +384,8 @@ var clientRetrieveCatCmd = &cli.Command{ Usage: "list IPLD datamodel links", }, &cli.StringFlag{ - Name: "datamodel-path", - Usage: "a rudimentary (DM-level-only) text-path selector", + Name: "data-selector", + Usage: "IPLD datamodel text-path selector, or IPLD json selector", }, }, retrFlagsCommon...), Action: func(cctx *cli.Context) error { @@ -372,7 +406,7 @@ var clientRetrieveCatCmd = &cli.Command{ ctx := ReqContext(cctx) afmt := NewAppFmt(cctx.App) - sel := lapi.Selector(cctx.String("datamodel-path")) + sel := lapi.Selector(cctx.String("data-selector")) selp := &sel if sel == "" { selp = nil @@ -416,7 +450,7 @@ func pathToSel(psel string, sub builder.SelectorSpec) (lapi.Selector, error) { var clientRetrieveLsCmd = &cli.Command{ Name: "ls", - Usage: "Show object links", + Usage: "List object links", ArgsUsage: "[dataCid]", Flags: append([]cli.Flag{ &cli.BoolFlag{ @@ -429,8 +463,8 @@ var clientRetrieveLsCmd = &cli.Command{ Value: 1, }, &cli.StringFlag{ - Name: "datamodel-path", - Usage: "a rudimentary (DM-level-only) text-path selector", + Name: "data-selector", + Usage: "IPLD datamodel text-path selector, or IPLD json selector", }, }, retrFlagsCommon...), Action: func(cctx *cli.Context) error { @@ -453,9 +487,9 @@ var clientRetrieveLsCmd = &cli.Command{ dataSelector := lapi.Selector(fmt.Sprintf(`{"R":{"l":{"depth":%d},":>":{"a":{">":{"|":[{"@":{}},{".":{}}]}}}}}`, cctx.Int("depth"))) - if cctx.IsSet("datamodel-path") { + if cctx.IsSet("data-selector") { ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) - dataSelector, err = pathToSel(cctx.String("datamodel-path"), + dataSelector, err = pathToSel(cctx.String("data-selector"), ssb.ExploreUnion( ssb.Matcher(), ssb.ExploreAll( @@ -518,9 +552,9 @@ var clientRetrieveLsCmd = &cli.Command{ } else { jsel := lapi.Selector(fmt.Sprintf(`{"R":{"l":{"depth":%d},":>":{"a":{">":{"|":[{"@":{}},{".":{}}]}}}}}`, cctx.Int("depth"))) - if cctx.IsSet("datamodel-path") { + if cctx.IsSet("data-selector") { ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) - jsel, err = pathToSel(cctx.String("datamodel-path"), + jsel, err = pathToSel(cctx.String("data-selector"), ssb.ExploreRecursive(selector.RecursionLimitDepth(int64(cctx.Int("depth"))), ssb.ExploreAll(ssb.ExploreUnion(ssb.Matcher(), ssb.ExploreRecursiveEdge()))), ) } diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index 051293ed6..ad286703c 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -427,7 +427,7 @@ COMMANDS: find Find data in the network retrieve Retrieve data from network cat Show data from network - ls Show object links + ls List object links cancel-retrieval Cancel a retrieval deal by deal ID; this also cancels the associated transfer list-retrievals List retrieval market deals STORAGE: @@ -546,15 +546,49 @@ USAGE: CATEGORY: RETRIEVAL +DESCRIPTION: + Retrieve data from the Filecoin network. + +The retrieve command will attempt to find a provider make a retrieval deal with +them. In case a provider can't be found, it can be specified with the --provider +flag. + +By default the data will be interpreted as DAG-PB UnixFSv1 File. Alternatively +a CAR file containing the raw IPLD graph can be exported by setting the --car +flag. + +Partial Retrieval: + +The --data-selector flag can be used to specify a sub-graph to fetch. The +selector can be specified as either IPLD datamodel text-path selector, or IPLD +json selector. + +In case of unixfs retrieval, the selector must point at a single root node, and +match the entire graph under that node. + +In case of CAR retrieval, the selector must have one common "sub-root" node. + +Examples: + +- Retrieve a file by CID + $ lotus client retrieve Qm... my-file.txt + +- Retrieve a file by CID from f0123 + $ lotus client retrieve --provider f0123 Qm... my-file.txt + +- Retrieve a first file from a specified directory + $ lotus client retrieve --data-selector /Links/0/Hash Qm... my-file.txt + + OPTIONS: - --car export to a car file instead of a regular file (default: false) - --datamodel-path-selector value a rudimentary (DM-level-only) text-path selector, allowing for sub-selection within a deal - --from value address to send transactions from - --miner value miner address for retrieval, if not present it'll use local discovery - --maxPrice value maximum price the client is willing to consider (default: 0 FIL) - --pieceCid value require data to be retrieved from a specific Piece CID - --allow-local (default: false) - --help, -h show help (default: false) + --car export to a car file instead of a regular file (default: false) + --data-selector value, --data-selector-selector value IPLD datamodel text-path selector, or IPLD json selector + --from value address to send transactions from + --provider value, --miner value provider to use for retrieval, if not present it'll use local discovery + --maxPrice value maximum price the client is willing to consider (default: 0 FIL) + --pieceCid value require data to be retrieved from a specific Piece CID + --allow-local (default: false) + --help, -h show help (default: false) ``` @@ -570,21 +604,21 @@ CATEGORY: RETRIEVAL OPTIONS: - --ipld list IPLD datamodel links (default: false) - --datamodel-path value a rudimentary (DM-level-only) text-path selector - --from value address to send transactions from - --miner value miner address for retrieval, if not present it'll use local discovery - --maxPrice value maximum price the client is willing to consider (default: 0 FIL) - --pieceCid value require data to be retrieved from a specific Piece CID - --allow-local (default: false) - --help, -h show help (default: false) + --ipld list IPLD datamodel links (default: false) + --data-selector value IPLD datamodel text-path selector, or IPLD json selector + --from value address to send transactions from + --provider value, --miner value provider to use for retrieval, if not present it'll use local discovery + --maxPrice value maximum price the client is willing to consider (default: 0 FIL) + --pieceCid value require data to be retrieved from a specific Piece CID + --allow-local (default: false) + --help, -h show help (default: false) ``` ### lotus client ls ``` NAME: - lotus client ls - Show object links + lotus client ls - List object links USAGE: lotus client ls [command options] [dataCid] @@ -593,15 +627,15 @@ CATEGORY: RETRIEVAL OPTIONS: - --ipld list IPLD datamodel links (default: false) - --depth value list links recursively up to the specified depth (default: 1) - --datamodel-path value a rudimentary (DM-level-only) text-path selector - --from value address to send transactions from - --miner value miner address for retrieval, if not present it'll use local discovery - --maxPrice value maximum price the client is willing to consider (default: 0 FIL) - --pieceCid value require data to be retrieved from a specific Piece CID - --allow-local (default: false) - --help, -h show help (default: false) + --ipld list IPLD datamodel links (default: false) + --depth value list links recursively up to the specified depth (default: 1) + --data-selector value IPLD datamodel text-path selector, or IPLD json selector + --from value address to send transactions from + --provider value, --miner value provider to use for retrieval, if not present it'll use local discovery + --maxPrice value maximum price the client is willing to consider (default: 0 FIL) + --pieceCid value require data to be retrieved from a specific Piece CID + --allow-local (default: false) + --help, -h show help (default: false) ``` From bd4927b4942b08d9e17fd354cccef61efdb8f426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 24 Nov 2021 20:13:49 +0100 Subject: [PATCH 33/33] retrieval: Cleanup some comments --- node/impl/client/client.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 0b030b23d..960ca76f3 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -1058,9 +1058,6 @@ func parseDagSpec(ctx context.Context, root cid.Cid, dsp []api.DagSpec, ds forma if p.LastBlock.Link == nil { // this is likely the root node that we've matched here - // todo: is this a correct assumption - // todo: is the n ipld.Node above the node we want as the (sub)root? - // todo: how to go from ipld.Node to a cid? newRoot = root return errHalt }