From ccf359d057ddbf75b552b53c6f2b45edfe965bb1 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Mon, 16 Dec 2019 19:17:46 -0800 Subject: [PATCH 1/6] feat(retrieval): extract retrievalmarket Extract retrieval market and modify shared types --- chain/deals/client.go | 8 +- go.mod | 3 +- go.sum | 7 +- node/builder.go | 6 +- node/impl/client/client.go | 15 +- node/modules/client.go | 5 +- node/modules/services.go | 9 +- node/modules/storageminer.go | 9 +- retrieval/cbor-gen/main.go | 20 -- retrieval/discovery/discovery.go | 15 - retrieval/discovery/local.go | 54 ---- retrieval/impl/cbor_gen.go | 476 ------------------------------- retrieval/impl/client.go | 389 ------------------------- retrieval/impl/provider.go | 323 --------------------- retrieval/impl/run_cbor_gen.go | 33 --- retrieval/impl/types.go | 67 ----- retrieval/impl/verify.go | 141 --------- retrieval/types.go | 364 ----------------------- retrievaladapter/client.go | 20 +- retrievaladapter/converters.go | 45 +++ retrievaladapter/provider.go | 35 ++- 21 files changed, 127 insertions(+), 1917 deletions(-) delete mode 100644 retrieval/cbor-gen/main.go delete mode 100644 retrieval/discovery/discovery.go delete mode 100644 retrieval/discovery/local.go delete mode 100644 retrieval/impl/cbor_gen.go delete mode 100644 retrieval/impl/client.go delete mode 100644 retrieval/impl/provider.go delete mode 100644 retrieval/impl/run_cbor_gen.go delete mode 100644 retrieval/impl/types.go delete mode 100644 retrieval/impl/verify.go delete mode 100644 retrieval/types.go create mode 100644 retrievaladapter/converters.go diff --git a/chain/deals/client.go b/chain/deals/client.go index 879e408e2..02c9bcce8 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -11,15 +11,15 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" + cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-fil-components/retrievalmarket" + "github.com/filecoin-project/go-fil-components/retrievalmarket/discovery" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" - "github.com/filecoin-project/lotus/retrieval/discovery" - "github.com/filecoin-project/lotus/storagemarket" + "github.com/filecoin-project/lotus/storagemarket" ) var log = logging.Logger("deals") diff --git a/go.mod b/go.mod index ee7c4d175..665c367c4 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce + github.com/filecoin-project/go-fil-components v0.0.0-20200110033229-671d79221133 github.com/filecoin-project/go-paramfetch v0.0.1 github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254 github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 @@ -29,7 +30,7 @@ require ( github.com/ipfs/go-bitswap v0.1.8 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c - github.com/ipfs/go-car v0.0.2 + github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1 github.com/ipfs/go-cid v0.0.4 github.com/ipfs/go-datastore v0.3.1 github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521 diff --git a/go.sum b/go.sum index 589a5d544..dc1b26e42 100644 --- a/go.sum +++ b/go.sum @@ -111,6 +111,8 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= +github.com/filecoin-project/go-fil-components v0.0.0-20200110033229-671d79221133 h1:/L916kY3hyq8w18rLO9VMSHqw25/9pwRB3nVW6b+Sm4= +github.com/filecoin-project/go-fil-components v0.0.0-20200110033229-671d79221133/go.mod h1:7M0YUI2CSVmqEmXNeXNq5L/pjk7C1Q5ifhirfMADD/k= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= @@ -120,6 +122,7 @@ github.com/filecoin-project/go-paramfetch v0.0.1 h1:gV7bs5YaqlgpGFMiLxInGK2L1FyC github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254 h1:4IvlPad82JaNBtqh8fEAUIKWv8I3tguAJjGvUyHNZS4= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= +github.com/filecoin-project/go-statestore v0.0.0-20191219195854-7a95521e8f15/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk= github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -210,8 +213,8 @@ github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbR github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c h1:lN5IQA07VtLiTLAp/Scezp1ljFhXErC6yq4O1cu+yJ0= github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= -github.com/ipfs/go-car v0.0.2 h1:j02lzgeijorstzoMl3nQmvvb8wjJUVCiOAl8XEwYMCQ= -github.com/ipfs/go-car v0.0.2/go.mod h1:60pzeu308k5kVFHzq0HIi2kPtITgor+1ll1xuGk5JwQ= +github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1 h1:Nq8xEW+2KZq7IkRlkOh0rTEUI8FgunhMoLj5EMkJzbQ= +github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1/go.mod h1:rmd887mJxQRDfndfDEY3Liyx8gQVyfFFRSHdsnDSAlk= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= diff --git a/node/builder.go b/node/builder.go index dcce895b7..d263e04fa 100644 --- a/node/builder.go +++ b/node/builder.go @@ -19,6 +19,8 @@ import ( "go.uber.org/fx" "golang.org/x/xerrors" + "github.com/filecoin-project/go-fil-components/retrievalmarket" + "github.com/filecoin-project/go-fil-components/retrievalmarket/discovery" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/blocksync" @@ -43,8 +45,6 @@ import ( "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/paych" "github.com/filecoin-project/lotus/peermgr" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" - "github.com/filecoin-project/lotus/retrieval/discovery" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" "github.com/filecoin-project/lotus/storagemarket" @@ -222,7 +222,7 @@ func Online() Option { Override(RunPeerMgrKey, modules.RunPeerMgr), Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), - Override(new(*discovery.Local), discovery.NewLocal), + Override(new(*discovery.Local), modules.NewLocalDiscovery), Override(new(retrievalmarket.PeerResolver), modules.RetrievalResolver), Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient), diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 58f250f67..a8b7ff17f 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -25,6 +25,7 @@ import ( "go.uber.org/fx" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-components/retrievalmarket" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/store" @@ -32,7 +33,7 @@ import ( "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" + "github.com/filecoin-project/lotus/retrievaladapter" "github.com/filecoin-project/lotus/storagemarket" ) @@ -164,7 +165,7 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffe out[k] = api.QueryOffer{ Root: root, Size: queryResponse.Size, - MinPrice: queryResponse.PieceRetrievalPrice(), + MinPrice: retrievaladapter.FromSharedTokenAmount(queryResponse.PieceRetrievalPrice()), Miner: p.Address, // TODO: check MinerPeerID: p.ID, } @@ -292,9 +293,13 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path }) a.Retrieval.Retrieve( - ctx, order.Root.Bytes(), retrievalmarket.Params{ - PricePerByte: types.BigDiv(order.Total, types.NewInt(order.Size)), - }, order.Total, order.MinerPeerID, order.Client, order.Miner) + ctx, + order.Root.Bytes(), + retrievalmarket.NewParamsV0(types.BigDiv(order.Total, types.NewInt(order.Size)).Int, 0, 0), + retrievaladapter.ToSharedTokenAmount(order.Total), + order.MinerPeerID, + order.Client, + order.Miner) select { case <-ctx.Done(): return xerrors.New("Retrieval Timed Out") diff --git a/node/modules/client.go b/node/modules/client.go index 6047df7af..67c6322ab 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -6,11 +6,12 @@ import ( "path/filepath" "reflect" + "github.com/filecoin-project/go-fil-components/retrievalmarket" + retrievalimpl "github.com/filecoin-project/go-fil-components/retrievalmarket/impl" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/paych" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" - retrievalimpl "github.com/filecoin-project/lotus/retrieval/impl" + "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" graphsync "github.com/ipfs/go-graphsync/impl" diff --git a/node/modules/services.go b/node/modules/services.go index bd0978085..bc99d49f3 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -8,15 +8,16 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" "go.uber.org/fx" + "github.com/filecoin-project/go-fil-components/retrievalmarket" + "github.com/filecoin-project/go-fil-components/retrievalmarket/discovery" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/blocksync" "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/sub" "github.com/filecoin-project/lotus/node/hello" + "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/peermgr" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" - "github.com/filecoin-project/lotus/retrieval/discovery" "github.com/filecoin-project/lotus/storagemarket" ) @@ -81,6 +82,10 @@ func RunDealClient(mctx helpers.MetricsCtx, lc fx.Lifecycle, c storagemarket.Sto }) } +func NewLocalDiscovery(ds dtypes.MetadataDS) *discovery.Local { + return discovery.NewLocal(ds) +} + func RetrievalResolver(l *discovery.Local) retrievalmarket.PeerResolver { return discovery.Multi(l) } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index d868414d9..876c2f368 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -25,6 +25,8 @@ import ( "github.com/filecoin-project/go-address" dtgraphsync "github.com/filecoin-project/go-data-transfer/impl/graphsync" + "github.com/filecoin-project/go-fil-components/retrievalmarket" + retrievalimpl "github.com/filecoin-project/go-fil-components/retrievalmarket/impl" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" @@ -35,8 +37,7 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" - retrievalimpl "github.com/filecoin-project/lotus/retrieval/impl" + "github.com/filecoin-project/lotus/retrievaladapter" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" @@ -265,6 +266,6 @@ func SealTicketGen(api api.FullNode) storage.TicketFn { // RetrievalProvider creates a new retrieval provider attached to the provider blockstore func RetrievalProvider(sblks *sectorblocks.SectorBlocks, full api.FullNode) retrievalmarket.RetrievalProvider { - adapter := retrievaladapter.NewRetrievalProviderNode(full) - return retrievalimpl.NewProvider(sblks, adapter) + adapter := retrievaladapter.NewRetrievalProviderNode(sblks, full) + return retrievalimpl.NewProvider(adapter) } diff --git a/retrieval/cbor-gen/main.go b/retrieval/cbor-gen/main.go deleted file mode 100644 index cc3fe9887..000000000 --- a/retrieval/cbor-gen/main.go +++ /dev/null @@ -1,20 +0,0 @@ -package main - -import ( - "fmt" - "os" - - retrievalimpl "github.com/filecoin-project/lotus/retrieval/impl" -) - -// main func has ONE JOB -func main() { - fmt.Print("Generating Cbor Marshal/Unmarshal...") - - if err := retrievalimpl.RunCborGen(); err != nil { - fmt.Println("Failed: ") - fmt.Println(err) - os.Exit(1) - } - fmt.Println("Done.") -} diff --git a/retrieval/discovery/discovery.go b/retrieval/discovery/discovery.go deleted file mode 100644 index 93da56b60..000000000 --- a/retrieval/discovery/discovery.go +++ /dev/null @@ -1,15 +0,0 @@ -package discovery - -import ( - cbor "github.com/ipfs/go-ipld-cbor" - - retrievalmarket "github.com/filecoin-project/lotus/retrieval" -) - -func init() { - cbor.RegisterCborType(retrievalmarket.RetrievalPeer{}) -} - -func Multi(r retrievalmarket.PeerResolver) retrievalmarket.PeerResolver { // TODO: actually support multiple mechanisms - return r -} diff --git a/retrieval/discovery/local.go b/retrieval/discovery/local.go deleted file mode 100644 index c777a0828..000000000 --- a/retrieval/discovery/local.go +++ /dev/null @@ -1,54 +0,0 @@ -package discovery - -import ( - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - dshelp "github.com/ipfs/go-ipfs-ds-help" - cbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log/v2" - - "github.com/filecoin-project/lotus/node/modules/dtypes" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" -) - -var log = logging.Logger("ret-discovery") - -type Local struct { - ds datastore.Datastore -} - -func NewLocal(ds dtypes.MetadataDS) *Local { - return &Local{ds: namespace.Wrap(ds, datastore.NewKey("/deals/local"))} -} - -func (l *Local) AddPeer(cid cid.Cid, peer retrievalmarket.RetrievalPeer) error { - // TODO: allow multiple peers here - // (implement an util for tracking map[thing][]otherThing, use in sectorBlockstore too) - - log.Warn("Tracking multiple retrieval peers not implemented") - - entry, err := cbor.DumpObject(peer) - if err != nil { - return err - } - - return l.ds.Put(dshelp.CidToDsKey(cid), entry) -} - -func (l *Local) GetPeers(data cid.Cid) ([]retrievalmarket.RetrievalPeer, error) { - entry, err := l.ds.Get(dshelp.CidToDsKey(data)) - if err == datastore.ErrNotFound { - return []retrievalmarket.RetrievalPeer{}, nil - } - if err != nil { - return nil, err - } - var peer retrievalmarket.RetrievalPeer - if err := cbor.DecodeInto(entry, &peer); err != nil { - return nil, err - } - return []retrievalmarket.RetrievalPeer{peer}, nil -} - -var _ retrievalmarket.PeerResolver = &Local{} diff --git a/retrieval/impl/cbor_gen.go b/retrieval/impl/cbor_gen.go deleted file mode 100644 index 220577089..000000000 --- a/retrieval/impl/cbor_gen.go +++ /dev/null @@ -1,476 +0,0 @@ -package retrievalimpl - -import ( - "fmt" - "io" - - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -var _ = xerrors.Errorf - -func (t *RetParams) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{129}); err != nil { - return err - } - - // t.Unixfs0 (retrievalimpl.Unixfs0Offer) (struct) - if err := t.Unixfs0.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *RetParams) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 1 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Unixfs0 (retrievalimpl.Unixfs0Offer) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.Unixfs0 = new(Unixfs0Offer) - if err := t.Unixfs0.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - return nil -} - -func (t *OldQuery) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{129}); err != nil { - return err - } - - // t.Piece (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Piece); err != nil { - return xerrors.Errorf("failed to write cid field t.Piece: %w", err) - } - - return nil -} - -func (t *OldQuery) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 1 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Piece (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Piece: %w", err) - } - - t.Piece = c - - } - return nil -} - -func (t *OldQueryResponse) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{131}); err != nil { - return err - } - - // t.Status (retrievalimpl.OldQueryResponseStatus) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil { - return err - } - - // t.Size (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil { - return err - } - - // t.MinPrice (types.BigInt) (struct) - if err := t.MinPrice.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *OldQueryResponse) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 3 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Status (retrievalimpl.OldQueryResponseStatus) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Status = OldQueryResponseStatus(extra) - // t.Size (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Size = uint64(extra) - // t.MinPrice (types.BigInt) (struct) - - { - - if err := t.MinPrice.UnmarshalCBOR(br); err != nil { - return err - } - - } - return nil -} - -func (t *Unixfs0Offer) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Offset (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Offset))); err != nil { - return err - } - - // t.Size (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil { - return err - } - return nil -} - -func (t *Unixfs0Offer) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Offset (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Offset = uint64(extra) - // t.Size (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Size = uint64(extra) - return nil -} - -func (t *OldDealProposal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{131}); err != nil { - return err - } - - // t.Payment (api.PaymentInfo) (struct) - if err := t.Payment.MarshalCBOR(w); err != nil { - return err - } - - // t.Ref (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Ref); err != nil { - return xerrors.Errorf("failed to write cid field t.Ref: %w", err) - } - - // t.Params (retrievalimpl.RetParams) (struct) - if err := t.Params.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *OldDealProposal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 3 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Payment (api.PaymentInfo) (struct) - - { - - if err := t.Payment.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.Ref (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Ref: %w", err) - } - - t.Ref = c - - } - // t.Params (retrievalimpl.RetParams) (struct) - - { - - if err := t.Params.UnmarshalCBOR(br); err != nil { - return err - } - - } - return nil -} - -func (t *OldDealResponse) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Status (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil { - return err - } - - // t.Message (string) (string) - if len(t.Message) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Message was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Message)); err != nil { - return err - } - return nil -} - -func (t *OldDealResponse) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Status (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Status = uint64(extra) - // t.Message (string) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Message = string(sval) - } - return nil -} - -func (t *Block) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Prefix ([]uint8) (slice) - if len(t.Prefix) > cbg.ByteArrayMaxLen { - return xerrors.Errorf("Byte array in field t.Prefix was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Prefix)))); err != nil { - return err - } - if _, err := w.Write(t.Prefix); err != nil { - return err - } - - // t.Data ([]uint8) (slice) - if len(t.Data) > cbg.ByteArrayMaxLen { - return xerrors.Errorf("Byte array in field t.Data was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Data)))); err != nil { - return err - } - if _, err := w.Write(t.Data); err != nil { - return err - } - return nil -} - -func (t *Block) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Prefix ([]uint8) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.Prefix: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.Prefix = make([]byte, extra) - if _, err := io.ReadFull(br, t.Prefix); err != nil { - return err - } - // t.Data ([]uint8) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.Data: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.Data = make([]byte, extra) - if _, err := io.ReadFull(br, t.Data); err != nil { - return err - } - return nil -} diff --git a/retrieval/impl/client.go b/retrieval/impl/client.go deleted file mode 100644 index 5087a893d..000000000 --- a/retrieval/impl/client.go +++ /dev/null @@ -1,389 +0,0 @@ -package retrievalimpl - -import ( - "context" - "reflect" - "sync" - - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" - logging "github.com/ipfs/go-log/v2" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" - - cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/types" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" -) - -var log = logging.Logger("retrieval") - -type client struct { - h host.Host - bs blockstore.Blockstore - node retrievalmarket.RetrievalClientNode - // The parameters should be replaced by RetrievalClientNode - - nextDealLk sync.Mutex - nextDealID retrievalmarket.DealID - subscribersLk sync.RWMutex - subscribers []retrievalmarket.ClientSubscriber -} - -// NewClient creates a new retrieval client -func NewClient(h host.Host, bs blockstore.Blockstore, node retrievalmarket.RetrievalClientNode) retrievalmarket.RetrievalClient { - return &client{h: h, bs: bs, node: node} -} - -// V0 - -// TODO: Implement for retrieval provider V0 epic -// https://github.com/filecoin-project/go-retrieval-market-project/issues/12 -func (c *client) FindProviders(pieceCID []byte) []retrievalmarket.RetrievalPeer { - panic("not implemented") -} - -// TODO: Update to match spec for V0 epic -// https://github.com/filecoin-project/go-retrieval-market-project/issues/8 -func (c *client) Query(ctx context.Context, p retrievalmarket.RetrievalPeer, pieceCID []byte, params retrievalmarket.QueryParams) (retrievalmarket.QueryResponse, error) { - cid, err := cid.Cast(pieceCID) - if err != nil { - log.Warn(err) - return retrievalmarket.QueryResponseUndefined, err - } - - s, err := c.h.NewStream(ctx, p.ID, retrievalmarket.QueryProtocolID) - if err != nil { - log.Warn(err) - return retrievalmarket.QueryResponseUndefined, err - } - defer s.Close() - - err = cborutil.WriteCborRPC(s, &OldQuery{ - Piece: cid, - }) - if err != nil { - log.Warn(err) - return retrievalmarket.QueryResponseUndefined, err - } - - var oldResp OldQueryResponse - if err := oldResp.UnmarshalCBOR(s); err != nil { - log.Warn(err) - return retrievalmarket.QueryResponseUndefined, err - } - - resp := retrievalmarket.QueryResponse{ - Status: retrievalmarket.QueryResponseStatus(oldResp.Status), - Size: oldResp.Size, - MinPricePerByte: types.BigDiv(oldResp.MinPrice, types.NewInt(oldResp.Size)), - } - return resp, nil -} - -// TODO: Update to match spec for V0 Epic: -// https://github.com/filecoin-project/go-retrieval-market-project/issues/9 -func (c *client) Retrieve(ctx context.Context, pieceCID []byte, params retrievalmarket.Params, totalFunds types.BigInt, miner peer.ID, clientWallet retrievalmarket.Address, minerWallet retrievalmarket.Address) retrievalmarket.DealID { - /* The implementation of this function is just wrapper for the old code which retrieves UnixFS pieces - -- it will be replaced when we do the V0 implementation of the module */ - c.nextDealLk.Lock() - c.nextDealID++ - dealID := c.nextDealID - c.nextDealLk.Unlock() - - dealState := retrievalmarket.ClientDealState{ - DealProposal: retrievalmarket.DealProposal{ - PieceCID: pieceCID, - ID: dealID, - Params: params, - }, - Status: retrievalmarket.DealStatusFailed, - Sender: miner, - } - - go func() { - evt := retrievalmarket.ClientEventError - converted, err := cid.Cast(pieceCID) - - if err == nil { - err = c.retrieveUnixfs(ctx, converted, types.BigDiv(totalFunds, params.PricePerByte).Uint64(), totalFunds, miner, clientWallet, minerWallet) - if err == nil { - evt = retrievalmarket.ClientEventComplete - dealState.Status = retrievalmarket.DealStatusCompleted - } - } - - c.notifySubscribers(evt, dealState) - }() - - return dealID -} - -// unsubscribeAt returns a function that removes an item from the subscribers list by comparing -// their reflect.ValueOf before pulling the item out of the slice. Does not preserve order. -// Subsequent, repeated calls to the func with the same Subscriber are a no-op. -func (c *client) unsubscribeAt(sub retrievalmarket.ClientSubscriber) retrievalmarket.Unsubscribe { - return func() { - c.subscribersLk.Lock() - defer c.subscribersLk.Unlock() - curLen := len(c.subscribers) - for i, el := range c.subscribers { - if reflect.ValueOf(sub) == reflect.ValueOf(el) { - c.subscribers[i] = c.subscribers[curLen-1] - c.subscribers = c.subscribers[:curLen-1] - return - } - } - } -} - -func (c *client) notifySubscribers(evt retrievalmarket.ClientEvent, ds retrievalmarket.ClientDealState) { - c.subscribersLk.RLock() - defer c.subscribersLk.RUnlock() - for _, cb := range c.subscribers { - cb(evt, ds) - } -} - -func (c *client) SubscribeToEvents(subscriber retrievalmarket.ClientSubscriber) retrievalmarket.Unsubscribe { - c.subscribersLk.Lock() - c.subscribers = append(c.subscribers, subscriber) - c.subscribersLk.Unlock() - - return c.unsubscribeAt(subscriber) -} - -// V1 -func (c *client) AddMoreFunds(id retrievalmarket.DealID, amount types.BigInt) error { - panic("not implemented") -} - -func (c *client) CancelDeal(id retrievalmarket.DealID) error { - panic("not implemented") -} - -func (c *client) RetrievalStatus(id retrievalmarket.DealID) { - panic("not implemented") -} - -func (c *client) ListDeals() map[retrievalmarket.DealID]retrievalmarket.ClientDealState { - panic("not implemented") -} - -type clientStream struct { - node retrievalmarket.RetrievalClientNode - stream network.Stream - peeker cbg.BytePeeker - - root cid.Cid - size types.BigInt - offset uint64 - - paych retrievalmarket.Address - lane uint64 - total types.BigInt - transferred types.BigInt - - windowSize uint64 // how much we "trust" the peer - verifier BlockVerifier - bs blockstore.Blockstore -} - -/* This is the old retrieval code that is NOT spec compliant */ - -// C > S -// -// Offset MUST be aligned on chunking boundaries, size is rounded up to leaf size -// -// > DealProposal{Mode: Unixfs0, RootCid, Offset, Size, Payment(nil if free)} -// < Resp{Accept} -// < ..(Intermediate Block) -// < ..Blocks -// < ..(Intermediate Block) -// < ..Blocks -// > DealProposal(...) -// < ... -func (c *client) retrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, total types.BigInt, miner peer.ID, client, minerAddr retrievalmarket.Address) error { - s, err := c.h.NewStream(ctx, miner, retrievalmarket.ProtocolID) - if err != nil { - return err - } - defer s.Close() - - initialOffset := uint64(0) // TODO: Check how much data we have locally - // TODO: Support in handler - // TODO: Allow client to specify this - - paych, err := c.node.GetOrCreatePaymentChannel(ctx, client, minerAddr, total) - if err != nil { - return xerrors.Errorf("getting payment channel: %w", err) - } - lane, err := c.node.AllocateLane(paych) - if err != nil { - return xerrors.Errorf("allocating payment lane: %w", err) - } - - cst := clientStream{ - node: c.node, - stream: s, - peeker: cbg.GetPeeker(s), - - root: root, - size: types.NewInt(size), - offset: initialOffset, - - paych: paych, - lane: lane, - total: total, - transferred: types.NewInt(0), - - windowSize: build.UnixfsChunkSize, - verifier: &UnixFs0Verifier{Root: root}, - bs: c.bs, - } - - for cst.offset != size+initialOffset { - toFetch := cst.windowSize - if toFetch+cst.offset > size { - toFetch = size - cst.offset - } - log.Infof("Retrieve %dB @%d", toFetch, cst.offset) - - err := cst.doOneExchange(ctx, toFetch) - if err != nil { - return xerrors.Errorf("retrieval exchange: %w", err) - } - - cst.offset += toFetch - } - log.Info("RETRIEVE SUCCESSFUL") - return nil -} - -func (cst *clientStream) doOneExchange(ctx context.Context, toFetch uint64) error { - payAmount := types.BigDiv(types.BigMul(cst.total, types.NewInt(toFetch)), cst.size) - - payment, err := cst.setupPayment(ctx, payAmount) - if err != nil { - return xerrors.Errorf("setting up retrieval payment: %w", err) - } - - deal := &OldDealProposal{ - Payment: payment, - Ref: cst.root, - Params: RetParams{ - Unixfs0: &Unixfs0Offer{ - Offset: cst.offset, - Size: toFetch, - }, - }, - } - - if err := cborutil.WriteCborRPC(cst.stream, deal); err != nil { - return err - } - - var resp OldDealResponse - if err := cborutil.ReadCborRPC(cst.peeker, &resp); err != nil { - log.Error(err) - return err - } - - if resp.Status != Accepted { - cst.windowSize = build.UnixfsChunkSize - // TODO: apply some 'penalty' to miner 'reputation' (needs to be the same in both cases) - - if resp.Status == Error { - return xerrors.Errorf("storage deal error: %s", resp.Message) - } - if resp.Status == Rejected { - return xerrors.Errorf("storage deal rejected: %s", resp.Message) - } - return xerrors.New("storage deal response had no Accepted section") - } - - log.Info("Retrieval accepted, fetching blocks") - - return cst.fetchBlocks(toFetch) - - // TODO: maybe increase miner window size after success -} - -func (cst *clientStream) fetchBlocks(toFetch uint64) error { - blocksToFetch := (toFetch + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize - - for i := uint64(0); i < blocksToFetch; { - log.Infof("block %d of %d", i+1, blocksToFetch) - - var block Block - if err := cborutil.ReadCborRPC(cst.peeker, &block); err != nil { - return xerrors.Errorf("reading fetchBlock response: %w", err) - } - - dataBlocks, err := cst.consumeBlockMessage(block) - if err != nil { - return xerrors.Errorf("consuming retrieved blocks: %w", err) - } - - i += dataBlocks - } - - return nil -} - -func (cst *clientStream) consumeBlockMessage(block Block) (uint64, error) { - prefix, err := cid.PrefixFromBytes(block.Prefix) - if err != nil { - return 0, err - } - - cid, err := prefix.Sum(block.Data) - - blk, err := blocks.NewBlockWithCid(block.Data, cid) - if err != nil { - return 0, err - } - - internal, err := cst.verifier.Verify(context.TODO(), blk) - if err != nil { - log.Warnf("block verify failed: %s", err) - return 0, err - } - - // TODO: Smarter out, maybe add to filestore automagically - // (Also, persist intermediate nodes) - err = cst.bs.Put(blk) - if err != nil { - log.Warnf("block write failed: %s", err) - return 0, err - } - - if internal { - return 0, nil - } - - return 1, nil -} - -func (cst *clientStream) setupPayment(ctx context.Context, toSend types.BigInt) (api.PaymentInfo, error) { - amount := types.BigAdd(cst.transferred, toSend) - - sv, err := cst.node.CreatePaymentVoucher(ctx, cst.paych, amount, cst.lane) - if err != nil { - return api.PaymentInfo{}, err - } - - cst.transferred = amount - - return api.PaymentInfo{ - Channel: cst.paych, - ChannelMessage: nil, - Vouchers: []*types.SignedVoucher{sv}, - }, nil -} diff --git a/retrieval/impl/provider.go b/retrieval/impl/provider.go deleted file mode 100644 index 9e7254b60..000000000 --- a/retrieval/impl/provider.go +++ /dev/null @@ -1,323 +0,0 @@ -package retrievalimpl - -import ( - "context" - "io" - "reflect" - "sync" - - "github.com/ipfs/go-blockservice" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-merkledag" - unixfile "github.com/ipfs/go-unixfs/file" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/network" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/types" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" - "github.com/filecoin-project/lotus/storage/sectorblocks" -) - -// RetrMinerAPI are the node functions needed by a retrieval provider -type RetrMinerAPI interface { - PaychVoucherAdd(context.Context, address.Address, *types.SignedVoucher, []byte, types.BigInt) (types.BigInt, error) -} - -type provider struct { - - // TODO: Replace with RetrievalProviderNode & FileStore for https://github.com/filecoin-project/go-retrieval-market-project/issues/9 - sectorBlocks *sectorblocks.SectorBlocks - - // TODO: Replace with RetrievalProviderNode for - // https://github.com/filecoin-project/go-retrieval-market-project/issues/4 - node retrievalmarket.RetrievalProviderNode - - pricePerByte retrievalmarket.BigInt - - subscribersLk sync.RWMutex - subscribers []retrievalmarket.ProviderSubscriber -} - -// NewProvider returns a new retrieval provider -func NewProvider(sblks *sectorblocks.SectorBlocks, node retrievalmarket.RetrievalProviderNode) retrievalmarket.RetrievalProvider { - return &provider{ - sectorBlocks: sblks, - node: node, - - pricePerByte: types.NewInt(2), // TODO: allow setting - } -} - -// Start begins listening for deals on the given host -func (p *provider) Start(host host.Host) { - host.SetStreamHandler(retrievalmarket.QueryProtocolID, p.handleQueryStream) - host.SetStreamHandler(retrievalmarket.ProtocolID, p.handleDealStream) -} - -// V0 -// SetPricePerByte sets the price per byte a miner charges for retrievals -func (p *provider) SetPricePerByte(price retrievalmarket.BigInt) { - p.pricePerByte = price -} - -// SetPaymentInterval sets the maximum number of bytes a a provider will send before -// requesting further payment, and the rate at which that value increases -// TODO: Implement for https://github.com/filecoin-project/go-retrieval-market-project/issues/7 -func (p *provider) SetPaymentInterval(paymentInterval uint64, paymentIntervalIncrease uint64) { - panic("not implemented") -} - -// unsubscribeAt returns a function that removes an item from the subscribers list by comparing -// their reflect.ValueOf before pulling the item out of the slice. Does not preserve order. -// Subsequent, repeated calls to the func with the same Subscriber are a no-op. -func (p *provider) unsubscribeAt(sub retrievalmarket.ProviderSubscriber) retrievalmarket.Unsubscribe { - return func() { - p.subscribersLk.Lock() - defer p.subscribersLk.Unlock() - curLen := len(p.subscribers) - for i, el := range p.subscribers { - if reflect.ValueOf(sub) == reflect.ValueOf(el) { - p.subscribers[i] = p.subscribers[curLen-1] - p.subscribers = p.subscribers[:curLen-1] - return - } - } - } -} - -func (p *provider) notifySubscribers(evt retrievalmarket.ProviderEvent, ds retrievalmarket.ProviderDealState) { - p.subscribersLk.RLock() - defer p.subscribersLk.RUnlock() - for _, cb := range p.subscribers { - cb(evt, ds) - } -} - -// SubscribeToEvents listens for events that happen related to client retrievals -// TODO: Implement updates as part of https://github.com/filecoin-project/go-retrieval-market-project/issues/7 -func (p *provider) SubscribeToEvents(subscriber retrievalmarket.ProviderSubscriber) retrievalmarket.Unsubscribe { - p.subscribersLk.Lock() - p.subscribers = append(p.subscribers, subscriber) - p.subscribersLk.Unlock() - - return p.unsubscribeAt(subscriber) -} - -// V1 -func (p *provider) SetPricePerUnseal(price retrievalmarket.BigInt) { - panic("not implemented") -} - -func (p *provider) ListDeals() map[retrievalmarket.ProviderDealID]retrievalmarket.ProviderDealState { - panic("not implemented") -} - -func writeErr(stream network.Stream, err error) { - log.Errorf("Retrieval deal error: %+v", err) - _ = cborutil.WriteCborRPC(stream, &OldDealResponse{ - Status: Error, - Message: err.Error(), - }) -} - -// TODO: Update for https://github.com/filecoin-project/go-retrieval-market-project/issues/8 -func (p *provider) handleQueryStream(stream network.Stream) { - defer stream.Close() - - var query OldQuery - if err := cborutil.ReadCborRPC(stream, &query); err != nil { - writeErr(stream, err) - return - } - - size, err := p.sectorBlocks.GetSize(query.Piece) - if err != nil && err != sectorblocks.ErrNotFound { - log.Errorf("Retrieval query: GetRefs: %s", err) - return - } - - answer := &OldQueryResponse{ - Status: Unavailable, - } - if err == nil { - answer.Status = Available - - // TODO: get price, look for already unsealed ref to reduce work - answer.MinPrice = types.BigMul(types.NewInt(uint64(size)), p.pricePerByte) - answer.Size = uint64(size) // TODO: verify on intermediate - } - - if err := cborutil.WriteCborRPC(stream, answer); err != nil { - log.Errorf("Retrieval query: WriteCborRPC: %s", err) - return - } -} - -type handlerDeal struct { - p *provider - stream network.Stream - - ufsr sectorblocks.UnixfsReader - open cid.Cid - at uint64 - size uint64 -} - -// TODO: Update for https://github.com/filecoin-project/go-retrieval-market-project/issues/7 -func (p *provider) handleDealStream(stream network.Stream) { - defer stream.Close() - - hnd := &handlerDeal{ - p: p, - - stream: stream, - } - - var err error - more := true - - for more { - more, err = hnd.handleNext() // TODO: 'more' bool - if err != nil { - writeErr(stream, err) - return - } - } - -} - -func (hnd *handlerDeal) handleNext() (bool, error) { - var deal OldDealProposal - if err := cborutil.ReadCborRPC(hnd.stream, &deal); err != nil { - if err == io.EOF { // client sent all deals - err = nil - } - return false, err - } - - if deal.Params.Unixfs0 == nil { - return false, xerrors.New("unknown deal type") - } - - unixfs0 := deal.Params.Unixfs0 - - if len(deal.Payment.Vouchers) != 1 { - return false, xerrors.Errorf("expected one signed voucher, got %d", len(deal.Payment.Vouchers)) - } - - expPayment := types.BigMul(hnd.p.pricePerByte, types.NewInt(deal.Params.Unixfs0.Size)) - if _, err := hnd.p.node.SavePaymentVoucher(context.TODO(), deal.Payment.Channel, deal.Payment.Vouchers[0], nil, expPayment); err != nil { - return false, xerrors.Errorf("processing retrieval payment: %w", err) - } - - // If the file isn't open (new deal stream), isn't the right file, or isn't - // at the right offset, (re)open it - if hnd.open != deal.Ref || hnd.at != unixfs0.Offset { - log.Infof("opening file for sending (open '%s') (@%d, want %d)", deal.Ref, hnd.at, unixfs0.Offset) - if err := hnd.openFile(deal); err != nil { - return false, err - } - } - - if unixfs0.Offset+unixfs0.Size > hnd.size { - return false, xerrors.Errorf("tried to read too much %d+%d > %d", unixfs0.Offset, unixfs0.Size, hnd.size) - } - - err := hnd.accept(deal) - if err != nil { - return false, err - } - return true, nil -} - -func (hnd *handlerDeal) openFile(deal OldDealProposal) error { - unixfs0 := deal.Params.Unixfs0 - - if unixfs0.Offset != 0 { - // TODO: Implement SeekBlock (like ReadBlock) in go-unixfs - return xerrors.New("sending merkle proofs for nonzero offset not supported yet") - } - hnd.at = unixfs0.Offset - - bstore := hnd.p.sectorBlocks.SealedBlockstore(func() error { - return nil // TODO: approve unsealing based on amount paid - }) - - ds := merkledag.NewDAGService(blockservice.New(bstore, nil)) - rootNd, err := ds.Get(context.TODO(), deal.Ref) - if err != nil { - return err - } - - fsr, err := unixfile.NewUnixfsFile(context.TODO(), ds, rootNd) - if err != nil { - return err - } - - var ok bool - hnd.ufsr, ok = fsr.(sectorblocks.UnixfsReader) - if !ok { - return xerrors.Errorf("file %s didn't implement sectorblocks.UnixfsReader", deal.Ref) - } - - isize, err := hnd.ufsr.Size() - if err != nil { - return err - } - hnd.size = uint64(isize) - - hnd.open = deal.Ref - - return nil -} - -func (hnd *handlerDeal) accept(deal OldDealProposal) error { - unixfs0 := deal.Params.Unixfs0 - - resp := &OldDealResponse{ - Status: Accepted, - } - if err := cborutil.WriteCborRPC(hnd.stream, resp); err != nil { - log.Errorf("Retrieval query: Write Accepted resp: %s", err) - return err - } - - blocksToSend := (unixfs0.Size + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize - for i := uint64(0); i < blocksToSend; { - data, offset, nd, err := hnd.ufsr.ReadBlock(context.TODO()) - if err != nil { - return err - } - - log.Infof("sending block for a deal: %s", nd.Cid()) - - if offset != unixfs0.Offset { - return xerrors.Errorf("ReadBlock on wrong offset: want %d, got %d", unixfs0.Offset, offset) - } - - /*if uint64(len(data)) != deal.Unixfs0.Size { // TODO: Fix for internal nodes (and any other node too) - writeErr(stream, xerrors.Errorf("ReadBlock data with wrong size: want %d, got %d", deal.Unixfs0.Size, len(data))) - return - }*/ - - block := &Block{ - Prefix: nd.Cid().Prefix().Bytes(), - Data: nd.RawData(), - } - - if err := cborutil.WriteCborRPC(hnd.stream, block); err != nil { - return err - } - - if len(data) > 0 { // don't count internal nodes - hnd.at += uint64(len(data)) - i++ - } - } - - return nil -} diff --git a/retrieval/impl/run_cbor_gen.go b/retrieval/impl/run_cbor_gen.go deleted file mode 100644 index 8dbd6f19c..000000000 --- a/retrieval/impl/run_cbor_gen.go +++ /dev/null @@ -1,33 +0,0 @@ -package retrievalimpl - -import ( - "fmt" - "os" - - cborgen "github.com/whyrusleeping/cbor-gen" -) - -func RunCborGen() error { - genName := "./impl/cbor_gen.go" - reName := "./impl/cbor_gen_old.go" - if err := os.Rename(genName, reName); err != nil { - return fmt.Errorf("could not rename %s to %s", genName, reName) - } - if err := cborgen.WriteTupleEncodersToFile( - genName, - "retrievalimpl", - RetParams{}, - OldQuery{}, - OldQueryResponse{}, - Unixfs0Offer{}, - OldDealProposal{}, - OldDealResponse{}, - Block{}, - ); err != nil { - return err - } - if err := os.Remove(reName); err != nil { - return err - } - return nil -} diff --git a/retrieval/impl/types.go b/retrieval/impl/types.go deleted file mode 100644 index 3af74f993..000000000 --- a/retrieval/impl/types.go +++ /dev/null @@ -1,67 +0,0 @@ -package retrievalimpl - -import ( - "github.com/filecoin-project/lotus/api" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lotus/chain/types" -) - -/* These types are all the types provided by Lotus, which diverge even from -spec V0 -- prior to the "update to spec epic", we are using these types internally -and switching to spec at the boundaries of the module */ - -type OldQueryResponseStatus uint64 - -const ( - Available OldQueryResponseStatus = iota - Unavailable -) - -const ( - Accepted = iota - Error - Rejected - Unsealing -) - -type OldQuery struct { - Piece cid.Cid - // TODO: payment -} - -type OldQueryResponse struct { - Status OldQueryResponseStatus - - Size uint64 // TODO: spec - // TODO: unseal price (+spec) - // TODO: sectors to unseal - // TODO: address to send money for the deal? - MinPrice types.BigInt -} - -type Unixfs0Offer struct { - Offset uint64 - Size uint64 -} - -type RetParams struct { - Unixfs0 *Unixfs0Offer -} - -type OldDealProposal struct { - Payment api.PaymentInfo - - Ref cid.Cid - Params RetParams -} - -type OldDealResponse struct { - Status uint64 - Message string -} - -type Block struct { // TODO: put in spec - Prefix []byte // TODO: fix cid.Prefix marshaling somehow - Data []byte -} diff --git a/retrieval/impl/verify.go b/retrieval/impl/verify.go deleted file mode 100644 index 892876ef5..000000000 --- a/retrieval/impl/verify.go +++ /dev/null @@ -1,141 +0,0 @@ -package retrievalimpl - -import ( - "context" - - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - ipld "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" - "github.com/ipfs/go-unixfs" - pb "github.com/ipfs/go-unixfs/pb" - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/build" -) - -type BlockVerifier interface { - Verify(context.Context, blocks.Block) (internal bool, err error) -} - -type OptimisticVerifier struct { -} - -func (o *OptimisticVerifier) Verify(context.Context, blocks.Block) (bool, error) { - // It's probably fine - return false, nil -} - -type UnixFs0Verifier struct { - Root cid.Cid - rootBlk blocks.Block - - expect int - seen int - - sub *UnixFs0Verifier -} - -func (b *UnixFs0Verifier) verify(ctx context.Context, blk blocks.Block) (last bool, internal bool, err error) { - if b.sub != nil { - // TODO: check links here (iff b.sub.sub == nil) - - subLast, internal, err := b.sub.verify(ctx, blk) - if err != nil { - return false, false, err - } - if subLast { - b.sub = nil - b.seen++ - } - - return b.seen == b.expect, internal, nil - } - - if b.seen >= b.expect { // this is probably impossible - return false, false, xerrors.New("unixfs verifier: too many nodes in level") - } - - links, err := b.checkInternal(blk) - if err != nil { - return false, false, err - } - - if links > 0 { // TODO: check if all links are intermediate (or all aren't) - if links > build.UnixfsLinksPerLevel { - return false, false, xerrors.New("unixfs verifier: too many links in intermediate node") - } - - if b.seen+1 == b.expect && links != build.UnixfsLinksPerLevel { - return false, false, xerrors.New("unixfs verifier: too few nodes in level") - } - - b.sub = &UnixFs0Verifier{ - Root: blk.Cid(), - rootBlk: blk, - expect: links, - } - - // don't mark as seen yet - return false, true, nil - } - - b.seen++ - return b.seen == b.expect, false, nil -} - -func (b *UnixFs0Verifier) checkInternal(blk blocks.Block) (int, error) { - nd, err := ipld.Decode(blk) - if err != nil { - log.Warnf("IPLD Decode failed: %s", err) - return 0, err - } - - // TODO: check size - switch nd := nd.(type) { - case *merkledag.ProtoNode: - fsn, err := unixfs.FSNodeFromBytes(nd.Data()) - if err != nil { - log.Warnf("unixfs.FSNodeFromBytes failed: %s", err) - return 0, err - } - if fsn.Type() != pb.Data_File { - return 0, xerrors.New("internal nodes must be a file") - } - if len(fsn.Data()) > 0 { - return 0, xerrors.New("internal node with data") - } - if len(nd.Links()) == 0 { - return 0, xerrors.New("internal node with no links") - } - return len(nd.Links()), nil - - case *merkledag.RawNode: - return 0, nil - default: - return 0, xerrors.New("verifier: unknown node type") - } -} - -func (b *UnixFs0Verifier) Verify(ctx context.Context, blk blocks.Block) (bool, error) { - // root is special - if b.rootBlk == nil { - if !b.Root.Equals(blk.Cid()) { - return false, xerrors.Errorf("unixfs verifier: root block CID didn't match: valid %s, got %s", b.Root, blk.Cid()) - } - b.rootBlk = blk - links, err := b.checkInternal(blk) - if err != nil { - return false, err - } - - b.expect = links - return links != 0, nil - } - - _, internal, err := b.verify(ctx, blk) - return internal, err -} - -var _ BlockVerifier = &OptimisticVerifier{} -var _ BlockVerifier = &UnixFs0Verifier{} diff --git a/retrieval/types.go b/retrieval/types.go deleted file mode 100644 index f82279a9d..000000000 --- a/retrieval/types.go +++ /dev/null @@ -1,364 +0,0 @@ -package retrievalmarket - -import ( - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/peer" -) - -// type aliases -// TODO: Remove and use native types or extract for -// https://github.com/filecoin-project/go-retrieval-market-project/issues/5 - -// BigInt is used for token amounts in retrieval deals -type BigInt = types.BigInt - -// Address is an address in the filecoin network -type Address = address.Address - -// SignedVoucher is a signed payment voucher -type SignedVoucher = types.SignedVoucher - -// ProtocolID is the protocol for proposing / responding to retrieval deals -const ProtocolID = "/fil/retrieval/0.0.1" - -// QueryProtocolID is the protocol for querying infromation about retrieval -// deal parameters -const QueryProtocolID = "/fil/retrieval/qry/0.0.1" // TODO: spec - -// Unsubscribe is a function that unsubscribes a subscriber for either the -// client or the provider -type Unsubscribe func() - -// ClientDealState is the current state of a deal from the point of view -// of a retrieval client -type ClientDealState struct { - DealProposal - Status DealStatus - Sender peer.ID - TotalReceived uint64 - FundsSpent BigInt -} - -// ClientEvent is an event that occurs in a deal lifecycle on the client -type ClientEvent uint64 - -const ( - // ClientEventOpen indicates a deal was initiated - ClientEventOpen ClientEvent = iota - - // ClientEventFundsExpended indicates a deal has run out of funds in the payment channel - // forcing the client to add more funds to continue the deal - ClientEventFundsExpended // when totalFunds is expended - - // ClientEventProgress indicates more data was received for a retrieval - ClientEventProgress - - // ClientEventError indicates an error occurred during a deal - ClientEventError - - // ClientEventComplete indicates a deal has completed - ClientEventComplete -) - -// ClientSubscriber is a callback that is registered to listen for retrieval events -type ClientSubscriber func(event ClientEvent, state ClientDealState) - -// RetrievalClient is a client interface for making retrieval deals -type RetrievalClient interface { - // V0 - - // Find Providers finds retrieval providers who may be storing a given piece - FindProviders(pieceCID []byte) []RetrievalPeer - - // Query asks a provider for information about a piece it is storing - Query( - ctx context.Context, - p RetrievalPeer, - pieceCID []byte, - params QueryParams, - ) (QueryResponse, error) - - // Retrieve retrieves all or part of a piece with the given retrieval parameters - Retrieve( - ctx context.Context, - pieceCID []byte, - params Params, - totalFunds BigInt, - miner peer.ID, - clientWallet Address, - minerWallet Address, - ) DealID - - // SubscribeToEvents listens for events that happen related to client retrievals - SubscribeToEvents(subscriber ClientSubscriber) Unsubscribe - - // V1 - AddMoreFunds(id DealID, amount BigInt) error - CancelDeal(id DealID) error - RetrievalStatus(id DealID) - ListDeals() map[DealID]ClientDealState -} - -// RetrievalClientNode are the node depedencies for a RetrevalClient -type RetrievalClientNode interface { - - // GetOrCreatePaymentChannel sets up a new payment channel if one does not exist - // between a client and a miner and insures the client has the given amount of funds available in the channel - GetOrCreatePaymentChannel(ctx context.Context, clientAddress Address, minerAddress Address, clientFundsAvailable BigInt) (Address, error) - - // Allocate late creates a lane within a payment channel so that calls to - // CreatePaymentVoucher will automatically make vouchers only for the difference - // in total - AllocateLane(paymentChannel Address) (uint64, error) - - // CreatePaymentVoucher creates a new payment voucher in the given lane for a - // given payment channel so that all the payment vouchers in the lane add up - // to the given amount (so the payment voucher will be for the difference) - CreatePaymentVoucher(ctx context.Context, paymentChannel Address, amount BigInt, lane uint64) (*SignedVoucher, error) -} - -// ProviderDealState is the current state of a deal from the point of view -// of a retrieval provider -type ProviderDealState struct { - DealProposal - Status DealStatus - Receiver peer.ID - TotalSent uint64 - FundsReceived BigInt -} - -// ProviderEvent is an event that occurs in a deal lifecycle on the provider -type ProviderEvent uint64 - -const ( - - // ProviderEventOpen indicates a new deal was received from a client - ProviderEventOpen ProviderEvent = iota - - // ProviderEventProgress indicates more data was sent to a client - ProviderEventProgress - - // ProviderEventError indicates an error occurred in processing a deal for a client - ProviderEventError - - // ProviderEventComplete indicates a retrieval deal was completed for a client - ProviderEventComplete -) - -// ProviderDealID is a unique identifier for a deal on a provider -- it is -// a combination of DealID set by the client and the peer ID of the client -type ProviderDealID struct { - From peer.ID - ID DealID -} - -// ProviderSubscriber is a callback that is registered to listen for retrieval events on a provider -type ProviderSubscriber func(event ProviderEvent, state ProviderDealState) - -// RetrievalProvider is an interface by which a provider configures their -// retrieval operations and monitors deals received and process -type RetrievalProvider interface { - // Start begins listening for deals on the given host - Start(host.Host) - - // V0 - - // SetPricePerByte sets the price per byte a miner charges for retrievals - SetPricePerByte(price BigInt) - - // SetPaymentInterval sets the maximum number of bytes a a provider will send before - // requesting further payment, and the rate at which that value increases - SetPaymentInterval(paymentInterval uint64, paymentIntervalIncrease uint64) - - // SubscribeToEvents listens for events that happen related to client retrievals - SubscribeToEvents(subscriber ProviderSubscriber) Unsubscribe - - // V1 - SetPricePerUnseal(price BigInt) - ListDeals() map[ProviderDealID]ProviderDealState -} - -// RetrievalProviderNode are the node depedencies for a RetrevalProvider -type RetrievalProviderNode interface { - SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *SignedVoucher, proof []byte, expectedAmount BigInt) (BigInt, error) -} - -// PeerResolver is an interface for looking up providers that may have a piece -type PeerResolver interface { - GetPeers(data cid.Cid) ([]RetrievalPeer, error) // TODO: channel -} - -// RetrievalPeer is a provider address/peer.ID pair (everything needed to make -// deals for with a miner) -type RetrievalPeer struct { - Address Address - ID peer.ID // optional -} - -// QueryResponseStatus indicates whether a queried piece is available -type QueryResponseStatus uint64 - -const ( - // QueryResponseAvailable indicates a provider has a piece and is prepared to - // return it - QueryResponseAvailable QueryResponseStatus = iota - - // QueryResponseUnavailable indicates a provider either does not have or cannot - // serve the queried piece to the client - QueryResponseUnavailable -) - -// QueryItemStatus (V1) indicates whether the requested part of a piece (payload or selector) -// is available for retrieval -type QueryItemStatus uint64 - -const ( - // QueryItemAvailable indicates requested part of the piece is available to be - // served - QueryItemAvailable QueryItemStatus = iota - - // QueryItemUnavailable indicates the piece either does not contain the requested - // item or it cannot be served - QueryItemUnavailable - - // QueryItemUnknown indicates the provider cannot determine if the given item - // is part of the requested piece (for example, if the piece is sealed and the - // miner does not maintain a payload CID index) - QueryItemUnknown -) - -// QueryParams indicate what specific information about a piece that a retrieval -// client is interested in, as well as specific parameters the client is seeking -// for the retrieval deal -type QueryParams struct { - PayloadCID cid.Cid // optional, query if miner has this cid in this piece. some miners may not be able to respond. - Selector ipld.Node // optional, query if miner has this cid in this piece. some miners may not be able to respond. - MaxPricePerByte BigInt // optional, tell miner uninterested if more expensive than this - MinPaymentInterval uint64 // optional, tell miner uninterested unless payment interval is greater than this - MinPaymentIntervalIncrease uint64 // optional, tell miner uninterested unless payment interval increase is greater than this -} - -// Query is a query to a given provider to determine information about a piece -// they may have available for retrieval -type Query struct { - PieceCID []byte // V0 - // QueryParams // V1 -} - -// QueryResponse is a miners response to a given retrieval query -type QueryResponse struct { - Status QueryResponseStatus - //PayloadCIDFound QueryItemStatus // V1 - if a PayloadCid was requested, the result - //SelectorFound QueryItemStatus // V1 - if a Selector was requested, the result - - Size uint64 // Total size of piece in bytes - //ExpectedPayloadSize uint64 // V1 - optional, if PayloadCID + selector are specified and miner knows, can offer an expected size - - PaymentAddress Address // address to send funds to -- may be different than miner addr - MinPricePerByte BigInt - MaxPaymentInterval uint64 - MaxPaymentIntervalIncrease uint64 -} - -// QueryResponseUndefined is an empty QueryResponse -var QueryResponseUndefined = QueryResponse{} - -// PieceRetrievalPrice is the total price to retrieve the piece (size * MinPricePerByte) -func (qr QueryResponse) PieceRetrievalPrice() BigInt { - return types.BigMul(qr.MinPricePerByte, types.NewInt(qr.Size)) -} - -// PayloadRetrievalPrice is the expected price to retrieve just the given payload -// & selector (V1) -//func (qr QueryResponse) PayloadRetrievalPrice() BigInt { -// return types.BigMul(qr.MinPricePerByte, types.NewInt(qr.ExpectedPayloadSize)) -//} - -// DealStatus is the status of a retrieval deal returned by a provider -// in a DealResponse -type DealStatus uint64 - -const ( - // DealStatusAccepted means a deal has been accepted by a provider - // and its is ready to proceed with retrieval - DealStatusAccepted DealStatus = iota - - // DealStatusFailed indicates something went wrong during a retrieval - DealStatusFailed - - // DealStatusRejected indicates the provider rejected a client's deal proposal - // for some reason - DealStatusRejected - - // DealStatusUnsealing indicates the provider is currently unsealing the sector - // needed to serve the retrieval deal - DealStatusUnsealing - - // DealStatusFundsNeeded indicates the provider is awaiting a payment voucher to - // continue processing the deal - DealStatusFundsNeeded - - // DealStatusOngoing indicates the provider is continuing to process a deal - DealStatusOngoing - - // DealStatusFundsNeededLastPayment indicates the provider is awaiting funds for - // a final payment in order to complete a deal - DealStatusFundsNeededLastPayment - - // DealStatusCompleted indicates a deal is complete - DealStatusCompleted - - // DealStatusDealNotFound indicates an update was received for a deal that could - // not be identified - DealStatusDealNotFound -) - -// Params are the parameters requested for a retrieval deal proposal -type Params struct { - //PayloadCID cid.Cid // V1 - //Selector ipld.Node // V1 - PricePerByte BigInt - PaymentInterval uint64 - PaymentIntervalIncrease uint64 -} - -// DealID is an identifier for a retrieval deal (unique to a client) -type DealID uint64 - -// DealProposal is a proposal for a new retrieval deal -type DealProposal struct { - PieceCID []byte - ID DealID - Params -} - -// Block is an IPLD block in bitswap format -type Block struct { - Prefix []byte - Data []byte -} - -// DealResponse is a response to a retrieval deal proposal -type DealResponse struct { - Status DealStatus - ID DealID - - // payment required to proceed - PaymentOwed BigInt - - Message string - Blocks []Block // V0 only -} - -// DealPayment is a payment for an in progress retrieval deal -type DealPayment struct { - ID DealID - PaymentChannel address.Address - PaymentVoucher *types.SignedVoucher -} diff --git a/retrievaladapter/client.go b/retrievaladapter/client.go index ff7d2f0fb..ae62632bd 100644 --- a/retrievaladapter/client.go +++ b/retrievaladapter/client.go @@ -3,9 +3,13 @@ package retrievaladapter import ( "context" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-components/retrievalmarket" + retrievaltoken "github.com/filecoin-project/go-fil-components/shared/tokenamount" + retrievaltypes "github.com/filecoin-project/go-fil-components/shared/types" + payapi "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/paych" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" ) type retrievalClientNode struct { @@ -21,21 +25,25 @@ func NewRetrievalClientNode(pmgr *paych.Manager, payapi payapi.PaychAPI) retriev // GetOrCreatePaymentChannel sets up a new payment channel if one does not exist // between a client and a miner and insures the client has the given amount of funds available in the channel -func (rcn *retrievalClientNode) GetOrCreatePaymentChannel(ctx context.Context, clientAddress retrievalmarket.Address, minerAddress retrievalmarket.Address, clientFundsAvailable retrievalmarket.BigInt) (retrievalmarket.Address, error) { - paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, clientFundsAvailable) +func (rcn *retrievalClientNode) GetOrCreatePaymentChannel(ctx context.Context, clientAddress address.Address, minerAddress address.Address, clientFundsAvailable retrievaltoken.TokenAmount) (address.Address, error) { + paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, FromSharedTokenAmount(clientFundsAvailable)) return paych, err } // Allocate late creates a lane within a payment channel so that calls to // CreatePaymentVoucher will automatically make vouchers only for the difference // in total -func (rcn *retrievalClientNode) AllocateLane(paymentChannel retrievalmarket.Address) (uint64, error) { +func (rcn *retrievalClientNode) AllocateLane(paymentChannel address.Address) (uint64, error) { return rcn.pmgr.AllocateLane(paymentChannel) } // CreatePaymentVoucher creates a new payment voucher in the given lane for a // given payment channel so that all the payment vouchers in the lane add up // to the given amount (so the payment voucher will be for the difference) -func (rcn *retrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel retrievalmarket.Address, amount retrievalmarket.BigInt, lane uint64) (*retrievalmarket.SignedVoucher, error) { - return rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, amount, lane) +func (rcn *retrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount retrievaltoken.TokenAmount, lane uint64) (*retrievaltypes.SignedVoucher, error) { + voucher, err := rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, FromSharedTokenAmount(amount), lane) + if err != nil { + return nil, err + } + return ToSharedSignedVoucher(voucher) } diff --git a/retrievaladapter/converters.go b/retrievaladapter/converters.go new file mode 100644 index 000000000..069c3ea34 --- /dev/null +++ b/retrievaladapter/converters.go @@ -0,0 +1,45 @@ +package retrievaladapter + +import ( + "bytes" + sharedamount "github.com/filecoin-project/go-fil-components/shared/tokenamount" + sharedtypes "github.com/filecoin-project/go-fil-components/shared/types" + "github.com/filecoin-project/lotus/chain/types" +) + + +func FromSharedTokenAmount(in sharedamount.TokenAmount) types.BigInt { + return types.BigInt{Int: in.Int} +} + +func ToSharedTokenAmount(in types.BigInt) sharedamount.TokenAmount { + return sharedamount.TokenAmount{Int: in.Int} +} + +func ToSharedSignedVoucher(in *types.SignedVoucher) (*sharedtypes.SignedVoucher, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out sharedtypes.SignedVoucher + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSharedSignedVoucher(in *sharedtypes.SignedVoucher) (*types.SignedVoucher, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out types.SignedVoucher + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} diff --git a/retrievaladapter/provider.go b/retrievaladapter/provider.go index 06757593b..fffd2c7b5 100644 --- a/retrievaladapter/provider.go +++ b/retrievaladapter/provider.go @@ -4,20 +4,43 @@ import ( "context" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-components/retrievalmarket" + retrievaltoken "github.com/filecoin-project/go-fil-components/shared/tokenamount" + retrievaltypes "github.com/filecoin-project/go-fil-components/shared/types" "github.com/filecoin-project/lotus/api" - retrievalmarket "github.com/filecoin-project/lotus/retrieval" + "github.com/filecoin-project/lotus/storage/sectorblocks" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" ) type retrievalProviderNode struct { - full api.FullNode + sectorBlocks *sectorblocks.SectorBlocks + full api.FullNode } // NewRetrievalProviderNode returns a new node adapter for a retrieval provider that talks to the // Lotus Node -func NewRetrievalProviderNode(full api.FullNode) retrievalmarket.RetrievalProviderNode { - return &retrievalProviderNode{full} +func NewRetrievalProviderNode(sectorBlocks *sectorblocks.SectorBlocks, full api.FullNode) retrievalmarket.RetrievalProviderNode { + return &retrievalProviderNode{sectorBlocks, full} } -func (rpn *retrievalProviderNode) SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *retrievalmarket.SignedVoucher, proof []byte, expectedAmount retrievalmarket.BigInt) (retrievalmarket.BigInt, error) { - return rpn.full.PaychVoucherAdd(ctx, paymentChannel, voucher, proof, expectedAmount) +func (rpn *retrievalProviderNode) GetPieceSize(pieceCid []byte) (uint64, error) { + asCid, err := cid.Cast(pieceCid) + if err != nil { + return 0, err + } + return rpn.sectorBlocks.GetSize(asCid) +} + +func (rpn *retrievalProviderNode) SealedBlockstore(approveUnseal func() error) blockstore.Blockstore { + return rpn.sectorBlocks.SealedBlockstore(approveUnseal) +} + +func (rpn *retrievalProviderNode) SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *retrievaltypes.SignedVoucher, proof []byte, expectedAmount retrievaltoken.TokenAmount) (retrievaltoken.TokenAmount, error) { + localVoucher, err := FromSharedSignedVoucher(voucher) + if err != nil { + return retrievaltoken.FromInt(0), err + } + added, err := rpn.full.PaychVoucherAdd(ctx, paymentChannel, localVoucher, proof, FromSharedTokenAmount(expectedAmount)) + return ToSharedTokenAmount(added), err } From 0a4eeb7cb1e4ae1873e6cc8f2c77cfbb798ff5b6 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 10 Jan 2020 09:13:12 -0800 Subject: [PATCH 2/6] refactor(retrievalmarket): switch to go-fil-markets repo Switch go-fil-components to go-fil-markets --- chain/deals/client.go | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- node/builder.go | 4 ++-- node/impl/client/client.go | 2 +- node/modules/client.go | 4 ++-- node/modules/services.go | 4 ++-- node/modules/storageminer.go | 4 ++-- retrievaladapter/client.go | 6 +++--- retrievaladapter/converters.go | 4 ++-- retrievaladapter/provider.go | 6 +++--- 11 files changed, 22 insertions(+), 22 deletions(-) diff --git a/chain/deals/client.go b/chain/deals/client.go index 02c9bcce8..e1067961c 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -12,8 +12,8 @@ import ( "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-fil-components/retrievalmarket" - "github.com/filecoin-project/go-fil-components/retrievalmarket/discovery" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" diff --git a/go.mod b/go.mod index 665c367c4..1bcb04ec4 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce - github.com/filecoin-project/go-fil-components v0.0.0-20200110033229-671d79221133 + github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42 github.com/filecoin-project/go-paramfetch v0.0.1 github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254 github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 diff --git a/go.sum b/go.sum index dc1b26e42..0e14ed7b1 100644 --- a/go.sum +++ b/go.sum @@ -111,8 +111,8 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= -github.com/filecoin-project/go-fil-components v0.0.0-20200110033229-671d79221133 h1:/L916kY3hyq8w18rLO9VMSHqw25/9pwRB3nVW6b+Sm4= -github.com/filecoin-project/go-fil-components v0.0.0-20200110033229-671d79221133/go.mod h1:7M0YUI2CSVmqEmXNeXNq5L/pjk7C1Q5ifhirfMADD/k= +github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42 h1:7OW2AiWtwxwtYnC16CiXZ/idQ7w3W7W1hhX+N4YSAyQ= +github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42/go.mod h1:Q5fvJGMISyUIna19DpoqiqTas4L9RVD7w3Udv7uCrTo= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= diff --git a/node/builder.go b/node/builder.go index d263e04fa..ee7286908 100644 --- a/node/builder.go +++ b/node/builder.go @@ -19,8 +19,8 @@ import ( "go.uber.org/fx" "golang.org/x/xerrors" - "github.com/filecoin-project/go-fil-components/retrievalmarket" - "github.com/filecoin-project/go-fil-components/retrievalmarket/discovery" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/blocksync" diff --git a/node/impl/client/client.go b/node/impl/client/client.go index a8b7ff17f..09bd5d9ee 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -25,7 +25,7 @@ import ( "go.uber.org/fx" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-fil-components/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/store" diff --git a/node/modules/client.go b/node/modules/client.go index 67c6322ab..a03ea3691 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -6,8 +6,8 @@ import ( "path/filepath" "reflect" - "github.com/filecoin-project/go-fil-components/retrievalmarket" - retrievalimpl "github.com/filecoin-project/go-fil-components/retrievalmarket/impl" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/paych" diff --git a/node/modules/services.go b/node/modules/services.go index bc99d49f3..bf1985041 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -8,8 +8,8 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" "go.uber.org/fx" - "github.com/filecoin-project/go-fil-components/retrievalmarket" - "github.com/filecoin-project/go-fil-components/retrievalmarket/discovery" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/blocksync" "github.com/filecoin-project/lotus/chain/messagepool" diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 876c2f368..1197870d5 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -25,8 +25,8 @@ import ( "github.com/filecoin-project/go-address" dtgraphsync "github.com/filecoin-project/go-data-transfer/impl/graphsync" - "github.com/filecoin-project/go-fil-components/retrievalmarket" - retrievalimpl "github.com/filecoin-project/go-fil-components/retrievalmarket/impl" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" diff --git a/retrievaladapter/client.go b/retrievaladapter/client.go index ae62632bd..8f0289921 100644 --- a/retrievaladapter/client.go +++ b/retrievaladapter/client.go @@ -4,9 +4,9 @@ import ( "context" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-fil-components/retrievalmarket" - retrievaltoken "github.com/filecoin-project/go-fil-components/shared/tokenamount" - retrievaltypes "github.com/filecoin-project/go-fil-components/shared/types" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + retrievaltoken "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + retrievaltypes "github.com/filecoin-project/go-fil-markets/shared/types" payapi "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/paych" diff --git a/retrievaladapter/converters.go b/retrievaladapter/converters.go index 069c3ea34..7ea3ffec0 100644 --- a/retrievaladapter/converters.go +++ b/retrievaladapter/converters.go @@ -2,8 +2,8 @@ package retrievaladapter import ( "bytes" - sharedamount "github.com/filecoin-project/go-fil-components/shared/tokenamount" - sharedtypes "github.com/filecoin-project/go-fil-components/shared/types" + sharedamount "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/lotus/chain/types" ) diff --git a/retrievaladapter/provider.go b/retrievaladapter/provider.go index fffd2c7b5..c5b238eda 100644 --- a/retrievaladapter/provider.go +++ b/retrievaladapter/provider.go @@ -4,9 +4,9 @@ import ( "context" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-fil-components/retrievalmarket" - retrievaltoken "github.com/filecoin-project/go-fil-components/shared/tokenamount" - retrievaltypes "github.com/filecoin-project/go-fil-components/shared/types" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + retrievaltoken "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + retrievaltypes "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/storage/sectorblocks" "github.com/ipfs/go-cid" From 65ecb33630a7ff9077d8e26d059500a47fb72d22 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Tue, 17 Dec 2019 02:46:39 -0800 Subject: [PATCH 3/6] feat(storagemarket): extract storage market remove all shared components, point at go-fil-components repo --- chain/deals/cbor_gen.go | 784 ----------------------- chain/deals/client.go | 293 --------- chain/deals/client_states.go | 97 --- chain/deals/client_storagemarket.go | 118 ---- chain/deals/client_utils.go | 170 ----- chain/deals/provider.go | 293 --------- chain/deals/provider_asks.go | 143 ----- chain/deals/provider_states.go | 172 ----- chain/deals/provider_storagemarket.go | 65 -- chain/deals/provider_utils.go | 180 ------ chain/deals/request_validation_test.go | 274 -------- chain/deals/types.go | 112 ---- cmd/lotus-storage-miner/init.go | 26 +- gen/main.go | 27 - go.mod | 1 - lib/sharedutils/converters.go | 101 +++ node/builder.go | 12 +- node/impl/client/client.go | 43 +- node/modules/client.go | 18 +- node/modules/services.go | 2 +- node/modules/storageminer.go | 15 +- retrievaladapter/client.go | 7 +- retrievaladapter/converters.go | 45 -- retrievaladapter/provider.go | 7 +- storagemarket/bigint.go | 241 ------- storagemarket/cbor_gen.go | 352 ---------- storagemarket/types.go | 199 ------ storagemarketadapter/client_adapter.go | 55 +- storagemarketadapter/provider_adapter.go | 48 +- storagemarketadapter/utilities.go | 79 +++ 30 files changed, 325 insertions(+), 3654 deletions(-) delete mode 100644 chain/deals/cbor_gen.go delete mode 100644 chain/deals/client.go delete mode 100644 chain/deals/client_states.go delete mode 100644 chain/deals/client_storagemarket.go delete mode 100644 chain/deals/client_utils.go delete mode 100644 chain/deals/provider.go delete mode 100644 chain/deals/provider_asks.go delete mode 100644 chain/deals/provider_states.go delete mode 100644 chain/deals/provider_storagemarket.go delete mode 100644 chain/deals/provider_utils.go delete mode 100644 chain/deals/request_validation_test.go delete mode 100644 chain/deals/types.go create mode 100644 lib/sharedutils/converters.go delete mode 100644 retrievaladapter/converters.go delete mode 100644 storagemarket/bigint.go delete mode 100644 storagemarket/cbor_gen.go delete mode 100644 storagemarket/types.go create mode 100644 storagemarketadapter/utilities.go diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go deleted file mode 100644 index 5abb92825..000000000 --- a/chain/deals/cbor_gen.go +++ /dev/null @@ -1,784 +0,0 @@ -package deals - -import ( - "fmt" - "io" - - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" - "github.com/libp2p/go-libp2p-core/peer" - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -var _ = xerrors.Errorf - -func (t *AskRequest) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{129}); err != nil { - return err - } - - // t.Miner (address.Address) (struct) - if err := t.Miner.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *AskRequest) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 1 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Miner (address.Address) (struct) - - { - - if err := t.Miner.UnmarshalCBOR(br); err != nil { - return err - } - - } - return nil -} - -func (t *AskResponse) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{129}); err != nil { - return err - } - - // t.Ask (types.SignedStorageAsk) (struct) - if err := t.Ask.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *AskResponse) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 1 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Ask (types.SignedStorageAsk) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.Ask = new(types.SignedStorageAsk) - if err := t.Ask.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - return nil -} - -func (t *Proposal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.DealProposal (actors.StorageDealProposal) (struct) - if err := t.DealProposal.MarshalCBOR(w); err != nil { - return err - } - - // t.Piece (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Piece); err != nil { - return xerrors.Errorf("failed to write cid field t.Piece: %w", err) - } - - return nil -} - -func (t *Proposal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.DealProposal (actors.StorageDealProposal) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.DealProposal = new(actors.StorageDealProposal) - if err := t.DealProposal.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - // t.Piece (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Piece: %w", err) - } - - t.Piece = c - - } - return nil -} - -func (t *Response) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{132}); err != nil { - return err - } - - // t.State (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { - return err - } - - // t.Message (string) (string) - if len(t.Message) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Message was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Message)); err != nil { - return err - } - - // t.Proposal (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Proposal); err != nil { - return xerrors.Errorf("failed to write cid field t.Proposal: %w", err) - } - - // t.PublishMessage (cid.Cid) (struct) - - if t.PublishMessage == nil { - if _, err := w.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(w, *t.PublishMessage); err != nil { - return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err) - } - } - - return nil -} - -func (t *Response) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 4 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.State (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.State = uint64(extra) - // t.Message (string) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Message = string(sval) - } - // t.Proposal (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Proposal: %w", err) - } - - t.Proposal = c - - } - // t.PublishMessage (cid.Cid) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err) - } - - t.PublishMessage = &c - } - - } - return nil -} - -func (t *SignedResponse) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Response (deals.Response) (struct) - if err := t.Response.MarshalCBOR(w); err != nil { - return err - } - - // t.Signature (types.Signature) (struct) - if err := t.Signature.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *SignedResponse) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Response (deals.Response) (struct) - - { - - if err := t.Response.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.Signature (types.Signature) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.Signature = new(types.Signature) - if err := t.Signature.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - return nil -} - -func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{136}); err != nil { - return err - } - - // t.Data (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Data); err != nil { - return xerrors.Errorf("failed to write cid field t.Data: %w", err) - } - - // t.PricePerEpoch (types.BigInt) (struct) - if err := t.PricePerEpoch.MarshalCBOR(w); err != nil { - return err - } - - // t.ProposalExpiration (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProposalExpiration))); err != nil { - return err - } - - // t.Duration (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Duration))); err != nil { - return err - } - - // t.ProviderAddress (address.Address) (struct) - if err := t.ProviderAddress.MarshalCBOR(w); err != nil { - return err - } - - // t.Client (address.Address) (struct) - if err := t.Client.MarshalCBOR(w); err != nil { - return err - } - - // t.MinerWorker (address.Address) (struct) - if err := t.MinerWorker.MarshalCBOR(w); err != nil { - return err - } - - // t.MinerID (peer.ID) (string) - if len(t.MinerID) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.MinerID was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.MinerID)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.MinerID)); err != nil { - return err - } - return nil -} - -func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 8 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Data (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Data: %w", err) - } - - t.Data = c - - } - // t.PricePerEpoch (types.BigInt) (struct) - - { - - if err := t.PricePerEpoch.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.ProposalExpiration (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.ProposalExpiration = uint64(extra) - // t.Duration (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Duration = uint64(extra) - // t.ProviderAddress (address.Address) (struct) - - { - - if err := t.ProviderAddress.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.Client (address.Address) (struct) - - { - - if err := t.Client.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.MinerWorker (address.Address) (struct) - - { - - if err := t.MinerWorker.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.MinerID (peer.ID) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.MinerID = peer.ID(sval) - } - return nil -} - -func (t *ClientDeal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{129}); err != nil { - return err - } - - // t.ClientDeal (storagemarket.ClientDeal) (struct) - if err := t.ClientDeal.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 1 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.ClientDeal (storagemarket.ClientDeal) (struct) - - { - - if err := t.ClientDeal.UnmarshalCBOR(br); err != nil { - return err - } - - } - return nil -} - -func (t *MinerDeal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{135}); err != nil { - return err - } - - // t.Client (peer.ID) (string) - if len(t.Client) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Client was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Client)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Client)); err != nil { - return err - } - - // t.Proposal (actors.StorageDealProposal) (struct) - if err := t.Proposal.MarshalCBOR(w); err != nil { - return err - } - - // t.ProposalCid (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.ProposalCid); err != nil { - return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) - } - - // t.State (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { - return err - } - - // t.Ref (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Ref); err != nil { - return xerrors.Errorf("failed to write cid field t.Ref: %w", err) - } - - // t.DealID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { - return err - } - - // t.SectorID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil { - return err - } - return nil -} - -func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 7 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Client (peer.ID) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Client = peer.ID(sval) - } - // t.Proposal (actors.StorageDealProposal) (struct) - - { - - if err := t.Proposal.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.ProposalCid (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err) - } - - t.ProposalCid = c - - } - // t.State (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.State = uint64(extra) - // t.Ref (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Ref: %w", err) - } - - t.Ref = c - - } - // t.DealID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DealID = uint64(extra) - // t.SectorID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.SectorID = uint64(extra) - return nil -} - -func (t *StorageDataTransferVoucher) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Proposal (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Proposal); err != nil { - return xerrors.Errorf("failed to write cid field t.Proposal: %w", err) - } - - // t.DealID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { - return err - } - return nil -} - -func (t *StorageDataTransferVoucher) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Proposal (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Proposal: %w", err) - } - - t.Proposal = c - - } - // t.DealID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DealID = uint64(extra) - return nil -} diff --git a/chain/deals/client.go b/chain/deals/client.go deleted file mode 100644 index e1067961c..000000000 --- a/chain/deals/client.go +++ /dev/null @@ -1,293 +0,0 @@ -package deals - -import ( - "context" - - "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log/v2" - "github.com/libp2p/go-libp2p-core/host" - inet "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-fil-markets/retrievalmarket" - "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" - "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storagemarket" -) - -var log = logging.Logger("deals") - -type ClientDeal struct { - storagemarket.ClientDeal - - s inet.Stream -} - -type Client struct { - h host.Host - - // dataTransfer - // TODO: once the data transfer module is complete, the - // client will listen to events on the data transfer module - // Because we are using only a fake DAGService - // implementation, there's no validation or events on the client side - dataTransfer dtypes.ClientDataTransfer - dag dtypes.ClientDAG - discovery *discovery.Local - - node storagemarket.StorageClientNode - - deals *statestore.StateStore - conns map[cid.Cid]inet.Stream - - incoming chan *ClientDeal - updated chan clientDealUpdate - - stop chan struct{} - stopped chan struct{} -} - -type clientDealUpdate struct { - newState api.DealState - id cid.Cid - err error - mut func(*ClientDeal) -} - -func NewClient(h host.Host, dag dtypes.ClientDAG, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, deals dtypes.ClientDealStore, scn storagemarket.StorageClientNode) *Client { - c := &Client{ - h: h, - dataTransfer: dataTransfer, - dag: dag, - discovery: discovery, - node: scn, - - deals: deals, - conns: map[cid.Cid]inet.Stream{}, - - incoming: make(chan *ClientDeal, 16), - updated: make(chan clientDealUpdate, 16), - - stop: make(chan struct{}), - stopped: make(chan struct{}), - } - - return c -} - -func (c *Client) Run(ctx context.Context) { - go func() { - defer close(c.stopped) - - for { - select { - case deal := <-c.incoming: - c.onIncoming(deal) - case update := <-c.updated: - c.onUpdated(ctx, update) - case <-c.stop: - return - } - } - }() -} - -func (c *Client) onIncoming(deal *ClientDeal) { - log.Info("incoming deal") - - if _, ok := c.conns[deal.ProposalCid]; ok { - log.Errorf("tracking deal connection: already tracking connection for deal %s", deal.ProposalCid) - return - } - c.conns[deal.ProposalCid] = deal.s - - if err := c.deals.Begin(deal.ProposalCid, deal); err != nil { - // We may have re-sent the proposal - log.Errorf("deal tracking failed: %s", err) - c.failDeal(deal.ProposalCid, err) - return - } - - go func() { - c.updated <- clientDealUpdate{ - newState: api.DealUnknown, - id: deal.ProposalCid, - err: nil, - } - }() -} - -func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) { - log.Infof("Client deal %s updated state to %s", update.id, api.DealStates[update.newState]) - var deal ClientDeal - err := c.deals.Mutate(update.id, func(d *ClientDeal) error { - d.State = update.newState - if update.mut != nil { - update.mut(d) - } - deal = *d - return nil - }) - if update.err != nil { - log.Errorf("deal %s failed: %s", update.id, update.err) - c.failDeal(update.id, update.err) - return - } - if err != nil { - c.failDeal(update.id, err) - return - } - - switch update.newState { - case api.DealUnknown: // new - c.handle(ctx, deal, c.new, api.DealAccepted) - case api.DealAccepted: - c.handle(ctx, deal, c.accepted, api.DealStaged) - case api.DealStaged: - c.handle(ctx, deal, c.staged, api.DealSealing) - case api.DealSealing: - c.handle(ctx, deal, c.sealing, api.DealNoUpdate) - // TODO: DealComplete -> watch for faults, expiration, etc. - } -} - -type ClientDealProposal struct { - Data cid.Cid - - PricePerEpoch types.BigInt - ProposalExpiration uint64 - Duration uint64 - - ProviderAddress address.Address - Client address.Address - MinerWorker address.Address - MinerID peer.ID -} - -func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, error) { - amount := types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration)) - if err := c.node.EnsureFunds(ctx, p.Client, storagemarket.TokenAmount(amount)); err != nil { - return cid.Undef, xerrors.Errorf("adding market funds failed: %w", err) - } - - commP, pieceSize, err := c.commP(ctx, p.Data) - if err != nil { - return cid.Undef, xerrors.Errorf("computing commP failed: %w", err) - } - - dealProposal := &actors.StorageDealProposal{ - PieceRef: commP, - PieceSize: uint64(pieceSize), - Client: p.Client, - Provider: p.ProviderAddress, - ProposalExpiration: p.ProposalExpiration, - Duration: p.Duration, - StoragePricePerEpoch: p.PricePerEpoch, - StorageCollateral: types.NewInt(uint64(pieceSize)), // TODO: real calc - } - - if err := c.node.SignProposal(ctx, p.Client, dealProposal); err != nil { - return cid.Undef, xerrors.Errorf("signing deal proposal failed: %w", err) - } - - proposalNd, err := cborutil.AsIpld(dealProposal) - if err != nil { - return cid.Undef, xerrors.Errorf("getting proposal node failed: %w", err) - } - - s, err := c.h.NewStream(ctx, p.MinerID, storagemarket.DealProtocolID) - if err != nil { - return cid.Undef, xerrors.Errorf("connecting to storage provider failed: %w", err) - } - - proposal := &Proposal{ - DealProposal: dealProposal, - Piece: p.Data, - } - - if err := cborutil.WriteCborRPC(s, proposal); err != nil { - s.Reset() - return cid.Undef, xerrors.Errorf("sending proposal to storage provider failed: %w", err) - } - - deal := &ClientDeal{ - ClientDeal: storagemarket.ClientDeal{ - ProposalCid: proposalNd.Cid(), - Proposal: *dealProposal, - State: api.DealUnknown, - Miner: p.MinerID, - MinerWorker: p.MinerWorker, - PayloadCid: p.Data, - }, - - s: s, - } - - c.incoming <- deal - - return deal.ProposalCid, c.discovery.AddPeer(p.Data, retrievalmarket.RetrievalPeer{ - Address: dealProposal.Provider, - ID: deal.Miner, - }) -} - -func (c *Client) QueryAsk(ctx context.Context, p peer.ID, a address.Address) (*types.SignedStorageAsk, error) { - s, err := c.h.NewStream(ctx, p, storagemarket.AskProtocolID) - if err != nil { - return nil, xerrors.Errorf("failed to open stream to miner: %w", err) - } - - req := &AskRequest{ - Miner: a, - } - if err := cborutil.WriteCborRPC(s, req); err != nil { - return nil, xerrors.Errorf("failed to send ask request: %w", err) - } - - var out AskResponse - if err := cborutil.ReadCborRPC(s, &out); err != nil { - return nil, xerrors.Errorf("failed to read ask response: %w", err) - } - - if out.Ask == nil { - return nil, xerrors.Errorf("got no ask back") - } - - if out.Ask.Ask.Miner != a { - return nil, xerrors.Errorf("got back ask for wrong miner") - } - - if err := c.checkAskSignature(out.Ask); err != nil { - return nil, xerrors.Errorf("ask was not properly signed") - } - - return out.Ask, nil -} - -func (c *Client) List() ([]ClientDeal, error) { - var out []ClientDeal - if err := c.deals.List(&out); err != nil { - return nil, err - } - return out, nil -} - -func (c *Client) GetDeal(d cid.Cid) (*ClientDeal, error) { - var out ClientDeal - if err := c.deals.Get(d, &out); err != nil { - return nil, err - } - return &out, nil -} - -func (c *Client) Stop() { - close(c.stop) - <-c.stopped -} diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go deleted file mode 100644 index e82beb126..000000000 --- a/chain/deals/client_states.go +++ /dev/null @@ -1,97 +0,0 @@ -package deals - -import ( - "context" - - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/types" -) - -type clientHandlerFunc func(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) - -func (c *Client) handle(ctx context.Context, deal ClientDeal, cb clientHandlerFunc, next api.DealState) { - go func() { - mut, err := cb(ctx, deal) - if err != nil { - next = api.DealError - } - - if err == nil && next == api.DealNoUpdate { - return - } - - select { - case c.updated <- clientDealUpdate{ - newState: next, - id: deal.ProposalCid, - err: err, - mut: mut, - }: - case <-c.stop: - } - }() -} - -func (c *Client) new(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { - resp, err := c.readStorageDealResp(deal) - if err != nil { - return nil, err - } - - // TODO: verify StorageDealSubmission - - if err := c.disconnect(deal); err != nil { - return nil, err - } - - /* data transfer happens */ - if resp.State != api.DealAccepted { - return nil, xerrors.Errorf("deal wasn't accepted (State=%d)", resp.State) - } - - return func(info *ClientDeal) { - info.PublishMessage = resp.PublishMessage - }, nil -} - -func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { - log.Infow("DEAL ACCEPTED!") - - dealId, err := c.node.ValidatePublishedDeal(ctx, deal.ClientDeal) - if err != nil { - return nil, err - } - - return func(info *ClientDeal) { - info.DealID = dealId - }, nil -} - -func (c *Client) staged(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { - // TODO: Maybe wait for pre-commit - - return nil, nil -} - -func (c *Client) sealing(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { - cb := func(err error) { - select { - case c.updated <- clientDealUpdate{ - newState: api.DealComplete, - id: deal.ProposalCid, - err: err, - }: - case <-c.stop: - } - } - - err := c.node.OnDealSectorCommitted(ctx, deal.Proposal.Provider, deal.DealID, cb) - - return nil, err -} - -func (c *Client) checkAskSignature(ask *types.SignedStorageAsk) error { - return c.node.ValidateAskSignature(ask) -} diff --git a/chain/deals/client_storagemarket.go b/chain/deals/client_storagemarket.go deleted file mode 100644 index 08dacc7a3..000000000 --- a/chain/deals/client_storagemarket.go +++ /dev/null @@ -1,118 +0,0 @@ -package deals - -// this file implements storagemarket.StorageClient - -import ( - "context" - - "github.com/ipfs/go-cid" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storagemarket" -) - -func (c *Client) ListProviders(ctx context.Context) (<-chan storagemarket.StorageProviderInfo, error) { - providers, err := c.node.ListStorageProviders(ctx) - if err != nil { - return nil, err - } - - out := make(chan storagemarket.StorageProviderInfo) - - go func() { - for _, p := range providers { - select { - case out <- *p: - case <-ctx.Done(): - return - } - - } - }() - - return out, nil -} - -func (c *Client) ListDeals(ctx context.Context, addr address.Address) ([]actors.OnChainDeal, error) { - return c.node.ListClientDeals(ctx, addr) -} - -func (c *Client) ListInProgressDeals(ctx context.Context) ([]storagemarket.ClientDeal, error) { - deals, err := c.List() - if err != nil { - return nil, err - } - - out := make([]storagemarket.ClientDeal, len(deals)) - for k, v := range deals { - out[k] = storagemarket.ClientDeal{ - ProposalCid: v.ProposalCid, - Proposal: v.Proposal, - State: v.State, - Miner: v.Miner, - MinerWorker: v.MinerWorker, - DealID: v.DealID, - PublishMessage: v.PublishMessage, - } - } - - return out, nil -} - -func (c *Client) GetInProgressDeal(ctx context.Context, cid cid.Cid) (storagemarket.ClientDeal, error) { - deals, err := c.ListInProgressDeals(ctx) - if err != nil { - return storagemarket.ClientDeal{}, err - } - - for _, deal := range deals { - if deal.ProposalCid == cid { - return deal, nil - } - } - - return storagemarket.ClientDeal{}, xerrors.Errorf("couldn't find client deal") -} - -func (c *Client) GetAsk(ctx context.Context, info storagemarket.StorageProviderInfo) (*storagemarket.StorageAsk, error) { - return c.QueryAsk(ctx, info.PeerID, info.Address) -} - -func (c *Client) ProposeStorageDeal(ctx context.Context, addr address.Address, info *storagemarket.StorageProviderInfo, payloadCid cid.Cid, proposalExpiration storagemarket.Epoch, duration storagemarket.Epoch, price storagemarket.TokenAmount, collateral storagemarket.TokenAmount) (*storagemarket.ProposeStorageDealResult, error) { - - proposal := ClientDealProposal{ - Data: payloadCid, - PricePerEpoch: types.BigInt(price), - ProposalExpiration: uint64(proposalExpiration), - Duration: uint64(duration), - Client: addr, - ProviderAddress: info.Address, - MinerWorker: info.Worker, - MinerID: info.PeerID, - } - - proposalCid, err := c.Start(ctx, proposal) - - result := &storagemarket.ProposeStorageDealResult{ - ProposalCid: proposalCid, - } - - return result, err -} - -func (c *Client) GetPaymentEscrow(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { - - balance, err := c.node.GetBalance(ctx, addr) - - return balance, err -} - -func (c *Client) AddPaymentEscrow(ctx context.Context, addr address.Address, amount storagemarket.TokenAmount) error { - - return c.node.AddFunds(ctx, addr, amount) -} - -var _ storagemarket.StorageClient = &Client{} diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go deleted file mode 100644 index f2d150a23..000000000 --- a/chain/deals/client_utils.go +++ /dev/null @@ -1,170 +0,0 @@ -package deals - -import ( - "context" - "runtime" - - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" - "github.com/ipfs/go-cid" - files "github.com/ipfs/go-ipfs-files" - unixfile "github.com/ipfs/go-unixfs/file" - "github.com/ipld/go-ipld-prime" - "github.com/libp2p/go-libp2p-core/peer" - "golang.org/x/xerrors" - - cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-statestore" - - "github.com/filecoin-project/lotus/lib/padreader" - "github.com/filecoin-project/lotus/node/modules/dtypes" -) - -func (c *Client) failDeal(id cid.Cid, cerr error) { - if cerr == nil { - _, f, l, _ := runtime.Caller(1) - cerr = xerrors.Errorf("unknown error (fail called at %s:%d)", f, l) - } - - s, ok := c.conns[id] - if ok { - _ = s.Reset() - delete(c.conns, id) - } - - // TODO: store in some sort of audit log - log.Errorf("deal %s failed: %+v", id, cerr) -} - -func (c *Client) commP(ctx context.Context, data cid.Cid) ([]byte, uint64, error) { - root, err := c.dag.Get(ctx, data) - if err != nil { - log.Errorf("failed to get file root for deal: %s", err) - return nil, 0, err - } - - n, err := unixfile.NewUnixfsFile(ctx, c.dag, root) - if err != nil { - log.Errorf("cannot open unixfs file: %s", err) - return nil, 0, err - } - - uf, ok := n.(files.File) - if !ok { - // TODO: we probably got directory, how should we handle this in unixfs mode? - return nil, 0, xerrors.New("unsupported unixfs type") - } - - s, err := uf.Size() - if err != nil { - return nil, 0, err - } - - pr, psize := padreader.New(uf, uint64(s)) - - commp, err := sectorbuilder.GeneratePieceCommitment(pr, psize) - if err != nil { - return nil, 0, xerrors.Errorf("generating CommP: %w", err) - } - - return commp[:], psize, nil -} - -func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) { - s, ok := c.conns[deal.ProposalCid] - if !ok { - // TODO: Try to re-establish the connection using query protocol - return nil, xerrors.Errorf("no connection to miner") - } - - var resp SignedResponse - if err := cborutil.ReadCborRPC(s, &resp); err != nil { - log.Errorw("failed to read Response message", "error", err) - return nil, err - } - - if err := resp.Verify(deal.MinerWorker); err != nil { - return nil, xerrors.Errorf("verifying response signature failed", err) - } - - if resp.Response.Proposal != deal.ProposalCid { - return nil, xerrors.Errorf("miner responded to a wrong proposal: %s != %s", resp.Response.Proposal, deal.ProposalCid) - } - - return &resp.Response, nil -} - -func (c *Client) disconnect(deal ClientDeal) error { - s, ok := c.conns[deal.ProposalCid] - if !ok { - return nil - } - - err := s.Close() - delete(c.conns, deal.ProposalCid) - return err -} - -var _ datatransfer.RequestValidator = &ClientRequestValidator{} - -// ClientRequestValidator validates data transfer requests for the client -// in a storage market -type ClientRequestValidator struct { - deals *statestore.StateStore -} - -// NewClientRequestValidator returns a new client request validator for the -// given datastore -func NewClientRequestValidator(deals dtypes.ClientDealStore) *ClientRequestValidator { - crv := &ClientRequestValidator{ - deals: deals, - } - return crv -} - -// ValidatePush validates a push request received from the peer that will send data -// Will always error because clients should not accept push requests from a provider -// in a storage deal (i.e. send data to client). -func (c *ClientRequestValidator) ValidatePush( - sender peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - Selector ipld.Node) error { - return ErrNoPushAccepted -} - -// ValidatePull validates a pull request received from the peer that will receive data -// Will succeed only if: -// - voucher has correct type -// - voucher references an active deal -// - referenced deal matches the receiver (miner) -// - referenced deal matches the given base CID -// - referenced deal is in an acceptable state -func (c *ClientRequestValidator) ValidatePull( - receiver peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - Selector ipld.Node) error { - dealVoucher, ok := voucher.(*StorageDataTransferVoucher) - if !ok { - return xerrors.Errorf("voucher type %s: %w", voucher.Type(), ErrWrongVoucherType) - } - - var deal ClientDeal - err := c.deals.Get(dealVoucher.Proposal, &deal) - if err != nil { - return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal) - } - if deal.Miner != receiver { - return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Miner.String(), receiver.String(), ErrWrongPeer) - } - if !deal.PayloadCid.Equals(baseCid) { - return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece) - } - for _, state := range DataTransferStates { - if deal.State == state { - return nil - } - } - return xerrors.Errorf("Deal State %s: %w", deal.State, ErrInacceptableDealState) -} diff --git a/chain/deals/provider.go b/chain/deals/provider.go deleted file mode 100644 index 1a020de71..000000000 --- a/chain/deals/provider.go +++ /dev/null @@ -1,293 +0,0 @@ -package deals - -import ( - "context" - "errors" - "sync" - - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - "github.com/libp2p/go-libp2p-core/host" - inet "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storagemarket" -) - -var ProviderDsPrefix = "/deals/provider" - -type MinerDeal struct { - Client peer.ID - Proposal actors.StorageDealProposal - ProposalCid cid.Cid - State api.DealState - - Ref cid.Cid - - DealID uint64 - SectorID uint64 // Set when State >= DealStaged - - s inet.Stream -} - -type Provider struct { - pricePerByteBlock types.BigInt // how much we want for storing one byte for one block - minPieceSize uint64 - - ask *types.SignedStorageAsk - askLk sync.Mutex - - spn storagemarket.StorageProviderNode - - // TODO: This will go away once storage market module + CAR - // is implemented - dag dtypes.StagingDAG - - // dataTransfer is the manager of data transfers used by this storage provider - dataTransfer dtypes.ProviderDataTransfer - - deals *statestore.StateStore - ds dtypes.MetadataDS - - conns map[cid.Cid]inet.Stream - - actor address.Address - - incoming chan MinerDeal - updated chan minerDealUpdate - stop chan struct{} - stopped chan struct{} -} - -type minerDealUpdate struct { - newState api.DealState - id cid.Cid - err error - mut func(*MinerDeal) -} - -var ( - // ErrDataTransferFailed means a data transfer for a deal failed - ErrDataTransferFailed = errors.New("deal data transfer failed") -) - -func NewProvider(ds dtypes.MetadataDS, dag dtypes.StagingDAG, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode) (storagemarket.StorageProvider, error) { - addr, err := ds.Get(datastore.NewKey("miner-address")) - if err != nil { - return nil, err - } - minerAddress, err := address.NewFromBytes(addr) - if err != nil { - return nil, err - } - - h := &Provider{ - dag: dag, - dataTransfer: dataTransfer, - spn: spn, - - pricePerByteBlock: types.NewInt(3), // TODO: allow setting - minPieceSize: 256, // TODO: allow setting (BUT KEEP MIN 256! (because of how we fill sectors up)) - - conns: map[cid.Cid]inet.Stream{}, - - incoming: make(chan MinerDeal), - updated: make(chan minerDealUpdate), - stop: make(chan struct{}), - stopped: make(chan struct{}), - - actor: minerAddress, - - deals: statestore.New(namespace.Wrap(ds, datastore.NewKey(ProviderDsPrefix))), - ds: ds, - } - - if err := h.tryLoadAsk(); err != nil { - return nil, err - } - - if h.ask == nil { - // TODO: we should be fine with this state, and just say it means 'not actively accepting deals' - // for now... lets just set a price - if err := h.SetPrice(types.NewInt(500_000_000), 1000000); err != nil { - return nil, xerrors.Errorf("failed setting a default price: %w", err) - } - } - - // register a data transfer event handler -- this will move deals from - // accepted to staged - h.dataTransfer.SubscribeToEvents(h.onDataTransferEvent) - - return h, nil -} - -func (p *Provider) Run(ctx context.Context, host host.Host) { - // TODO: restore state - - host.SetStreamHandler(storagemarket.DealProtocolID, p.HandleStream) - host.SetStreamHandler(storagemarket.AskProtocolID, p.HandleAskStream) - - go func() { - defer log.Warn("quitting deal provider loop") - defer close(p.stopped) - - for { - select { - case deal := <-p.incoming: // DealAccepted - p.onIncoming(deal) - case update := <-p.updated: // DealStaged - p.onUpdated(ctx, update) - case <-p.stop: - return - } - } - }() -} - -func (p *Provider) onIncoming(deal MinerDeal) { - log.Info("incoming deal") - - p.conns[deal.ProposalCid] = deal.s - - if err := p.deals.Begin(deal.ProposalCid, &deal); err != nil { - // This can happen when client re-sends proposal - p.failDeal(context.TODO(), deal.ProposalCid, err) - log.Errorf("deal tracking failed: %s", err) - return - } - - go func() { - p.updated <- minerDealUpdate{ - newState: api.DealAccepted, - id: deal.ProposalCid, - err: nil, - } - }() -} - -func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { - log.Infof("Deal %s updated state to %s", update.id, api.DealStates[update.newState]) - if update.err != nil { - log.Errorf("deal %s (newSt: %d) failed: %+v", update.id, update.newState, update.err) - p.failDeal(ctx, update.id, update.err) - return - } - var deal MinerDeal - err := p.deals.Mutate(update.id, func(d *MinerDeal) error { - d.State = update.newState - if update.mut != nil { - update.mut(d) - } - deal = *d - return nil - }) - if err != nil { - p.failDeal(ctx, update.id, err) - return - } - - switch update.newState { - case api.DealAccepted: - p.handle(ctx, deal, p.accept, api.DealNoUpdate) - case api.DealStaged: - p.handle(ctx, deal, p.staged, api.DealSealing) - case api.DealSealing: - p.handle(ctx, deal, p.sealing, api.DealComplete) - case api.DealComplete: - p.handle(ctx, deal, p.complete, api.DealNoUpdate) - } -} - -// onDataTransferEvent is the function called when an event occurs in a data -// transfer -- it reads the voucher to verify this even occurred in a storage -// market deal, then, based on the data transfer event that occurred, it generates -// and update message for the deal -- either moving to staged for a completion -// event or moving to error if a data transfer error occurs -func (p *Provider) onDataTransferEvent(event datatransfer.Event, channelState datatransfer.ChannelState) { - voucher, ok := channelState.Voucher().(*StorageDataTransferVoucher) - // if this event is for a transfer not related to storage, ignore - if !ok { - return - } - - // data transfer events for opening and progress do not affect deal state - var next api.DealState - var err error - var mut func(*MinerDeal) - switch event.Code { - case datatransfer.Complete: - next = api.DealStaged - mut = func(deal *MinerDeal) { - deal.DealID = voucher.DealID - } - case datatransfer.Error: - next = api.DealFailed - err = ErrDataTransferFailed - default: - // the only events we care about are complete and error - return - } - - select { - case p.updated <- minerDealUpdate{ - newState: next, - id: voucher.Proposal, - err: err, - mut: mut, - }: - case <-p.stop: - } -} - -func (p *Provider) newDeal(s inet.Stream, proposal Proposal) (MinerDeal, error) { - proposalNd, err := cborutil.AsIpld(proposal.DealProposal) - if err != nil { - return MinerDeal{}, err - } - - return MinerDeal{ - Client: s.Conn().RemotePeer(), - Proposal: *proposal.DealProposal, - ProposalCid: proposalNd.Cid(), - State: api.DealUnknown, - - Ref: proposal.Piece, - - s: s, - }, nil -} - -func (p *Provider) HandleStream(s inet.Stream) { - log.Info("Handling storage deal proposal!") - - proposal, err := p.readProposal(s) - if err != nil { - log.Error(err) - s.Close() - return - } - - deal, err := p.newDeal(s, proposal) - if err != nil { - log.Errorf("%+v", err) - s.Close() - return - } - - p.incoming <- deal -} - -func (p *Provider) Stop() { - close(p.stop) - <-p.stopped -} diff --git a/chain/deals/provider_asks.go b/chain/deals/provider_asks.go deleted file mode 100644 index a0cdd9c71..000000000 --- a/chain/deals/provider_asks.go +++ /dev/null @@ -1,143 +0,0 @@ -package deals - -import ( - "bytes" - "context" - "time" - - "github.com/ipfs/go-datastore" - inet "github.com/libp2p/go-libp2p-core/network" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/lotus/chain/types" -) - -func (p *Provider) SetPrice(price types.BigInt, ttlsecs int64) error { - p.askLk.Lock() - defer p.askLk.Unlock() - - var seqno uint64 - if p.ask != nil { - seqno = p.ask.Ask.SeqNo + 1 - } - - now := time.Now().Unix() - ask := &types.StorageAsk{ - Price: price, - Timestamp: uint64(now), - Expiry: uint64(now + ttlsecs), - Miner: p.actor, - SeqNo: seqno, - MinPieceSize: p.minPieceSize, - } - - ssa, err := p.signAsk(ask) - if err != nil { - return err - } - - return p.saveAsk(ssa) -} - -func (p *Provider) GetAsk(m address.Address) *types.SignedStorageAsk { - p.askLk.Lock() - defer p.askLk.Unlock() - if m != p.actor { - return nil - } - - return p.ask -} - -func (p *Provider) HandleAskStream(s inet.Stream) { - defer s.Close() - var ar AskRequest - if err := cborutil.ReadCborRPC(s, &ar); err != nil { - log.Errorf("failed to read AskRequest from incoming stream: %s", err) - return - } - - resp := p.processAskRequest(&ar) - - if err := cborutil.WriteCborRPC(s, resp); err != nil { - log.Errorf("failed to write ask response: %s", err) - return - } -} - -func (p *Provider) processAskRequest(ar *AskRequest) *AskResponse { - return &AskResponse{ - Ask: p.GetAsk(ar.Miner), - } -} - -var bestAskKey = datastore.NewKey("latest-ask") - -func (p *Provider) tryLoadAsk() error { - p.askLk.Lock() - defer p.askLk.Unlock() - - err := p.loadAsk() - if err != nil { - if xerrors.Is(err, datastore.ErrNotFound) { - log.Warn("no previous ask found, miner will not accept deals until a price is set") - return nil - } - return err - } - - return nil -} - -func (p *Provider) loadAsk() error { - askb, err := p.ds.Get(datastore.NewKey("latest-ask")) - if err != nil { - return xerrors.Errorf("failed to load most recent ask from disk: %w", err) - } - - var ssa types.SignedStorageAsk - if err := cborutil.ReadCborRPC(bytes.NewReader(askb), &ssa); err != nil { - return err - } - - p.ask = &ssa - return nil -} - -func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) { - b, err := cborutil.Dump(a) - if err != nil { - return nil, err - } - - worker, err := p.spn.GetMinerWorker(context.TODO(), p.actor) - if err != nil { - return nil, xerrors.Errorf("failed to get worker to sign ask: %w", err) - } - - sig, err := p.spn.SignBytes(context.TODO(), worker, b) - if err != nil { - return nil, err - } - - return &types.SignedStorageAsk{ - Ask: a, - Signature: sig, - }, nil -} - -func (p *Provider) saveAsk(a *types.SignedStorageAsk) error { - b, err := cborutil.Dump(a) - if err != nil { - return err - } - - if err := p.ds.Put(bestAskKey, b); err != nil { - return err - } - - p.ask = a - return nil -} diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go deleted file mode 100644 index 76fa9a99b..000000000 --- a/chain/deals/provider_states.go +++ /dev/null @@ -1,172 +0,0 @@ -package deals - -import ( - "context" - - ipldfree "github.com/ipld/go-ipld-prime/impl/free" - "github.com/ipld/go-ipld-prime/traversal/selector" - "github.com/ipld/go-ipld-prime/traversal/selector/builder" - - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storagemarket" -) - -type providerHandlerFunc func(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) - -func (p *Provider) handle(ctx context.Context, deal MinerDeal, cb providerHandlerFunc, next api.DealState) { - go func() { - mut, err := cb(ctx, deal) - - if err == nil && next == api.DealNoUpdate { - return - } - - select { - case p.updated <- minerDealUpdate{ - newState: next, - id: deal.ProposalCid, - err: err, - mut: mut, - }: - case <-p.stop: - } - }() -} - -// ACCEPTED -func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - - head, err := p.spn.MostRecentStateId(ctx) - if err != nil { - return nil, err - } - if head.Height() >= deal.Proposal.ProposalExpiration { - return nil, xerrors.Errorf("deal proposal already expired") - } - - // TODO: check StorageCollateral - - minPrice := types.BigDiv(types.BigMul(p.ask.Ask.Price, types.NewInt(deal.Proposal.PieceSize)), types.NewInt(1<<30)) - if deal.Proposal.StoragePricePerEpoch.LessThan(minPrice) { - return nil, xerrors.Errorf("storage price per epoch less than asking price: %s < %s", deal.Proposal.StoragePricePerEpoch, minPrice) - } - - if deal.Proposal.PieceSize < p.ask.Ask.MinPieceSize { - return nil, xerrors.Errorf("piece size less than minimum required size: %d < %d", deal.Proposal.PieceSize, p.ask.Ask.MinPieceSize) - } - - // check market funds - clientMarketBalance, err := p.spn.GetBalance(ctx, deal.Proposal.Client) - if err != nil { - return nil, xerrors.Errorf("getting client market balance failed: %w", err) - } - - // This doesn't guarantee that the client won't withdraw / lock those funds - // but it's a decent first filter - if clientMarketBalance.Available.LessThan(deal.Proposal.TotalStoragePrice()) { - return nil, xerrors.New("clientMarketBalance.Available too small") - } - - waddr, err := p.spn.GetMinerWorker(ctx, deal.Proposal.Provider) - if err != nil { - return nil, err - } - - // TODO: check StorageCollateral (may be too large (or too small)) - if err := p.spn.EnsureFunds(ctx, waddr, storagemarket.TokenAmount(deal.Proposal.StorageCollateral)); err != nil { - return nil, err - } - - smDeal := storagemarket.MinerDeal{ - Client: deal.Client, - Proposal: deal.Proposal, - ProposalCid: deal.ProposalCid, - State: deal.State, - Ref: deal.Ref, - SectorID: deal.SectorID, - } - - dealId, mcid, err := p.spn.PublishDeals(ctx, smDeal) - if err != nil { - return nil, err - } - - log.Infof("fetching data for a deal %d", dealId) - err = p.sendSignedResponse(ctx, &Response{ - State: api.DealAccepted, - - Proposal: deal.ProposalCid, - PublishMessage: &mcid, - }) - if err != nil { - return nil, err - } - - if err := p.disconnect(deal); err != nil { - log.Warnf("closing client connection: %+v", err) - } - - ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder()) - - // this is the selector for "get the whole DAG" - // TODO: support storage deals with custom payload selectors - allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(), - ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() - - // initiate a pull data transfer. This will complete asynchronously and the - // completion of the data transfer will trigger a change in deal state - // (see onDataTransferEvent) - _, err = p.dataTransfer.OpenPullDataChannel(ctx, - deal.Client, - &StorageDataTransferVoucher{Proposal: deal.ProposalCid, DealID: uint64(dealId)}, - deal.Ref, - allSelector, - ) - if err != nil { - return nil, xerrors.Errorf("failed to open pull data channel: %w", err) - } - - return nil, nil -} - -// STAGED - -func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - sectorID, err := p.spn.OnDealComplete( - ctx, - storagemarket.MinerDeal{ - Client: deal.Client, - Proposal: deal.Proposal, - ProposalCid: deal.ProposalCid, - State: deal.State, - Ref: deal.Ref, - DealID: deal.DealID, - }, - "", - ) - - if err != nil { - return nil, err - } - - return func(deal *MinerDeal) { - deal.SectorID = sectorID - }, nil -} - -// SEALING - -func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - // TODO: consider waiting for seal to happen - - return nil, nil -} - -func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - // TODO: observe sector lifecycle, status, expiration.. - - return nil, nil -} diff --git a/chain/deals/provider_storagemarket.go b/chain/deals/provider_storagemarket.go deleted file mode 100644 index 4d9863bd9..000000000 --- a/chain/deals/provider_storagemarket.go +++ /dev/null @@ -1,65 +0,0 @@ -package deals - -// this file implements storagemarket.StorageClient - -import ( - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storagemarket" -) - -func (p *Provider) AddAsk(price storagemarket.TokenAmount, ttlsecs int64) error { - return p.SetPrice(types.BigInt(price), ttlsecs) -} - -func (p *Provider) ListAsks(addr address.Address) []*types.SignedStorageAsk { - ask := p.GetAsk(addr) - - if ask != nil { - return []*types.SignedStorageAsk{ask} - } - - return nil -} - -func (p *Provider) ListDeals(ctx context.Context) ([]actors.OnChainDeal, error) { - return p.spn.ListProviderDeals(ctx, p.actor) -} - -func (p *Provider) AddStorageCollateral(ctx context.Context, amount storagemarket.TokenAmount) error { - return p.spn.AddFunds(ctx, p.actor, amount) -} - -func (p *Provider) GetStorageCollateral(ctx context.Context) (storagemarket.Balance, error) { - balance, err := p.spn.GetBalance(ctx, p.actor) - - return balance, err -} - -func (p *Provider) ListIncompleteDeals() ([]storagemarket.MinerDeal, error) { - var out []storagemarket.MinerDeal - - var deals []MinerDeal - if err := p.deals.List(&deals); err != nil { - return nil, err - } - - for _, deal := range deals { - out = append(out, storagemarket.MinerDeal{ - Client: deal.Client, - Proposal: deal.Proposal, - ProposalCid: deal.ProposalCid, - State: deal.State, - Ref: deal.Ref, - DealID: deal.DealID, - SectorID: deal.SectorID, - }) - } - - return out, nil -} - -var _ storagemarket.StorageProvider = &Provider{} diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go deleted file mode 100644 index 54b838a43..000000000 --- a/chain/deals/provider_utils.go +++ /dev/null @@ -1,180 +0,0 @@ -package deals - -import ( - "context" - "runtime" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-statestore" - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" - inet "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/node/modules/dtypes" -) - -func (p *Provider) failDeal(ctx context.Context, id cid.Cid, cerr error) { - if err := p.deals.End(id); err != nil { - log.Warnf("deals.End: %s", err) - } - - if cerr == nil { - _, f, l, _ := runtime.Caller(1) - cerr = xerrors.Errorf("unknown error (fail called at %s:%d)", f, l) - } - - log.Warnf("deal %s failed: %s", id, cerr) - - err := p.sendSignedResponse(ctx, &Response{ - State: api.DealFailed, - Message: cerr.Error(), - Proposal: id, - }) - - s, ok := p.conns[id] - if ok { - _ = s.Reset() - delete(p.conns, id) - } - - if err != nil { - log.Warnf("notifying client about deal failure: %s", err) - } -} - -func (p *Provider) readProposal(s inet.Stream) (proposal Proposal, err error) { - if err := cborutil.ReadCborRPC(s, &proposal); err != nil { - log.Errorw("failed to read proposal message", "error", err) - return proposal, err - } - - if proposal.DealProposal.ProposerSignature == nil { - return proposal, xerrors.Errorf("incoming deal proposal has no signature") - } - - if err := proposal.DealProposal.Verify(address.Undef); err != nil { - return proposal, xerrors.Errorf("verifying StorageDealProposal: %w", err) - } - - if proposal.DealProposal.Provider != p.actor { - log.Errorf("proposal with wrong ProviderAddress: %s", proposal.DealProposal.Provider) - return proposal, err - } - - return -} - -func (p *Provider) sendSignedResponse(ctx context.Context, resp *Response) error { - s, ok := p.conns[resp.Proposal] - if !ok { - return xerrors.New("couldn't send response: not connected") - } - - msg, err := cborutil.Dump(resp) - if err != nil { - return xerrors.Errorf("serializing response: %w", err) - } - - worker, err := p.spn.GetMinerWorker(ctx, p.actor) - if err != nil { - return err - } - - sig, err := p.spn.SignBytes(ctx, worker, msg) - if err != nil { - return xerrors.Errorf("failed to sign response message: %w", err) - } - - signedResponse := &SignedResponse{ - Response: *resp, - Signature: sig, - } - - err = cborutil.WriteCborRPC(s, signedResponse) - if err != nil { - // Assume client disconnected - s.Close() - delete(p.conns, resp.Proposal) - } - return err -} - -func (p *Provider) disconnect(deal MinerDeal) error { - s, ok := p.conns[deal.ProposalCid] - if !ok { - return nil - } - - err := s.Close() - delete(p.conns, deal.ProposalCid) - return err -} - -var _ datatransfer.RequestValidator = &ProviderRequestValidator{} - -// ProviderRequestValidator validates data transfer requests for the provider -// in a storage market -type ProviderRequestValidator struct { - deals *statestore.StateStore -} - -// NewProviderRequestValidator returns a new client request validator for the -// given datastore -func NewProviderRequestValidator(deals dtypes.ProviderDealStore) *ProviderRequestValidator { - return &ProviderRequestValidator{ - deals: deals, - } -} - -// ValidatePush validates a push request received from the peer that will send data -// Will succeed only if: -// - voucher has correct type -// - voucher references an active deal -// - referenced deal matches the client -// - referenced deal matches the given base CID -// - referenced deal is in an acceptable state -func (m *ProviderRequestValidator) ValidatePush( - sender peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - Selector ipld.Node) error { - dealVoucher, ok := voucher.(*StorageDataTransferVoucher) - if !ok { - return xerrors.Errorf("voucher type %s: %w", voucher.Type(), ErrWrongVoucherType) - } - - var deal MinerDeal - err := m.deals.Get(dealVoucher.Proposal, &deal) - if err != nil { - return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal) - } - if deal.Client != sender { - return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Client.String(), sender.String(), ErrWrongPeer) - } - - if !deal.Ref.Equals(baseCid) { - return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece) - } - for _, state := range DataTransferStates { - if deal.State == state { - return nil - } - } - return xerrors.Errorf("Deal State %s: %w", deal.State, ErrInacceptableDealState) -} - -// ValidatePull validates a pull request received from the peer that will receive data. -// Will always error because providers should not accept pull requests from a client -// in a storage deal (i.e. send data to client). -func (m *ProviderRequestValidator) ValidatePull( - receiver peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - Selector ipld.Node) error { - return ErrNoPullAccepted -} diff --git a/chain/deals/request_validation_test.go b/chain/deals/request_validation_test.go deleted file mode 100644 index 7607eacb5..000000000 --- a/chain/deals/request_validation_test.go +++ /dev/null @@ -1,274 +0,0 @@ -package deals_test - -import ( - "fmt" - "math/rand" - "testing" - - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - dss "github.com/ipfs/go-datastore/sync" - blocksutil "github.com/ipfs/go-ipfs-blocksutil" - "github.com/libp2p/go-libp2p-core/peer" - xerrors "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/deals" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storagemarket" -) - -var blockGenerator = blocksutil.NewBlockGenerator() - -type wrongDTType struct { -} - -func (wrongDTType) ToBytes() ([]byte, error) { - return []byte{}, nil -} - -func (wrongDTType) FromBytes([]byte) error { - return fmt.Errorf("not implemented") -} - -func (wrongDTType) Type() string { - return "WrongDTTYPE" -} - -func uniqueStorageDealProposal() (actors.StorageDealProposal, error) { - clientAddr, err := address.NewIDAddress(uint64(rand.Int())) - if err != nil { - return actors.StorageDealProposal{}, err - } - providerAddr, err := address.NewIDAddress(uint64(rand.Int())) - if err != nil { - return actors.StorageDealProposal{}, err - } - return actors.StorageDealProposal{ - PieceRef: blockGenerator.Next().Cid().Bytes(), - Client: clientAddr, - Provider: providerAddr, - ProposerSignature: &types.Signature{ - Data: []byte("foo bar cat dog"), - Type: types.KTBLS, - }, - }, nil -} - -func newClientDeal(minerID peer.ID, state api.DealState) (deals.ClientDeal, error) { - newProposal, err := uniqueStorageDealProposal() - if err != nil { - return deals.ClientDeal{}, err - } - proposalNd, err := cborutil.AsIpld(&newProposal) - if err != nil { - return deals.ClientDeal{}, err - } - minerAddr, err := address.NewIDAddress(uint64(rand.Int())) - if err != nil { - return deals.ClientDeal{}, err - } - - return deals.ClientDeal{ - ClientDeal: storagemarket.ClientDeal{ - Proposal: newProposal, - ProposalCid: proposalNd.Cid(), - PayloadCid: blockGenerator.Next().Cid(), - Miner: minerID, - MinerWorker: minerAddr, - State: state, - }, - }, nil -} - -func newMinerDeal(clientID peer.ID, state api.DealState) (deals.MinerDeal, error) { - newProposal, err := uniqueStorageDealProposal() - if err != nil { - return deals.MinerDeal{}, err - } - proposalNd, err := cborutil.AsIpld(&newProposal) - if err != nil { - return deals.MinerDeal{}, err - } - ref := blockGenerator.Next().Cid() - - return deals.MinerDeal{ - Proposal: newProposal, - ProposalCid: proposalNd.Cid(), - Client: clientID, - State: state, - Ref: ref, - }, nil -} - -func TestClientRequestValidation(t *testing.T) { - ds := dss.MutexWrap(datastore.NewMapDatastore()) - state := statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))) - - crv := deals.NewClientRequestValidator(state) - minerID := peer.ID("fakepeerid") - block := blockGenerator.Next() - t.Run("ValidatePush fails", func(t *testing.T) { - if !xerrors.Is(crv.ValidatePush(minerID, wrongDTType{}, block.Cid(), nil), deals.ErrNoPushAccepted) { - t.Fatal("Push should fail for the client request validator for storage deals") - } - }) - t.Run("ValidatePull fails deal not found", func(t *testing.T) { - proposal, err := uniqueStorageDealProposal() - if err != nil { - t.Fatal("error creating proposal") - } - proposalNd, err := cborutil.AsIpld(&proposal) - if err != nil { - t.Fatal("error serializing proposal") - } - pieceRef, err := cid.Cast(proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{proposalNd.Cid(), 1}, pieceRef, nil), deals.ErrNoDeal) { - t.Fatal("Pull should fail if there is no deal stored") - } - }) - t.Run("ValidatePull fails wrong client", func(t *testing.T) { - otherMiner := peer.ID("otherminer") - clientDeal, err := newClientDeal(otherMiner, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { - t.Fatal("deal tracking failed") - } - payloadCid := clientDeal.PayloadCid - if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, payloadCid, nil), deals.ErrWrongPeer) { - t.Fatal("Pull should fail if miner address is incorrect") - } - }) - t.Run("ValidatePull fails wrong piece ref", func(t *testing.T) { - clientDeal, err := newClientDeal(minerID, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { - t.Fatal("deal tracking failed") - } - if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) { - t.Fatal("Pull should fail if piece ref is incorrect") - } - }) - t.Run("ValidatePull fails wrong deal state", func(t *testing.T) { - clientDeal, err := newClientDeal(minerID, api.DealComplete) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { - t.Fatal("deal tracking failed") - } - payloadCid := clientDeal.PayloadCid - if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, payloadCid, nil), deals.ErrInacceptableDealState) { - t.Fatal("Pull should fail if deal is in a state that cannot be data transferred") - } - }) - t.Run("ValidatePull succeeds", func(t *testing.T) { - clientDeal, err := newClientDeal(minerID, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { - t.Fatal("deal tracking failed") - } - payloadCid := clientDeal.PayloadCid - if crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, payloadCid, nil) != nil { - t.Fatal("Pull should should succeed when all parameters are correct") - } - }) -} - -func TestProviderRequestValidation(t *testing.T) { - ds := dss.MutexWrap(datastore.NewMapDatastore()) - state := statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))) - - mrv := deals.NewProviderRequestValidator(state) - clientID := peer.ID("fakepeerid") - block := blockGenerator.Next() - t.Run("ValidatePull fails", func(t *testing.T) { - if !xerrors.Is(mrv.ValidatePull(clientID, wrongDTType{}, block.Cid(), nil), deals.ErrNoPullAccepted) { - t.Fatal("Pull should fail for the provider request validator for storage deals") - } - }) - - t.Run("ValidatePush fails deal not found", func(t *testing.T) { - proposal, err := uniqueStorageDealProposal() - if err != nil { - t.Fatal("error creating proposal") - } - proposalNd, err := cborutil.AsIpld(&proposal) - if err != nil { - t.Fatal("error serializing proposal") - } - pieceRef, err := cid.Cast(proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{proposalNd.Cid(), 1}, pieceRef, nil), deals.ErrNoDeal) { - t.Fatal("Push should fail if there is no deal stored") - } - }) - t.Run("ValidatePush fails wrong miner", func(t *testing.T) { - otherClient := peer.ID("otherclient") - minerDeal, err := newMinerDeal(otherClient, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { - t.Fatal("deal tracking failed") - } - ref := minerDeal.Ref - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, ref, nil), deals.ErrWrongPeer) { - t.Fatal("Push should fail if miner address is incorrect") - } - }) - t.Run("ValidatePush fails wrong piece ref", func(t *testing.T) { - minerDeal, err := newMinerDeal(clientID, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { - t.Fatal("deal tracking failed") - } - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) { - t.Fatal("Push should fail if piece ref is incorrect") - } - }) - t.Run("ValidatePush fails wrong deal state", func(t *testing.T) { - minerDeal, err := newMinerDeal(clientID, api.DealComplete) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { - t.Fatal("deal tracking failed") - } - ref := minerDeal.Ref - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, ref, nil), deals.ErrInacceptableDealState) { - t.Fatal("Push should fail if deal is in a state that cannot be data transferred") - } - }) - t.Run("ValidatePush succeeds", func(t *testing.T) { - minerDeal, err := newMinerDeal(clientID, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { - t.Fatal("deal tracking failed") - } - ref := minerDeal.Ref - if mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, ref, nil) != nil { - t.Fatal("Push should should succeed when all parameters are correct") - } - }) -} diff --git a/chain/deals/types.go b/chain/deals/types.go deleted file mode 100644 index a35f537ab..000000000 --- a/chain/deals/types.go +++ /dev/null @@ -1,112 +0,0 @@ -package deals - -import ( - "bytes" - "errors" - - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" -) - -var ( - // ErrWrongVoucherType means the voucher was not the correct type can validate against - ErrWrongVoucherType = errors.New("cannot validate voucher type.") - - // ErrNoPushAccepted just means clients do not accept pushes for storage deals - ErrNoPushAccepted = errors.New("client should not receive data for a storage deal.") - - // ErrNoPullAccepted just means providers do not accept pulls for storage deals - ErrNoPullAccepted = errors.New("provider should not send data for a storage deal.") - - // ErrNoDeal means no active deal was found for this vouchers proposal cid - ErrNoDeal = errors.New("no deal found for this proposal.") - - // ErrWrongPeer means that the other peer for this data transfer request does not match - // the other peer for the deal - ErrWrongPeer = errors.New("data Transfer peer id and Deal peer id do not match.") - - // ErrWrongPiece means that the pieceref for this data transfer request does not match - // the one specified in the deal - ErrWrongPiece = errors.New("base CID for deal does not match CID for piece.") - - // ErrInacceptableDealState means the deal for this transfer is not in a deal state - // where transfer can be performed - ErrInacceptableDealState = errors.New("deal is not a in a state where deals are accepted.") - - // DataTransferStates are the states in which it would make sense to actually start a data transfer - DataTransferStates = []api.DealState{api.DealAccepted, api.DealUnknown} -) - -type Proposal struct { - DealProposal *actors.StorageDealProposal - - Piece cid.Cid // Used for retrieving from the client -} - -type Response struct { - State api.DealState - - // DealProposalRejected - Message string - Proposal cid.Cid - - // DealAccepted - PublishMessage *cid.Cid -} - -// TODO: Do we actually need this to be signed? -type SignedResponse struct { - Response Response - - Signature *types.Signature -} - -func (r *SignedResponse) Verify(addr address.Address) error { - b, err := cborutil.Dump(&r.Response) - if err != nil { - return err - } - - return r.Signature.Verify(addr, b) -} - -type AskRequest struct { - Miner address.Address -} - -type AskResponse struct { - Ask *types.SignedStorageAsk -} - -// StorageDataTransferVoucher is the voucher type for data transfers -// used by the storage market -type StorageDataTransferVoucher struct { - Proposal cid.Cid - DealID uint64 -} - -// ToBytes converts the StorageDataTransferVoucher to raw bytes -func (dv *StorageDataTransferVoucher) ToBytes() ([]byte, error) { - var buf bytes.Buffer - err := dv.MarshalCBOR(&buf) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// FromBytes converts the StorageDataTransferVoucher to raw bytes -func (dv *StorageDataTransferVoucher) FromBytes(raw []byte) error { - r := bytes.NewReader(raw) - return dv.UnmarshalCBOR(r) -} - -// Type is the unique string identifier for a StorageDataTransferVoucher -func (dv *StorageDataTransferVoucher) Type() string { - return "StorageDataTransferVoucher" -} diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index ba547d65e..9704eca09 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -6,6 +6,7 @@ import ( "crypto/rand" "encoding/json" "fmt" + "github.com/filecoin-project/lotus/storagemarketadapter" "io/ioutil" "os" "path/filepath" @@ -23,10 +24,11 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/go-fil-markets/storagemarket" + deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/genesis" @@ -281,15 +283,21 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin dealKey := datastore.NewKey(deals.ProviderDsPrefix).ChildString(proposalCid.String()) + proposal, err := storagemarketadapter.ToSharedStorageDealProposal(§or.Deal) + if err != nil { + return err + } deal := &deals.MinerDeal{ - Proposal: sector.Deal, - ProposalCid: proposalCid, - State: lapi.DealComplete, - Ref: proposalCid, // TODO: This is super wrong, but there - // are no params for CommP CIDs, we can't recover unixfs cid easily, - // and this isn't even used after the deal enters Complete state - DealID: dealID, - SectorID: sector.SectorID, + MinerDeal: storagemarket.MinerDeal{ + Proposal: *proposal, + ProposalCid: proposalCid, + State: lapi.DealComplete, + Ref: proposalCid, // TODO: This is super wrong, but there + // are no params for CommP CIDs, we can't recover unixfs cid easily, + // and this isn't even used after the deal enters Complete state + DealID: dealID, + SectorID: sector.SectorID, + }, } b, err = cborutil.Dump(deal) diff --git a/gen/main.go b/gen/main.go index 0cb569148..7778135ad 100644 --- a/gen/main.go +++ b/gen/main.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "github.com/filecoin-project/lotus/storagemarket" "os" gen "github.com/whyrusleeping/cbor-gen" @@ -10,7 +9,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/blocksync" - "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/paych" "github.com/filecoin-project/lotus/storage" @@ -123,31 +121,6 @@ func main() { os.Exit(1) } - err = gen.WriteTupleEncodersToFile("./storagemarket/cbor_gen.go", "storagemarket", - storagemarket.ClientDeal{}, - storagemarket.MinerDeal{}, - ) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - err = gen.WriteTupleEncodersToFile("./chain/deals/cbor_gen.go", "deals", - deals.AskRequest{}, - deals.AskResponse{}, - deals.Proposal{}, - deals.Response{}, - deals.SignedResponse{}, - deals.ClientDealProposal{}, - deals.ClientDeal{}, - deals.MinerDeal{}, - deals.StorageDataTransferVoucher{}, - ) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - err = gen.WriteMapEncodersToFile("./storage/cbor_gen.go", "storage", storage.SealTicket{}, storage.SealSeed{}, diff --git a/go.mod b/go.mod index 1bcb04ec4..ee48d7fe5 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,6 @@ require ( github.com/ipfs/go-graphsync v0.0.4 github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1 github.com/ipfs/go-ipfs-blockstore v0.1.1 - github.com/ipfs/go-ipfs-blocksutil v0.0.1 github.com/ipfs/go-ipfs-chunker v0.0.1 github.com/ipfs/go-ipfs-ds-help v0.0.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 diff --git a/lib/sharedutils/converters.go b/lib/sharedutils/converters.go new file mode 100644 index 000000000..58a5b44f9 --- /dev/null +++ b/lib/sharedutils/converters.go @@ -0,0 +1,101 @@ +package sharedutils + +import ( + "bytes" + sharedamount "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/lotus/chain/types" +) + + +func FromSharedTokenAmount(in sharedamount.TokenAmount) types.BigInt { + return types.BigInt{Int: in.Int} +} + +func ToSharedTokenAmount(in types.BigInt) sharedamount.TokenAmount { + return sharedamount.TokenAmount{Int: in.Int} +} + +func ToSharedSignedVoucher(in *types.SignedVoucher) (*sharedtypes.SignedVoucher, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out sharedtypes.SignedVoucher + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSharedSignedVoucher(in *sharedtypes.SignedVoucher) (*types.SignedVoucher, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out types.SignedVoucher + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func ToSharedSignature(in *types.Signature) (*sharedtypes.Signature, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out sharedtypes.Signature + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSharedSignature(in *sharedtypes.Signature) (*types.Signature, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out types.Signature + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func ToSharedStorageAsk(in *types.SignedStorageAsk) (*sharedtypes.SignedStorageAsk, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out sharedtypes.SignedStorageAsk + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSignedStorageAsk(in *sharedtypes.SignedStorageAsk) (*types.SignedStorageAsk, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out types.SignedStorageAsk + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} diff --git a/node/builder.go b/node/builder.go index ee7286908..9a656563c 100644 --- a/node/builder.go +++ b/node/builder.go @@ -21,10 +21,11 @@ import ( "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" + "github.com/filecoin-project/go-fil-markets/storagemarket" + deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/blocksync" - "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/market" "github.com/filecoin-project/lotus/chain/messagepool" @@ -47,7 +48,6 @@ import ( "github.com/filecoin-project/lotus/peermgr" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" - "github.com/filecoin-project/lotus/storagemarket" "github.com/filecoin-project/lotus/storagemarketadapter" ) @@ -228,8 +228,8 @@ func Online() Option { Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient), Override(new(dtypes.ClientDealStore), modules.NewClientDealStore), Override(new(dtypes.ClientDataTransfer), modules.NewClientDAGServiceDataTransfer), - Override(new(*deals.ClientRequestValidator), deals.NewClientRequestValidator), - Override(new(storagemarket.StorageClient), deals.NewClient), + Override(new(*deals.ClientRequestValidator), modules.NewClientRequestValidator), + Override(new(storagemarket.StorageClient), modules.StorageClient), Override(new(storagemarket.StorageClientNode), storagemarketadapter.NewClientNodeAdapter), Override(RegisterClientValidatorKey, modules.RegisterClientValidator), Override(RunDealClientKey, modules.RunDealClient), @@ -252,8 +252,8 @@ func Online() Option { Override(new(retrievalmarket.RetrievalProvider), modules.RetrievalProvider), Override(new(dtypes.ProviderDealStore), modules.NewProviderDealStore), Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), - Override(new(*deals.ProviderRequestValidator), deals.NewProviderRequestValidator), - Override(new(storagemarket.StorageProvider), deals.NewProvider), + Override(new(*deals.ProviderRequestValidator), modules.NewProviderRequestValidator), + Override(new(storagemarket.StorageProvider), modules.StorageProvider), Override(new(storagemarket.StorageProviderNode), storagemarketadapter.NewProviderNodeAdapter), Override(RegisterProviderValidatorKey, modules.RegisterProviderValidator), Override(HandleRetrievalKey, modules.HandleRetrieval), diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 09bd5d9ee..70aff2b0c 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -26,15 +26,18 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sharedutils" "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/retrievaladapter" - "github.com/filecoin-project/lotus/storagemarket" + "github.com/filecoin-project/lotus/storagemarketadapter" ) type API struct { @@ -73,20 +76,16 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, addr address.Ad if err != nil { return nil, xerrors.Errorf("failed getting miner worker: %w", err) } - providerInfo := storagemarket.StorageProviderInfo{ - Address: miner, - Worker: mw, - PeerID: pid, - } + providerInfo := storagemarketadapter.NewStorageProviderInfo(miner, mw, 0, pid) result, err := a.SMDealClient.ProposeStorageDeal( ctx, - addr, + sharedutils.ToSharedAddress(addr), &providerInfo, data, storagemarket.Epoch(math.MaxUint64), storagemarket.Epoch(blocksDuration), - storagemarket.TokenAmount(epochPrice), - storagemarket.TokenAmount(storagemarket.EmptyInt)) + sharedutils.ToSharedTokenAmount(epochPrice), + tokenamount.Empty) if err != nil { return nil, xerrors.Errorf("failed to start deal: %w", err) @@ -106,12 +105,12 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { out[k] = api.DealInfo{ ProposalCid: v.ProposalCid, State: v.State, - Provider: v.Proposal.Provider, + Provider: sharedutils.FromSharedAddress(v.Proposal.Provider), PieceRef: v.Proposal.PieceRef, Size: v.Proposal.PieceSize, - PricePerEpoch: v.Proposal.StoragePricePerEpoch, + PricePerEpoch: sharedutils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch), Duration: v.Proposal.Duration, } } @@ -128,10 +127,10 @@ func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo, return &api.DealInfo{ ProposalCid: v.ProposalCid, State: v.State, - Provider: v.Proposal.Provider, + Provider: sharedutils.FromSharedAddress(v.Proposal.Provider), PieceRef: v.Proposal.PieceRef, Size: v.Proposal.PieceSize, - PricePerEpoch: v.Proposal.StoragePricePerEpoch, + PricePerEpoch: sharedutils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch), Duration: v.Proposal.Duration, }, nil } @@ -160,13 +159,13 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffe for k, p := range peers { queryResponse, err := a.Retrieval.Query(ctx, p, root.Bytes(), retrievalmarket.QueryParams{}) if err != nil { - out[k] = api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} + out[k] = api.QueryOffer{Err: err.Error(), Miner: retrievaladapter.FromSharedAddress(p.Address), MinerPeerID: p.ID} } else { out[k] = api.QueryOffer{ Root: root, Size: queryResponse.Size, MinPrice: retrievaladapter.FromSharedTokenAmount(queryResponse.PieceRetrievalPrice()), - Miner: p.Address, // TODO: check + Miner: retrievaladapter.FromSharedAddress(p.Address), // TODO: check MinerPeerID: p.ID, } } @@ -296,10 +295,10 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path ctx, order.Root.Bytes(), retrievalmarket.NewParamsV0(types.BigDiv(order.Total, types.NewInt(order.Size)).Int, 0, 0), - retrievaladapter.ToSharedTokenAmount(order.Total), + sharedutils.ToSharedTokenAmount(order.Total), order.MinerPeerID, - order.Client, - order.Miner) + retrievaladapter.ToSharedAddress(order.Client), + retrievaladapter.ToSharedAddress(order.Miner)) select { case <-ctx.Done(): return xerrors.New("Retrieval Timed Out") @@ -323,6 +322,10 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path } func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) { - info := storagemarket.StorageProviderInfo{Address: miner, PeerID: p} - return a.SMDealClient.GetAsk(ctx, info) + info := storagemarketadapter.NewStorageProviderInfo(miner, address.Undef, 0, p) + signedAsk, err := a.SMDealClient.GetAsk(ctx, info) + if err != nil { + return nil, err + } + return sharedutils.FromSignedStorageAsk(signedAsk) } diff --git a/node/modules/client.go b/node/modules/client.go index a03ea3691..313672c6b 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -2,13 +2,18 @@ package modules import ( "context" - "github.com/filecoin-project/lotus/retrievaladapter" "path/filepath" "reflect" + "github.com/filecoin-project/lotus/retrievaladapter" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" + + "github.com/filecoin-project/go-statestore" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/paych" @@ -30,7 +35,8 @@ import ( "go.uber.org/fx" "github.com/filecoin-project/go-data-transfer/impl/graphsync" - "github.com/filecoin-project/lotus/chain/deals" + deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" + storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl" payapi "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" @@ -104,6 +110,14 @@ func ClientGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.Client return gs } +func NewClientRequestValidator(deals dtypes.ClientDealStore) *storageimpl.ClientRequestValidator { + return storageimpl.NewClientRequestValidator(deals) +} + +func StorageClient(h host.Host, dag dtypes.ClientDAG, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, deals dtypes.ClientDealStore, scn storagemarket.StorageClientNode) storagemarket.StorageClient { + return storageimpl.NewClient(h, dag, dataTransfer, discovery, deals, scn) +} + // RetrievalClient creates a new retrieval client attached to the client blockstore func RetrievalClient(h host.Host, bs dtypes.ClientBlockstore, pmgr *paych.Manager, payapi payapi.PaychAPI) retrievalmarket.RetrievalClient { adapter := retrievaladapter.NewRetrievalClientNode(pmgr, payapi) diff --git a/node/modules/services.go b/node/modules/services.go index bf1985041..e12b9a310 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -18,7 +18,7 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/peermgr" - "github.com/filecoin-project/lotus/storagemarket" + "github.com/filecoin-project/go-fil-components/storagemarket" ) func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello.Service) { diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 1197870d5..1ddbb433a 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -27,21 +27,26 @@ import ( dtgraphsync "github.com/filecoin-project/go-data-transfer/impl/graphsync" "github.com/filecoin-project/go-fil-markets/retrievalmarket" retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" + deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" + storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/gen" + "github.com/filecoin-project/lotus/lib/sectorbuilder" + "github.com/filecoin-project/lotus/lib/statestore" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/go-fil-components/storagemarket" "github.com/filecoin-project/lotus/retrievaladapter" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" - "github.com/filecoin-project/lotus/storagemarket" ) func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { @@ -264,6 +269,14 @@ func SealTicketGen(api api.FullNode) storage.TicketFn { } } +func NewProviderRequestValidator(deals dtypes.ProviderDealStore) *storageimpl.ProviderRequestValidator { + return storageimpl.NewProviderRequestValidator(deals) +} + +func StorageProvider(ds dtypes.MetadataDS, dag dtypes.StagingDAG, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode) (storagemarket.StorageProvider, error) { + return storageimpl.NewProvider(ds, dag, dataTransfer, spn) +} + // RetrievalProvider creates a new retrieval provider attached to the provider blockstore func RetrievalProvider(sblks *sectorblocks.SectorBlocks, full api.FullNode) retrievalmarket.RetrievalProvider { adapter := retrievaladapter.NewRetrievalProviderNode(sblks, full) diff --git a/retrievaladapter/client.go b/retrievaladapter/client.go index 8f0289921..cd5f819e5 100644 --- a/retrievaladapter/client.go +++ b/retrievaladapter/client.go @@ -2,6 +2,7 @@ package retrievaladapter import ( "context" + "github.com/filecoin-project/lotus/lib/sharedutils" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" @@ -26,7 +27,7 @@ func NewRetrievalClientNode(pmgr *paych.Manager, payapi payapi.PaychAPI) retriev // GetOrCreatePaymentChannel sets up a new payment channel if one does not exist // between a client and a miner and insures the client has the given amount of funds available in the channel func (rcn *retrievalClientNode) GetOrCreatePaymentChannel(ctx context.Context, clientAddress address.Address, minerAddress address.Address, clientFundsAvailable retrievaltoken.TokenAmount) (address.Address, error) { - paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, FromSharedTokenAmount(clientFundsAvailable)) + paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, sharedutils.FromSharedTokenAmount(clientFundsAvailable)) return paych, err } @@ -41,9 +42,9 @@ func (rcn *retrievalClientNode) AllocateLane(paymentChannel address.Address) (ui // given payment channel so that all the payment vouchers in the lane add up // to the given amount (so the payment voucher will be for the difference) func (rcn *retrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount retrievaltoken.TokenAmount, lane uint64) (*retrievaltypes.SignedVoucher, error) { - voucher, err := rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, FromSharedTokenAmount(amount), lane) + voucher, err := rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, sharedutils.FromSharedTokenAmount(amount), lane) if err != nil { return nil, err } - return ToSharedSignedVoucher(voucher) + return sharedutils.ToSharedSignedVoucher(voucher) } diff --git a/retrievaladapter/converters.go b/retrievaladapter/converters.go deleted file mode 100644 index 7ea3ffec0..000000000 --- a/retrievaladapter/converters.go +++ /dev/null @@ -1,45 +0,0 @@ -package retrievaladapter - -import ( - "bytes" - sharedamount "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" - "github.com/filecoin-project/lotus/chain/types" -) - - -func FromSharedTokenAmount(in sharedamount.TokenAmount) types.BigInt { - return types.BigInt{Int: in.Int} -} - -func ToSharedTokenAmount(in types.BigInt) sharedamount.TokenAmount { - return sharedamount.TokenAmount{Int: in.Int} -} - -func ToSharedSignedVoucher(in *types.SignedVoucher) (*sharedtypes.SignedVoucher, error) { - var encoded bytes.Buffer - err := in.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out sharedtypes.SignedVoucher - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} - -func FromSharedSignedVoucher(in *sharedtypes.SignedVoucher) (*types.SignedVoucher, error) { - var encoded bytes.Buffer - err := in.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out types.SignedVoucher - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} diff --git a/retrievaladapter/provider.go b/retrievaladapter/provider.go index c5b238eda..d2499757b 100644 --- a/retrievaladapter/provider.go +++ b/retrievaladapter/provider.go @@ -2,6 +2,7 @@ package retrievaladapter import ( "context" + "github.com/filecoin-project/lotus/lib/sharedutils" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" @@ -37,10 +38,10 @@ func (rpn *retrievalProviderNode) SealedBlockstore(approveUnseal func() error) b } func (rpn *retrievalProviderNode) SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *retrievaltypes.SignedVoucher, proof []byte, expectedAmount retrievaltoken.TokenAmount) (retrievaltoken.TokenAmount, error) { - localVoucher, err := FromSharedSignedVoucher(voucher) + localVoucher, err := sharedutils.FromSharedSignedVoucher(voucher) if err != nil { return retrievaltoken.FromInt(0), err } - added, err := rpn.full.PaychVoucherAdd(ctx, paymentChannel, localVoucher, proof, FromSharedTokenAmount(expectedAmount)) - return ToSharedTokenAmount(added), err + added, err := rpn.full.PaychVoucherAdd(ctx, paymentChannel, localVoucher, proof, sharedutils.FromSharedTokenAmount(expectedAmount)) + return sharedutils.ToSharedTokenAmount(added), err } diff --git a/storagemarket/bigint.go b/storagemarket/bigint.go deleted file mode 100644 index f36e242b0..000000000 --- a/storagemarket/bigint.go +++ /dev/null @@ -1,241 +0,0 @@ -// Copied from lotus until this can be extracted into shared types - -package storagemarket - -import ( - "encoding/json" - "fmt" - "io" - "math/big" - - "github.com/filecoin-project/lotus/build" - cbor "github.com/ipfs/go-ipld-cbor" - "github.com/polydawn/refmt/obj/atlas" - - cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" -) - -const BigIntMaxSerializedLen = 128 // is this big enough? or too big? - -var TotalFilecoinInt = FromFil(build.TotalFilecoin) - -func init() { - cbor.RegisterCborType(atlas.BuildEntry(BigInt{}).Transform(). - TransformMarshal(atlas.MakeMarshalTransformFunc( - func(i BigInt) ([]byte, error) { - return i.cborBytes(), nil - })). - TransformUnmarshal(atlas.MakeUnmarshalTransformFunc( - func(x []byte) (BigInt, error) { - return fromCborBytes(x) - })). - Complete()) -} - -var EmptyInt = BigInt{} - -type BigInt struct { - *big.Int -} - -func NewInt(i uint64) BigInt { - return BigInt{big.NewInt(0).SetUint64(i)} -} - -func FromFil(i uint64) BigInt { - return BigMul(NewInt(i), NewInt(build.FilecoinPrecision)) -} - -func BigFromBytes(b []byte) BigInt { - i := big.NewInt(0).SetBytes(b) - return BigInt{i} -} - -func BigFromString(s string) (BigInt, error) { - v, ok := big.NewInt(0).SetString(s, 10) - if !ok { - return BigInt{}, fmt.Errorf("failed to parse string as a big int") - } - - return BigInt{v}, nil -} - -func BigMul(a, b BigInt) BigInt { - return BigInt{big.NewInt(0).Mul(a.Int, b.Int)} -} - -func BigDiv(a, b BigInt) BigInt { - return BigInt{big.NewInt(0).Div(a.Int, b.Int)} -} - -func BigMod(a, b BigInt) BigInt { - return BigInt{big.NewInt(0).Mod(a.Int, b.Int)} -} - -func BigAdd(a, b BigInt) BigInt { - return BigInt{big.NewInt(0).Add(a.Int, b.Int)} -} - -func BigSub(a, b BigInt) BigInt { - return BigInt{big.NewInt(0).Sub(a.Int, b.Int)} -} - -func BigCmp(a, b BigInt) int { - return a.Int.Cmp(b.Int) -} - -func (bi BigInt) Nil() bool { - return bi.Int == nil -} - -// LessThan returns true if bi < o -func (bi BigInt) LessThan(o BigInt) bool { - return BigCmp(bi, o) < 0 -} - -// GreaterThan returns true if bi > o -func (bi BigInt) GreaterThan(o BigInt) bool { - return BigCmp(bi, o) > 0 -} - -// Equals returns true if bi == o -func (bi BigInt) Equals(o BigInt) bool { - return BigCmp(bi, o) == 0 -} - -func (bi *BigInt) MarshalJSON() ([]byte, error) { - return json.Marshal(bi.String()) -} - -func (bi *BigInt) UnmarshalJSON(b []byte) error { - var s string - if err := json.Unmarshal(b, &s); err != nil { - return err - } - - i, ok := big.NewInt(0).SetString(s, 10) - if !ok { - if string(s) == "" { - return nil - } - return xerrors.Errorf("failed to parse bigint string: '%s'", string(b)) - } - - bi.Int = i - return nil -} - -func (bi *BigInt) Scan(value interface{}) error { - switch value := value.(type) { - case string: - i, ok := big.NewInt(0).SetString(value, 10) - if !ok { - if value == "" { - return nil - } - return xerrors.Errorf("failed to parse bigint string: '%s'", value) - } - - bi.Int = i - - return nil - case int64: - bi.Int = big.NewInt(value) - return nil - default: - return xerrors.Errorf("non-string types unsupported: %T", value) - } -} - -func (bi *BigInt) cborBytes() []byte { - if bi.Int == nil { - return []byte{} - } - - switch { - case bi.Sign() > 0: - return append([]byte{0}, bi.Bytes()...) - case bi.Sign() < 0: - return append([]byte{1}, bi.Bytes()...) - default: // bi.Sign() == 0: - return []byte{} - } -} - -func fromCborBytes(buf []byte) (BigInt, error) { - if len(buf) == 0 { - return NewInt(0), nil - } - - var negative bool - switch buf[0] { - case 0: - negative = false - case 1: - negative = true - default: - return EmptyInt, fmt.Errorf("big int prefix should be either 0 or 1, got %d", buf[0]) - } - - i := big.NewInt(0).SetBytes(buf[1:]) - if negative { - i.Neg(i) - } - - return BigInt{i}, nil -} - -func (bi *BigInt) MarshalCBOR(w io.Writer) error { - if bi.Int == nil { - zero := NewInt(0) - return zero.MarshalCBOR(w) - } - - enc := bi.cborBytes() - - header := cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(enc))) - if _, err := w.Write(header); err != nil { - return err - } - - if _, err := w.Write(enc); err != nil { - return err - } - - return nil -} - -func (bi *BigInt) UnmarshalCBOR(br io.Reader) error { - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - - if maj != cbg.MajByteString { - return fmt.Errorf("cbor input for fil big int was not a byte string (%x)", maj) - } - - if extra == 0 { - bi.Int = big.NewInt(0) - return nil - } - - if extra > BigIntMaxSerializedLen { - return fmt.Errorf("big integer byte array too long") - } - - buf := make([]byte, extra) - if _, err := io.ReadFull(br, buf); err != nil { - return err - } - - i, err := fromCborBytes(buf) - if err != nil { - return err - } - - *bi = i - - return nil -} diff --git a/storagemarket/cbor_gen.go b/storagemarket/cbor_gen.go deleted file mode 100644 index e4a9efcfa..000000000 --- a/storagemarket/cbor_gen.go +++ /dev/null @@ -1,352 +0,0 @@ -package storagemarket - -import ( - "fmt" - "io" - - "github.com/libp2p/go-libp2p-core/peer" - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -var _ = xerrors.Errorf - -func (t *ClientDeal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{136}); err != nil { - return err - } - - // t.ProposalCid (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.ProposalCid); err != nil { - return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) - } - - // t.Proposal (actors.StorageDealProposal) (struct) - if err := t.Proposal.MarshalCBOR(w); err != nil { - return err - } - - // t.State (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { - return err - } - - // t.Miner (peer.ID) (string) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Miner)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Miner)); err != nil { - return err - } - - // t.MinerWorker (address.Address) (struct) - if err := t.MinerWorker.MarshalCBOR(w); err != nil { - return err - } - - // t.DealID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { - return err - } - - // t.PayloadCid (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.PayloadCid); err != nil { - return xerrors.Errorf("failed to write cid field t.PayloadCid: %w", err) - } - - // t.PublishMessage (cid.Cid) (struct) - - if t.PublishMessage == nil { - if _, err := w.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(w, *t.PublishMessage); err != nil { - return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err) - } - } - - return nil -} - -func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 8 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.ProposalCid (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err) - } - - t.ProposalCid = c - - } - // t.Proposal (actors.StorageDealProposal) (struct) - - { - - if err := t.Proposal.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.State (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.State = uint64(extra) - // t.Miner (peer.ID) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Miner = peer.ID(sval) - } - // t.MinerWorker (address.Address) (struct) - - { - - if err := t.MinerWorker.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.DealID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DealID = uint64(extra) - // t.PayloadCid (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PayloadCid: %w", err) - } - - t.PayloadCid = c - - } - // t.PublishMessage (cid.Cid) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err) - } - - t.PublishMessage = &c - } - - } - return nil -} - -func (t *MinerDeal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{136}); err != nil { - return err - } - - // t.ProposalCid (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.ProposalCid); err != nil { - return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) - } - - // t.Proposal (actors.StorageDealProposal) (struct) - if err := t.Proposal.MarshalCBOR(w); err != nil { - return err - } - - // t.Miner (peer.ID) (string) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Miner)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Miner)); err != nil { - return err - } - - // t.Client (peer.ID) (string) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Client)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Client)); err != nil { - return err - } - - // t.State (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { - return err - } - - // t.Ref (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Ref); err != nil { - return xerrors.Errorf("failed to write cid field t.Ref: %w", err) - } - - // t.DealID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { - return err - } - - // t.SectorID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil { - return err - } - return nil -} - -func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 8 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.ProposalCid (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err) - } - - t.ProposalCid = c - - } - // t.Proposal (actors.StorageDealProposal) (struct) - - { - - if err := t.Proposal.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.Miner (peer.ID) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Miner = peer.ID(sval) - } - // t.Client (peer.ID) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Client = peer.ID(sval) - } - // t.State (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.State = uint64(extra) - // t.Ref (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Ref: %w", err) - } - - t.Ref = c - - } - // t.DealID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DealID = uint64(extra) - // t.SectorID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.SectorID = uint64(extra) - return nil -} diff --git a/storagemarket/types.go b/storagemarket/types.go deleted file mode 100644 index 1497063c4..000000000 --- a/storagemarket/types.go +++ /dev/null @@ -1,199 +0,0 @@ -package storagemarket - -import ( - "context" - - "github.com/ipfs/go-cid" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/peer" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" -) - -const DealProtocolID = "/fil/storage/mk/1.0.1" -const AskProtocolID = "/fil/storage/ask/1.0.1" - -// type shims - used during migration into separate module -type Balance = actors.StorageParticipantBalance -type DealID uint64 -type Signature = types.Signature -type StorageDeal = actors.OnChainDeal -type StorageAsk = types.SignedStorageAsk -type StateKey = *types.TipSet -type Epoch uint64 -type TokenAmount BigInt - -// Duplicated from deals package for now -type MinerDeal struct { - ProposalCid cid.Cid - Proposal actors.StorageDealProposal - Miner peer.ID - Client peer.ID - State api.DealState - - Ref cid.Cid - - DealID uint64 - SectorID uint64 // Set when sm >= DealStaged -} - -type ClientDeal struct { - ProposalCid cid.Cid - Proposal actors.StorageDealProposal - State api.DealState - Miner peer.ID - MinerWorker address.Address - DealID uint64 - PayloadCid cid.Cid - PublishMessage *cid.Cid -} - -// The interface provided for storage providers -type StorageProvider interface { - Run(ctx context.Context, host host.Host) - - Stop() - - AddAsk(price TokenAmount, ttlsecs int64) error - - // ListAsks lists current asks - ListAsks(addr address.Address) []*StorageAsk - - // ListDeals lists on-chain deals associated with this provider - ListDeals(ctx context.Context) ([]StorageDeal, error) - - // ListIncompleteDeals lists deals that are in progress or rejected - ListIncompleteDeals() ([]MinerDeal, error) - - // AddStorageCollateral adds storage collateral - AddStorageCollateral(ctx context.Context, amount TokenAmount) error - - // GetStorageCollateral returns the current collateral balance - GetStorageCollateral(ctx context.Context) (Balance, error) -} - -// Node dependencies for a StorageProvider -type StorageProviderNode interface { - MostRecentStateId(ctx context.Context) (StateKey, error) - - // Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. - AddFunds(ctx context.Context, addr address.Address, amount TokenAmount) error - - // Ensures that a storage market participant has a certain amount of available funds - EnsureFunds(ctx context.Context, addr address.Address, amount TokenAmount) error - - // GetBalance returns locked/unlocked for a storage participant. Used by both providers and clients. - GetBalance(ctx context.Context, addr address.Address) (Balance, error) - - // Publishes deal on chain - PublishDeals(ctx context.Context, deal MinerDeal) (DealID, cid.Cid, error) - - // ListProviderDeals lists all deals associated with a storage provider - ListProviderDeals(ctx context.Context, addr address.Address) ([]StorageDeal, error) - - // Called when a deal is complete and on chain, and data has been transferred and is ready to be added to a sector - // returns sector id - OnDealComplete(ctx context.Context, deal MinerDeal, piecePath string) (uint64, error) - - // returns the worker address associated with a miner - GetMinerWorker(ctx context.Context, miner address.Address) (address.Address, error) - - // Signs bytes - SignBytes(ctx context.Context, signer address.Address, b []byte) (*types.Signature, error) -} - -type DealSectorCommittedCallback func(error) - -// Node dependencies for a StorageClient -type StorageClientNode interface { - MostRecentStateId(ctx context.Context) (StateKey, error) - - // Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. - AddFunds(ctx context.Context, addr address.Address, amount TokenAmount) error - - EnsureFunds(ctx context.Context, addr address.Address, amount TokenAmount) error - - // GetBalance returns locked/unlocked for a storage participant. Used by both providers and clients. - GetBalance(ctx context.Context, addr address.Address) (Balance, error) - - //// ListClientDeals lists all on-chain deals associated with a storage client - ListClientDeals(ctx context.Context, addr address.Address) ([]StorageDeal, error) - - // GetProviderInfo returns information about a single storage provider - //GetProviderInfo(stateId StateID, addr Address) *StorageProviderInfo - - // GetStorageProviders returns information about known miners - ListStorageProviders(ctx context.Context) ([]*StorageProviderInfo, error) - - // Subscribes to storage market actor state changes for a given address. - // TODO: Should there be a timeout option for this? In the case that we are waiting for funds to be deposited and it never happens? - //SubscribeStorageMarketEvents(addr Address, handler StorageMarketEventHandler) (SubID, error) - - // Cancels a subscription - //UnsubscribeStorageMarketEvents(subId SubID) - ValidatePublishedDeal(ctx context.Context, deal ClientDeal) (uint64, error) - - // SignProposal signs a proposal - SignProposal(ctx context.Context, signer address.Address, proposal *actors.StorageDealProposal) error - - GetDefaultWalletAddress(ctx context.Context) (address.Address, error) - - OnDealSectorCommitted(ctx context.Context, provider address.Address, dealId uint64, cb DealSectorCommittedCallback) error - - ValidateAskSignature(ask *StorageAsk) error -} - -type StorageClientProofs interface { - //GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (CommP, error) -} - -// Closely follows the MinerInfo struct in the spec -type StorageProviderInfo struct { - Address address.Address // actor address - Owner address.Address - Worker address.Address // signs messages - SectorSize uint64 - PeerID peer.ID - // probably more like how much storage power, available collateral etc -} - -type ProposeStorageDealResult struct { - ProposalCid cid.Cid -} - -// The interface provided by the module to the outside world for storage clients. -type StorageClient interface { - Run(ctx context.Context) - - Stop() - - // ListProviders queries chain state and returns active storage providers - ListProviders(ctx context.Context) (<-chan StorageProviderInfo, error) - - // ListDeals lists on-chain deals associated with this provider - ListDeals(ctx context.Context, addr address.Address) ([]StorageDeal, error) - - // ListInProgressDeals lists deals that are in progress or rejected - ListInProgressDeals(ctx context.Context) ([]ClientDeal, error) - - // ListInProgressDeals lists deals that are in progress or rejected - GetInProgressDeal(ctx context.Context, cid cid.Cid) (ClientDeal, error) - - // GetAsk returns the current ask for a storage provider - GetAsk(ctx context.Context, info StorageProviderInfo) (*StorageAsk, error) - - //// FindStorageOffers lists providers and queries them to find offers that satisfy some criteria based on price, duration, etc. - //FindStorageOffers(criteria AskCriteria, limit uint) []*StorageOffer - - // ProposeStorageDeal initiates deal negotiation with a Storage Provider - ProposeStorageDeal(ctx context.Context, addr address.Address, info *StorageProviderInfo, payloadCid cid.Cid, proposalExpiration Epoch, duration Epoch, price TokenAmount, collateral TokenAmount) (*ProposeStorageDealResult, error) - - // GetPaymentEscrow returns the current funds available for deal payment - GetPaymentEscrow(ctx context.Context, addr address.Address) (Balance, error) - - // AddStorageCollateral adds storage collateral - AddPaymentEscrow(ctx context.Context, addr address.Address, amount TokenAmount) error -} diff --git a/storagemarketadapter/client_adapter.go b/storagemarketadapter/client_adapter.go index 9e23771f5..90e764da9 100644 --- a/storagemarketadapter/client_adapter.go +++ b/storagemarketadapter/client_adapter.go @@ -5,11 +5,14 @@ package storagemarketadapter import ( "bytes" "context" + "github.com/filecoin-project/lotus/lib/sharedutils" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" + sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" @@ -19,7 +22,6 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/impl/full" - "github.com/filecoin-project/lotus/storagemarket" ) type ClientNodeAdapter struct { @@ -79,13 +81,8 @@ func (n *ClientNodeAdapter) ListStorageProviders(ctx context.Context) ([]*storag if err != nil { return nil, err } - - out = append(out, &storagemarket.StorageProviderInfo{ - Address: addr, - Worker: workerAddr, - SectorSize: sectorSize, - PeerID: peerId, - }) + storageProviderInfo := NewStorageProviderInfo(addr, workerAddr, sectorSize, peerId) + out = append(out, &storageProviderInfo) } return out, nil @@ -97,11 +94,12 @@ func (n *ClientNodeAdapter) ListClientDeals(ctx context.Context, addr address.Ad return nil, err } - var out []actors.OnChainDeal + var out []storagemarket.StorageDeal for _, deal := range allDeals { - if deal.Client == addr { - out = append(out, deal) + storageDeal := FromOnChainDeal(deal) + if storageDeal.Client == addr { + out = append(out, storageDeal) } } @@ -113,12 +111,12 @@ func (n *ClientNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarke } // Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. -func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount storagemarket.TokenAmount) error { +func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { // (Provider Node API) smsg, err := n.MpoolPushMessage(ctx, &types.Message{ To: actors.StorageMarketAddress, - From: addr, - Value: types.BigInt(amount), + From: sharedutils.FromSharedAddress(addr), + Value: sharedutils.FromSharedTokenAmount(amount), GasPrice: types.NewInt(0), GasLimit: types.NewInt(1000000), Method: actors.SMAMethods.AddBalance, @@ -139,17 +137,17 @@ func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, return nil } -func (n *ClientNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amount storagemarket.TokenAmount) error { - return n.fm.EnsureAvailable(ctx, addr, types.BigInt(amount)) +func (n *ClientNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { + return n.fm.EnsureAvailable(ctx, sharedutils.FromSharedAddress(addr), sharedutils.FromSharedTokenAmount(amount)) } func (n *ClientNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { - bal, err := n.StateMarketBalance(ctx, addr, nil) + bal, err := n.StateMarketBalance(ctx, sharedutils.FromSharedAddress(addr), nil) if err != nil { return storagemarket.Balance{}, err } - return bal, nil + return ToSharedBalance(bal), nil } // ValidatePublishedDeal validates that the provided deal has appeared on chain and references the same ClientDeal @@ -162,7 +160,7 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor return 0, xerrors.Errorf("getting deal pubsish message: %w", err) } - pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider) + pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, sharedutils.FromSharedAddress(deal.Proposal.Provider)) if err != nil { return 0, xerrors.Errorf("getting miner worker failed: %w", err) } @@ -270,7 +268,7 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider } matchEvent := func(msg *types.Message) (bool, error) { - if msg.To != provider { + if msg.To != sharedutils.FromSharedAddress(provider) { return false, nil } @@ -301,18 +299,23 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider return nil } -func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Address, proposal *actors.StorageDealProposal) error { - return api.SignWith(ctx, n.Wallet.Sign, signer, proposal) +func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Address, proposal *storagemarket.StorageDealProposal) error { + localProposal, err := FromSharedStorageDealProposal(proposal) + if err != nil { + return err + } + return api.SignWith(ctx, n.Wallet.Sign, sharedutils.FromSharedAddress(signer), localProposal) } func (n *ClientNodeAdapter) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) { - return n.Wallet.GetDefault() + addr, err := n.Wallet.GetDefault() + return sharedutils.ToSharedAddress(addr), err } -func (n *ClientNodeAdapter) ValidateAskSignature(ask *types.SignedStorageAsk) error { +func (n *ClientNodeAdapter) ValidateAskSignature(ask *sharedtypes.SignedStorageAsk) error { tss := n.cs.GetHeaviestTipSet().ParentState() - w, err := stmgr.GetMinerWorkerRaw(context.TODO(), n.StateManager, tss, ask.Ask.Miner) + w, err := stmgr.GetMinerWorkerRaw(context.TODO(), n.StateManager, tss, sharedutils.FromSharedAddress(ask.Ask.Miner)) if err != nil { return xerrors.Errorf("failed to get worker for miner in ask", err) } @@ -322,7 +325,7 @@ func (n *ClientNodeAdapter) ValidateAskSignature(ask *types.SignedStorageAsk) er return xerrors.Errorf("failed to re-serialize ask") } - return ask.Signature.Verify(w, sigb) + return ask.Signature.Verify(sharedutils.ToSharedAddress(w), sigb) } var _ storagemarket.StorageClientNode = &ClientNodeAdapter{} diff --git a/storagemarketadapter/provider_adapter.go b/storagemarketadapter/provider_adapter.go index 0691bee36..0db1fde5e 100644 --- a/storagemarketadapter/provider_adapter.go +++ b/storagemarketadapter/provider_adapter.go @@ -6,19 +6,23 @@ import ( "bytes" "context" + "github.com/filecoin-project/lotus/lib/sharedutils" + "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" unixfile "github.com/ipfs/go-unixfs/file" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/padreader" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage/sectorblocks" - "github.com/filecoin-project/lotus/storagemarket" ) var log = logging.Logger("provideradapter") @@ -43,13 +47,17 @@ func NewProviderNodeAdapter(dag dtypes.StagingDAG, secb *sectorblocks.SectorBloc func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemarket.MinerDeal) (storagemarket.DealID, cid.Cid, error) { log.Info("publishing deal") - worker, err := n.StateMinerWorker(ctx, deal.Proposal.Provider, nil) + worker, err := n.StateMinerWorker(ctx, sharedutils.FromSharedAddress(deal.Proposal.Provider), nil) if err != nil { return 0, cid.Undef, err } + localProposal, err := FromSharedStorageDealProposal(&deal.Proposal) + if err != nil { + return 0, cid.Undef, err + } params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{ - Deals: []actors.StorageDealProposal{deal.Proposal}, + Deals: []actors.StorageDealProposal{*localProposal}, }) if err != nil { @@ -124,17 +132,18 @@ func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagema return sectorID, nil } -func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr address.Address) ([]actors.OnChainDeal, error) { +func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr address.Address) ([]storagemarket.StorageDeal, error) { allDeals, err := n.StateMarketDeals(ctx, nil) if err != nil { return nil, err } - var out []actors.OnChainDeal + var out []storagemarket.StorageDeal for _, deal := range allDeals { - if deal.Provider == addr { - out = append(out, deal) + sharedDeal := FromOnChainDeal(deal) + if sharedDeal.Provider == addr { + out = append(out, sharedDeal) } } @@ -142,15 +151,20 @@ func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr addres } func (n *ProviderNodeAdapter) GetMinerWorker(ctx context.Context, miner address.Address) (address.Address, error) { - return n.StateMinerWorker(ctx, miner, nil) + addr, err := n.StateMinerWorker(ctx, sharedutils.FromSharedAddress(miner), nil) + return sharedutils.ToSharedAddress(addr), err } -func (n *ProviderNodeAdapter) SignBytes(ctx context.Context, signer address.Address, b []byte) (*types.Signature, error) { - return n.WalletSign(ctx, signer, b) +func (n *ProviderNodeAdapter) SignBytes(ctx context.Context, signer address.Address, b []byte) (*sharedtypes.Signature, error) { + localSignature, err := n.WalletSign(ctx, sharedutils.FromSharedAddress(signer), b) + if err != nil { + return nil, err + } + return sharedutils.ToSharedSignature(localSignature) } -func (n *ProviderNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amt storagemarket.TokenAmount) error { - return n.MarketEnsureAvailable(ctx, addr, types.BigInt(amt)) +func (n *ProviderNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amt tokenamount.TokenAmount) error { + return n.MarketEnsureAvailable(ctx, sharedutils.FromSharedAddress(addr), sharedutils.FromSharedTokenAmount(amt)) } func (n *ProviderNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarket.StateKey, error) { @@ -158,12 +172,12 @@ func (n *ProviderNodeAdapter) MostRecentStateId(ctx context.Context) (storagemar } // Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. -func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount storagemarket.TokenAmount) error { +func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { // (Provider Node API) smsg, err := n.MpoolPushMessage(ctx, &types.Message{ To: actors.StorageMarketAddress, - From: addr, - Value: types.BigInt(amount), + From: sharedutils.FromSharedAddress(addr), + Value: sharedutils.FromSharedTokenAmount(amount), GasPrice: types.NewInt(0), GasLimit: types.NewInt(1000000), Method: actors.SMAMethods.AddBalance, @@ -185,12 +199,12 @@ func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address } func (n *ProviderNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { - bal, err := n.StateMarketBalance(ctx, addr, nil) + bal, err := n.StateMarketBalance(ctx, sharedutils.FromSharedAddress(addr), nil) if err != nil { return storagemarket.Balance{}, err } - return bal, nil + return ToSharedBalance(bal), nil } var _ storagemarket.StorageProviderNode = &ProviderNodeAdapter{} diff --git a/storagemarketadapter/utilities.go b/storagemarketadapter/utilities.go new file mode 100644 index 000000000..9b815180f --- /dev/null +++ b/storagemarketadapter/utilities.go @@ -0,0 +1,79 @@ +package storagemarketadapter + +import ( + "bytes" + + "github.com/filecoin-project/go-fil-components/storagemarket" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/lib/sharedutils" + peer "github.com/libp2p/go-libp2p-peer" +) + +func NewStorageProviderInfo(address address.Address, miner address.Address, sectorSize uint64, peer peer.ID) storagemarket.StorageProviderInfo { + return storagemarket.StorageProviderInfo{ + Address: sharedutils.ToSharedAddress(address), + Worker: sharedutils.ToSharedAddress(miner), + SectorSize: sectorSize, + PeerID: peer, + } +} + +func FromOnChainDeal(deal actors.OnChainDeal) storagemarket.StorageDeal { + return storagemarket.StorageDeal{ + PieceRef: deal.PieceRef, + PieceSize: deal.PieceSize, + Client: sharedutils.ToSharedAddress(deal.Client), + Provider: sharedutils.ToSharedAddress(deal.Provider), + StoragePricePerEpoch: sharedutils.ToSharedTokenAmount(deal.StoragePricePerEpoch), + StorageCollateral: sharedutils.ToSharedTokenAmount(deal.StorageCollateral), + ActivationEpoch: deal.ActivationEpoch, + } +} + +func ToOnChainDeal(deal storagemarket.StorageDeal) actors.OnChainDeal { + return actors.OnChainDeal{ + PieceRef: deal.PieceRef, + PieceSize: deal.PieceSize, + Client: sharedutils.FromSharedAddress(deal.Client), + Provider: sharedutils.FromSharedAddress(deal.Provider), + StoragePricePerEpoch: sharedutils.FromSharedTokenAmount(deal.StoragePricePerEpoch), + StorageCollateral: sharedutils.FromSharedTokenAmount(deal.StorageCollateral), + ActivationEpoch: deal.ActivationEpoch, + } +} + +func ToSharedBalance(balance actors.StorageParticipantBalance) storagemarket.Balance { + return storagemarket.Balance{ + Locked: sharedutils.ToSharedTokenAmount(balance.Locked), + Available: sharedutils.ToSharedTokenAmount(balance.Available), + } +} + +func ToSharedStorageDealProposal(proposal *actors.StorageDealProposal) (*storagemarket.StorageDealProposal, error) { + var encoded bytes.Buffer + err := proposal.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out storagemarket.StorageDealProposal + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSharedStorageDealProposal(proposal *storagemarket.StorageDealProposal) (*actors.StorageDealProposal, error) { + var encoded bytes.Buffer + err := proposal.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out actors.StorageDealProposal + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} From 353f93fddf1ad95ab4170a9a5324028495d93dfe Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Tue, 17 Dec 2019 09:44:26 -0800 Subject: [PATCH 4/6] fix(storagemarketadapter): minor fix for test --- storagemarketadapter/client_adapter.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/storagemarketadapter/client_adapter.go b/storagemarketadapter/client_adapter.go index 90e764da9..8984dd882 100644 --- a/storagemarketadapter/client_adapter.go +++ b/storagemarketadapter/client_adapter.go @@ -304,7 +304,16 @@ func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add if err != nil { return err } - return api.SignWith(ctx, n.Wallet.Sign, sharedutils.FromSharedAddress(signer), localProposal) + err = api.SignWith(ctx, n.Wallet.Sign, sharedutils.FromSharedAddress(signer), localProposal) + if err != nil { + return err + } + signature, err := sharedutils.ToSharedSignature(localProposal.ProposerSignature) + if err != nil { + return err + } + proposal.ProposerSignature = signature + return nil } func (n *ClientNodeAdapter) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) { From 56653eeca6d4619ddaaf4cf111b2aeb05144da8d Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 10 Jan 2020 10:01:48 -0800 Subject: [PATCH 5/6] fix(storagemarket): fix compile errors --- go.sum | 1 + node/impl/client/client.go | 17 ++++++++--------- node/modules/services.go | 2 +- node/modules/storageminer.go | 6 +----- storagemarketadapter/client_adapter.go | 22 ++++++++++++---------- storagemarketadapter/provider_adapter.go | 14 +++++++------- storagemarketadapter/utilities.go | 17 +++++++++-------- 7 files changed, 39 insertions(+), 40 deletions(-) diff --git a/go.sum b/go.sum index 0e14ed7b1..b6e36c0bd 100644 --- a/go.sum +++ b/go.sum @@ -113,6 +113,7 @@ github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42 h1:7OW2AiWtwxwtYnC16CiXZ/idQ7w3W7W1hhX+N4YSAyQ= github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42/go.mod h1:Q5fvJGMISyUIna19DpoqiqTas4L9RVD7w3Udv7uCrTo= +github.com/filecoin-project/go-fil-markets v0.0.0-20200110172620-7ac5a8af146a h1:mLz0GBjx8nlztlo2sZ6w1JbuyN7EX7zoCYQBblrgoBM= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 70aff2b0c..12eaf117c 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -36,7 +36,6 @@ import ( "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/retrievaladapter" "github.com/filecoin-project/lotus/storagemarketadapter" ) @@ -79,7 +78,7 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, addr address.Ad providerInfo := storagemarketadapter.NewStorageProviderInfo(miner, mw, 0, pid) result, err := a.SMDealClient.ProposeStorageDeal( ctx, - sharedutils.ToSharedAddress(addr), + addr, &providerInfo, data, storagemarket.Epoch(math.MaxUint64), @@ -105,7 +104,7 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { out[k] = api.DealInfo{ ProposalCid: v.ProposalCid, State: v.State, - Provider: sharedutils.FromSharedAddress(v.Proposal.Provider), + Provider: v.Proposal.Provider, PieceRef: v.Proposal.PieceRef, Size: v.Proposal.PieceSize, @@ -127,7 +126,7 @@ func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo, return &api.DealInfo{ ProposalCid: v.ProposalCid, State: v.State, - Provider: sharedutils.FromSharedAddress(v.Proposal.Provider), + Provider: v.Proposal.Provider, PieceRef: v.Proposal.PieceRef, Size: v.Proposal.PieceSize, PricePerEpoch: sharedutils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch), @@ -159,13 +158,13 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffe for k, p := range peers { queryResponse, err := a.Retrieval.Query(ctx, p, root.Bytes(), retrievalmarket.QueryParams{}) if err != nil { - out[k] = api.QueryOffer{Err: err.Error(), Miner: retrievaladapter.FromSharedAddress(p.Address), MinerPeerID: p.ID} + out[k] = api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} } else { out[k] = api.QueryOffer{ Root: root, Size: queryResponse.Size, - MinPrice: retrievaladapter.FromSharedTokenAmount(queryResponse.PieceRetrievalPrice()), - Miner: retrievaladapter.FromSharedAddress(p.Address), // TODO: check + MinPrice: sharedutils.FromSharedTokenAmount(queryResponse.PieceRetrievalPrice()), + Miner: p.Address, // TODO: check MinerPeerID: p.ID, } } @@ -297,8 +296,8 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path retrievalmarket.NewParamsV0(types.BigDiv(order.Total, types.NewInt(order.Size)).Int, 0, 0), sharedutils.ToSharedTokenAmount(order.Total), order.MinerPeerID, - retrievaladapter.ToSharedAddress(order.Client), - retrievaladapter.ToSharedAddress(order.Miner)) + order.Client, + order.Miner) select { case <-ctx.Done(): return xerrors.New("Retrieval Timed Out") diff --git a/node/modules/services.go b/node/modules/services.go index e12b9a310..33f2733b5 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -10,6 +10,7 @@ import ( "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/blocksync" "github.com/filecoin-project/lotus/chain/messagepool" @@ -18,7 +19,6 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/peermgr" - "github.com/filecoin-project/go-fil-components/storagemarket" ) func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello.Service) { diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 1ddbb433a..4909a8e67 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -29,21 +29,17 @@ import ( retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/address" - "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/gen" - "github.com/filecoin-project/lotus/lib/sectorbuilder" - "github.com/filecoin-project/lotus/lib/statestore" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/go-fil-components/storagemarket" "github.com/filecoin-project/lotus/retrievaladapter" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" diff --git a/storagemarketadapter/client_adapter.go b/storagemarketadapter/client_adapter.go index 8984dd882..d517c726b 100644 --- a/storagemarketadapter/client_adapter.go +++ b/storagemarketadapter/client_adapter.go @@ -5,12 +5,14 @@ package storagemarketadapter import ( "bytes" "context" + "github.com/filecoin-project/lotus/lib/sharedutils" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" + cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-fil-markets/shared/tokenamount" sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/api" @@ -115,7 +117,7 @@ func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, // (Provider Node API) smsg, err := n.MpoolPushMessage(ctx, &types.Message{ To: actors.StorageMarketAddress, - From: sharedutils.FromSharedAddress(addr), + From: addr, Value: sharedutils.FromSharedTokenAmount(amount), GasPrice: types.NewInt(0), GasLimit: types.NewInt(1000000), @@ -138,11 +140,11 @@ func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, } func (n *ClientNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { - return n.fm.EnsureAvailable(ctx, sharedutils.FromSharedAddress(addr), sharedutils.FromSharedTokenAmount(amount)) + return n.fm.EnsureAvailable(ctx, addr, sharedutils.FromSharedTokenAmount(amount)) } func (n *ClientNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { - bal, err := n.StateMarketBalance(ctx, sharedutils.FromSharedAddress(addr), nil) + bal, err := n.StateMarketBalance(ctx, addr, nil) if err != nil { return storagemarket.Balance{}, err } @@ -160,7 +162,7 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor return 0, xerrors.Errorf("getting deal pubsish message: %w", err) } - pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, sharedutils.FromSharedAddress(deal.Proposal.Provider)) + pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider) if err != nil { return 0, xerrors.Errorf("getting miner worker failed: %w", err) } @@ -268,7 +270,7 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider } matchEvent := func(msg *types.Message) (bool, error) { - if msg.To != sharedutils.FromSharedAddress(provider) { + if msg.To != provider { return false, nil } @@ -304,7 +306,7 @@ func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add if err != nil { return err } - err = api.SignWith(ctx, n.Wallet.Sign, sharedutils.FromSharedAddress(signer), localProposal) + err = api.SignWith(ctx, n.Wallet.Sign, signer, localProposal) if err != nil { return err } @@ -318,13 +320,13 @@ func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add func (n *ClientNodeAdapter) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) { addr, err := n.Wallet.GetDefault() - return sharedutils.ToSharedAddress(addr), err + return addr, err } func (n *ClientNodeAdapter) ValidateAskSignature(ask *sharedtypes.SignedStorageAsk) error { tss := n.cs.GetHeaviestTipSet().ParentState() - w, err := stmgr.GetMinerWorkerRaw(context.TODO(), n.StateManager, tss, sharedutils.FromSharedAddress(ask.Ask.Miner)) + w, err := stmgr.GetMinerWorkerRaw(context.TODO(), n.StateManager, tss, ask.Ask.Miner) if err != nil { return xerrors.Errorf("failed to get worker for miner in ask", err) } @@ -334,7 +336,7 @@ func (n *ClientNodeAdapter) ValidateAskSignature(ask *sharedtypes.SignedStorageA return xerrors.Errorf("failed to re-serialize ask") } - return ask.Signature.Verify(sharedutils.ToSharedAddress(w), sigb) + return ask.Signature.Verify(w, sigb) } var _ storagemarket.StorageClientNode = &ClientNodeAdapter{} diff --git a/storagemarketadapter/provider_adapter.go b/storagemarketadapter/provider_adapter.go index 0db1fde5e..a0df86096 100644 --- a/storagemarketadapter/provider_adapter.go +++ b/storagemarketadapter/provider_adapter.go @@ -47,7 +47,7 @@ func NewProviderNodeAdapter(dag dtypes.StagingDAG, secb *sectorblocks.SectorBloc func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemarket.MinerDeal) (storagemarket.DealID, cid.Cid, error) { log.Info("publishing deal") - worker, err := n.StateMinerWorker(ctx, sharedutils.FromSharedAddress(deal.Proposal.Provider), nil) + worker, err := n.StateMinerWorker(ctx, deal.Proposal.Provider, nil) if err != nil { return 0, cid.Undef, err } @@ -151,12 +151,12 @@ func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr addres } func (n *ProviderNodeAdapter) GetMinerWorker(ctx context.Context, miner address.Address) (address.Address, error) { - addr, err := n.StateMinerWorker(ctx, sharedutils.FromSharedAddress(miner), nil) - return sharedutils.ToSharedAddress(addr), err + addr, err := n.StateMinerWorker(ctx, miner, nil) + return addr, err } func (n *ProviderNodeAdapter) SignBytes(ctx context.Context, signer address.Address, b []byte) (*sharedtypes.Signature, error) { - localSignature, err := n.WalletSign(ctx, sharedutils.FromSharedAddress(signer), b) + localSignature, err := n.WalletSign(ctx, signer, b) if err != nil { return nil, err } @@ -164,7 +164,7 @@ func (n *ProviderNodeAdapter) SignBytes(ctx context.Context, signer address.Addr } func (n *ProviderNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amt tokenamount.TokenAmount) error { - return n.MarketEnsureAvailable(ctx, sharedutils.FromSharedAddress(addr), sharedutils.FromSharedTokenAmount(amt)) + return n.MarketEnsureAvailable(ctx, addr, sharedutils.FromSharedTokenAmount(amt)) } func (n *ProviderNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarket.StateKey, error) { @@ -176,7 +176,7 @@ func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address // (Provider Node API) smsg, err := n.MpoolPushMessage(ctx, &types.Message{ To: actors.StorageMarketAddress, - From: sharedutils.FromSharedAddress(addr), + From: addr, Value: sharedutils.FromSharedTokenAmount(amount), GasPrice: types.NewInt(0), GasLimit: types.NewInt(1000000), @@ -199,7 +199,7 @@ func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address } func (n *ProviderNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { - bal, err := n.StateMarketBalance(ctx, sharedutils.FromSharedAddress(addr), nil) + bal, err := n.StateMarketBalance(ctx, addr, nil) if err != nil { return storagemarket.Balance{}, err } diff --git a/storagemarketadapter/utilities.go b/storagemarketadapter/utilities.go index 9b815180f..2922f9d1a 100644 --- a/storagemarketadapter/utilities.go +++ b/storagemarketadapter/utilities.go @@ -3,17 +3,18 @@ package storagemarketadapter import ( "bytes" - "github.com/filecoin-project/go-fil-components/storagemarket" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/lib/sharedutils" peer "github.com/libp2p/go-libp2p-peer" ) func NewStorageProviderInfo(address address.Address, miner address.Address, sectorSize uint64, peer peer.ID) storagemarket.StorageProviderInfo { return storagemarket.StorageProviderInfo{ - Address: sharedutils.ToSharedAddress(address), - Worker: sharedutils.ToSharedAddress(miner), + Address: address, + Worker: miner, SectorSize: sectorSize, PeerID: peer, } @@ -23,8 +24,8 @@ func FromOnChainDeal(deal actors.OnChainDeal) storagemarket.StorageDeal { return storagemarket.StorageDeal{ PieceRef: deal.PieceRef, PieceSize: deal.PieceSize, - Client: sharedutils.ToSharedAddress(deal.Client), - Provider: sharedutils.ToSharedAddress(deal.Provider), + Client: deal.Client, + Provider: deal.Provider, StoragePricePerEpoch: sharedutils.ToSharedTokenAmount(deal.StoragePricePerEpoch), StorageCollateral: sharedutils.ToSharedTokenAmount(deal.StorageCollateral), ActivationEpoch: deal.ActivationEpoch, @@ -35,8 +36,8 @@ func ToOnChainDeal(deal storagemarket.StorageDeal) actors.OnChainDeal { return actors.OnChainDeal{ PieceRef: deal.PieceRef, PieceSize: deal.PieceSize, - Client: sharedutils.FromSharedAddress(deal.Client), - Provider: sharedutils.FromSharedAddress(deal.Provider), + Client: deal.Client, + Provider: deal.Provider, StoragePricePerEpoch: sharedutils.FromSharedTokenAmount(deal.StoragePricePerEpoch), StorageCollateral: sharedutils.FromSharedTokenAmount(deal.StorageCollateral), ActivationEpoch: deal.ActivationEpoch, From 0ce1bf706b817f25d9673f45ff3c5f41211ed46a Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 10 Jan 2020 10:21:46 -0800 Subject: [PATCH 6/6] refactor(markets): reorg file structure --- cmd/lotus-storage-miner/init.go | 8 +- go.mod | 1 - go.sum | 2 +- lib/sharedutils/converters.go | 101 ---------- .../retrievaladapter}/client.go | 9 +- .../retrievaladapter}/provider.go | 13 +- .../storageadapter/client.go | 19 +- .../storageadapter/provider.go | 17 +- markets/utils/converters.go | 174 ++++++++++++++++++ node/builder.go | 6 +- node/impl/client/client.go | 19 +- node/modules/client.go | 2 +- node/modules/storageminer.go | 2 +- storagemarketadapter/utilities.go | 80 -------- 14 files changed, 221 insertions(+), 232 deletions(-) delete mode 100644 lib/sharedutils/converters.go rename {retrievaladapter => markets/retrievaladapter}/client.go (90%) rename {retrievaladapter => markets/retrievaladapter}/provider.go (86%) rename storagemarketadapter/client_adapter.go => markets/storageadapter/client.go (94%) rename storagemarketadapter/provider_adapter.go => markets/storageadapter/provider.go (93%) create mode 100644 markets/utils/converters.go delete mode 100644 storagemarketadapter/utilities.go diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 9704eca09..4aefb004e 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -6,7 +6,6 @@ import ( "crypto/rand" "encoding/json" "fmt" - "github.com/filecoin-project/lotus/storagemarketadapter" "io/ioutil" "os" "path/filepath" @@ -22,16 +21,17 @@ import ( "gopkg.in/urfave/cli.v2" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-sectorbuilder" + cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-fil-markets/storagemarket" deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" + "github.com/filecoin-project/go-sectorbuilder" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -283,7 +283,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin dealKey := datastore.NewKey(deals.ProviderDsPrefix).ChildString(proposalCid.String()) - proposal, err := storagemarketadapter.ToSharedStorageDealProposal(§or.Deal) + proposal, err := utils.ToSharedStorageDealProposal(§or.Deal) if err != nil { return err } diff --git a/go.mod b/go.mod index ee48d7fe5..b2a08de0d 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,6 @@ require ( github.com/ipfs/go-merkledag v0.2.4 github.com/ipfs/go-path v0.0.7 github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb - github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 github.com/libp2p/go-libp2p v0.4.2 github.com/libp2p/go-libp2p-circuit v0.1.4 github.com/libp2p/go-libp2p-connmgr v0.1.0 diff --git a/go.sum b/go.sum index b6e36c0bd..a15c42b49 100644 --- a/go.sum +++ b/go.sum @@ -113,7 +113,6 @@ github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42 h1:7OW2AiWtwxwtYnC16CiXZ/idQ7w3W7W1hhX+N4YSAyQ= github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42/go.mod h1:Q5fvJGMISyUIna19DpoqiqTas4L9RVD7w3Udv7uCrTo= -github.com/filecoin-project/go-fil-markets v0.0.0-20200110172620-7ac5a8af146a h1:mLz0GBjx8nlztlo2sZ6w1JbuyN7EX7zoCYQBblrgoBM= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= @@ -700,6 +699,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= diff --git a/lib/sharedutils/converters.go b/lib/sharedutils/converters.go deleted file mode 100644 index 58a5b44f9..000000000 --- a/lib/sharedutils/converters.go +++ /dev/null @@ -1,101 +0,0 @@ -package sharedutils - -import ( - "bytes" - sharedamount "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" - "github.com/filecoin-project/lotus/chain/types" -) - - -func FromSharedTokenAmount(in sharedamount.TokenAmount) types.BigInt { - return types.BigInt{Int: in.Int} -} - -func ToSharedTokenAmount(in types.BigInt) sharedamount.TokenAmount { - return sharedamount.TokenAmount{Int: in.Int} -} - -func ToSharedSignedVoucher(in *types.SignedVoucher) (*sharedtypes.SignedVoucher, error) { - var encoded bytes.Buffer - err := in.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out sharedtypes.SignedVoucher - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} - -func FromSharedSignedVoucher(in *sharedtypes.SignedVoucher) (*types.SignedVoucher, error) { - var encoded bytes.Buffer - err := in.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out types.SignedVoucher - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} - -func ToSharedSignature(in *types.Signature) (*sharedtypes.Signature, error) { - var encoded bytes.Buffer - err := in.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out sharedtypes.Signature - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} - -func FromSharedSignature(in *sharedtypes.Signature) (*types.Signature, error) { - var encoded bytes.Buffer - err := in.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out types.Signature - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} - -func ToSharedStorageAsk(in *types.SignedStorageAsk) (*sharedtypes.SignedStorageAsk, error) { - var encoded bytes.Buffer - err := in.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out sharedtypes.SignedStorageAsk - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} - -func FromSignedStorageAsk(in *sharedtypes.SignedStorageAsk) (*types.SignedStorageAsk, error) { - var encoded bytes.Buffer - err := in.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out types.SignedStorageAsk - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} diff --git a/retrievaladapter/client.go b/markets/retrievaladapter/client.go similarity index 90% rename from retrievaladapter/client.go rename to markets/retrievaladapter/client.go index cd5f819e5..676398014 100644 --- a/retrievaladapter/client.go +++ b/markets/retrievaladapter/client.go @@ -2,13 +2,12 @@ package retrievaladapter import ( "context" - "github.com/filecoin-project/lotus/lib/sharedutils" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" retrievaltoken "github.com/filecoin-project/go-fil-markets/shared/tokenamount" retrievaltypes "github.com/filecoin-project/go-fil-markets/shared/types" - + "github.com/filecoin-project/lotus/markets/utils" payapi "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/paych" ) @@ -27,7 +26,7 @@ func NewRetrievalClientNode(pmgr *paych.Manager, payapi payapi.PaychAPI) retriev // GetOrCreatePaymentChannel sets up a new payment channel if one does not exist // between a client and a miner and insures the client has the given amount of funds available in the channel func (rcn *retrievalClientNode) GetOrCreatePaymentChannel(ctx context.Context, clientAddress address.Address, minerAddress address.Address, clientFundsAvailable retrievaltoken.TokenAmount) (address.Address, error) { - paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, sharedutils.FromSharedTokenAmount(clientFundsAvailable)) + paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, utils.FromSharedTokenAmount(clientFundsAvailable)) return paych, err } @@ -42,9 +41,9 @@ func (rcn *retrievalClientNode) AllocateLane(paymentChannel address.Address) (ui // given payment channel so that all the payment vouchers in the lane add up // to the given amount (so the payment voucher will be for the difference) func (rcn *retrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount retrievaltoken.TokenAmount, lane uint64) (*retrievaltypes.SignedVoucher, error) { - voucher, err := rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, sharedutils.FromSharedTokenAmount(amount), lane) + voucher, err := rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, utils.FromSharedTokenAmount(amount), lane) if err != nil { return nil, err } - return sharedutils.ToSharedSignedVoucher(voucher) + return utils.ToSharedSignedVoucher(voucher) } diff --git a/retrievaladapter/provider.go b/markets/retrievaladapter/provider.go similarity index 86% rename from retrievaladapter/provider.go rename to markets/retrievaladapter/provider.go index d2499757b..82c765123 100644 --- a/retrievaladapter/provider.go +++ b/markets/retrievaladapter/provider.go @@ -2,16 +2,17 @@ package retrievaladapter import ( "context" - "github.com/filecoin-project/lotus/lib/sharedutils" + + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" retrievaltoken "github.com/filecoin-project/go-fil-markets/shared/tokenamount" retrievaltypes "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/storage/sectorblocks" - "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" ) type retrievalProviderNode struct { @@ -38,10 +39,10 @@ func (rpn *retrievalProviderNode) SealedBlockstore(approveUnseal func() error) b } func (rpn *retrievalProviderNode) SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *retrievaltypes.SignedVoucher, proof []byte, expectedAmount retrievaltoken.TokenAmount) (retrievaltoken.TokenAmount, error) { - localVoucher, err := sharedutils.FromSharedSignedVoucher(voucher) + localVoucher, err := utils.FromSharedSignedVoucher(voucher) if err != nil { return retrievaltoken.FromInt(0), err } - added, err := rpn.full.PaychVoucherAdd(ctx, paymentChannel, localVoucher, proof, sharedutils.FromSharedTokenAmount(expectedAmount)) - return sharedutils.ToSharedTokenAmount(added), err + added, err := rpn.full.PaychVoucherAdd(ctx, paymentChannel, localVoucher, proof, utils.FromSharedTokenAmount(expectedAmount)) + return utils.ToSharedTokenAmount(added), err } diff --git a/storagemarketadapter/client_adapter.go b/markets/storageadapter/client.go similarity index 94% rename from storagemarketadapter/client_adapter.go rename to markets/storageadapter/client.go index d517c726b..519e12e57 100644 --- a/storagemarketadapter/client_adapter.go +++ b/markets/storageadapter/client.go @@ -1,4 +1,4 @@ -package storagemarketadapter +package storageadapter // this file implements storagemarket.StorageClientNode @@ -6,8 +6,6 @@ import ( "bytes" "context" - "github.com/filecoin-project/lotus/lib/sharedutils" - "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -23,6 +21,7 @@ import ( "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/node/impl/full" ) @@ -83,7 +82,7 @@ func (n *ClientNodeAdapter) ListStorageProviders(ctx context.Context) ([]*storag if err != nil { return nil, err } - storageProviderInfo := NewStorageProviderInfo(addr, workerAddr, sectorSize, peerId) + storageProviderInfo := utils.NewStorageProviderInfo(addr, workerAddr, sectorSize, peerId) out = append(out, &storageProviderInfo) } @@ -99,7 +98,7 @@ func (n *ClientNodeAdapter) ListClientDeals(ctx context.Context, addr address.Ad var out []storagemarket.StorageDeal for _, deal := range allDeals { - storageDeal := FromOnChainDeal(deal) + storageDeal := utils.FromOnChainDeal(deal) if storageDeal.Client == addr { out = append(out, storageDeal) } @@ -118,7 +117,7 @@ func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, smsg, err := n.MpoolPushMessage(ctx, &types.Message{ To: actors.StorageMarketAddress, From: addr, - Value: sharedutils.FromSharedTokenAmount(amount), + Value: utils.FromSharedTokenAmount(amount), GasPrice: types.NewInt(0), GasLimit: types.NewInt(1000000), Method: actors.SMAMethods.AddBalance, @@ -140,7 +139,7 @@ func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, } func (n *ClientNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { - return n.fm.EnsureAvailable(ctx, addr, sharedutils.FromSharedTokenAmount(amount)) + return n.fm.EnsureAvailable(ctx, addr, utils.FromSharedTokenAmount(amount)) } func (n *ClientNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { @@ -149,7 +148,7 @@ func (n *ClientNodeAdapter) GetBalance(ctx context.Context, addr address.Address return storagemarket.Balance{}, err } - return ToSharedBalance(bal), nil + return utils.ToSharedBalance(bal), nil } // ValidatePublishedDeal validates that the provided deal has appeared on chain and references the same ClientDeal @@ -302,7 +301,7 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider } func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Address, proposal *storagemarket.StorageDealProposal) error { - localProposal, err := FromSharedStorageDealProposal(proposal) + localProposal, err := utils.FromSharedStorageDealProposal(proposal) if err != nil { return err } @@ -310,7 +309,7 @@ func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add if err != nil { return err } - signature, err := sharedutils.ToSharedSignature(localProposal.ProposerSignature) + signature, err := utils.ToSharedSignature(localProposal.ProposerSignature) if err != nil { return err } diff --git a/storagemarketadapter/provider_adapter.go b/markets/storageadapter/provider.go similarity index 93% rename from storagemarketadapter/provider_adapter.go rename to markets/storageadapter/provider.go index a0df86096..abe7f66eb 100644 --- a/storagemarketadapter/provider_adapter.go +++ b/markets/storageadapter/provider.go @@ -1,4 +1,4 @@ -package storagemarketadapter +package storageadapter // this file implements storagemarket.StorageProviderNode @@ -6,8 +6,6 @@ import ( "bytes" "context" - "github.com/filecoin-project/lotus/lib/sharedutils" - "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" unixfile "github.com/ipfs/go-unixfs/file" @@ -21,6 +19,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/padreader" + "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -52,7 +51,7 @@ func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemark return 0, cid.Undef, err } - localProposal, err := FromSharedStorageDealProposal(&deal.Proposal) + localProposal, err := utils.FromSharedStorageDealProposal(&deal.Proposal) if err != nil { return 0, cid.Undef, err } @@ -141,7 +140,7 @@ func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr addres var out []storagemarket.StorageDeal for _, deal := range allDeals { - sharedDeal := FromOnChainDeal(deal) + sharedDeal := utils.FromOnChainDeal(deal) if sharedDeal.Provider == addr { out = append(out, sharedDeal) } @@ -160,11 +159,11 @@ func (n *ProviderNodeAdapter) SignBytes(ctx context.Context, signer address.Addr if err != nil { return nil, err } - return sharedutils.ToSharedSignature(localSignature) + return utils.ToSharedSignature(localSignature) } func (n *ProviderNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amt tokenamount.TokenAmount) error { - return n.MarketEnsureAvailable(ctx, addr, sharedutils.FromSharedTokenAmount(amt)) + return n.MarketEnsureAvailable(ctx, addr, utils.FromSharedTokenAmount(amt)) } func (n *ProviderNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarket.StateKey, error) { @@ -177,7 +176,7 @@ func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address smsg, err := n.MpoolPushMessage(ctx, &types.Message{ To: actors.StorageMarketAddress, From: addr, - Value: sharedutils.FromSharedTokenAmount(amount), + Value: utils.FromSharedTokenAmount(amount), GasPrice: types.NewInt(0), GasLimit: types.NewInt(1000000), Method: actors.SMAMethods.AddBalance, @@ -204,7 +203,7 @@ func (n *ProviderNodeAdapter) GetBalance(ctx context.Context, addr address.Addre return storagemarket.Balance{}, err } - return ToSharedBalance(bal), nil + return utils.ToSharedBalance(bal), nil } var _ storagemarket.StorageProviderNode = &ProviderNodeAdapter{} diff --git a/markets/utils/converters.go b/markets/utils/converters.go new file mode 100644 index 000000000..58080896f --- /dev/null +++ b/markets/utils/converters.go @@ -0,0 +1,174 @@ +package utils + +import ( + "bytes" + + peer "github.com/libp2p/go-libp2p-peer" + + "github.com/filecoin-project/go-address" + sharedamount "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" +) + +func FromSharedTokenAmount(in sharedamount.TokenAmount) types.BigInt { + return types.BigInt{Int: in.Int} +} + +func ToSharedTokenAmount(in types.BigInt) sharedamount.TokenAmount { + return sharedamount.TokenAmount{Int: in.Int} +} + +func ToSharedSignedVoucher(in *types.SignedVoucher) (*sharedtypes.SignedVoucher, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out sharedtypes.SignedVoucher + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSharedSignedVoucher(in *sharedtypes.SignedVoucher) (*types.SignedVoucher, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out types.SignedVoucher + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func ToSharedSignature(in *types.Signature) (*sharedtypes.Signature, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out sharedtypes.Signature + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSharedSignature(in *sharedtypes.Signature) (*types.Signature, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out types.Signature + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func ToSharedStorageAsk(in *types.SignedStorageAsk) (*sharedtypes.SignedStorageAsk, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out sharedtypes.SignedStorageAsk + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSignedStorageAsk(in *sharedtypes.SignedStorageAsk) (*types.SignedStorageAsk, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out types.SignedStorageAsk + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func NewStorageProviderInfo(address address.Address, miner address.Address, sectorSize uint64, peer peer.ID) storagemarket.StorageProviderInfo { + return storagemarket.StorageProviderInfo{ + Address: address, + Worker: miner, + SectorSize: sectorSize, + PeerID: peer, + } +} + +func FromOnChainDeal(deal actors.OnChainDeal) storagemarket.StorageDeal { + return storagemarket.StorageDeal{ + PieceRef: deal.PieceRef, + PieceSize: deal.PieceSize, + Client: deal.Client, + Provider: deal.Provider, + StoragePricePerEpoch: ToSharedTokenAmount(deal.StoragePricePerEpoch), + StorageCollateral: ToSharedTokenAmount(deal.StorageCollateral), + ActivationEpoch: deal.ActivationEpoch, + } +} + +func ToOnChainDeal(deal storagemarket.StorageDeal) actors.OnChainDeal { + return actors.OnChainDeal{ + PieceRef: deal.PieceRef, + PieceSize: deal.PieceSize, + Client: deal.Client, + Provider: deal.Provider, + StoragePricePerEpoch: FromSharedTokenAmount(deal.StoragePricePerEpoch), + StorageCollateral: FromSharedTokenAmount(deal.StorageCollateral), + ActivationEpoch: deal.ActivationEpoch, + } +} + +func ToSharedBalance(balance actors.StorageParticipantBalance) storagemarket.Balance { + return storagemarket.Balance{ + Locked: ToSharedTokenAmount(balance.Locked), + Available: ToSharedTokenAmount(balance.Available), + } +} + +func ToSharedStorageDealProposal(proposal *actors.StorageDealProposal) (*storagemarket.StorageDealProposal, error) { + var encoded bytes.Buffer + err := proposal.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out storagemarket.StorageDealProposal + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSharedStorageDealProposal(proposal *storagemarket.StorageDealProposal) (*actors.StorageDealProposal, error) { + var encoded bytes.Buffer + err := proposal.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out actors.StorageDealProposal + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} diff --git a/node/builder.go b/node/builder.go index 9a656563c..178f973e5 100644 --- a/node/builder.go +++ b/node/builder.go @@ -34,6 +34,7 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/hello" @@ -48,7 +49,6 @@ import ( "github.com/filecoin-project/lotus/peermgr" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" - "github.com/filecoin-project/lotus/storagemarketadapter" ) // special is a type used to give keys to modules which @@ -230,7 +230,7 @@ func Online() Option { Override(new(dtypes.ClientDataTransfer), modules.NewClientDAGServiceDataTransfer), Override(new(*deals.ClientRequestValidator), modules.NewClientRequestValidator), Override(new(storagemarket.StorageClient), modules.StorageClient), - Override(new(storagemarket.StorageClientNode), storagemarketadapter.NewClientNodeAdapter), + Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter), Override(RegisterClientValidatorKey, modules.RegisterClientValidator), Override(RunDealClientKey, modules.RunDealClient), @@ -254,7 +254,7 @@ func Online() Option { Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), Override(new(*deals.ProviderRequestValidator), modules.NewProviderRequestValidator), Override(new(storagemarket.StorageProvider), modules.StorageProvider), - Override(new(storagemarket.StorageProviderNode), storagemarketadapter.NewProviderNodeAdapter), + Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter), Override(RegisterProviderValidatorKey, modules.RegisterProviderValidator), Override(HandleRetrievalKey, modules.HandleRetrieval), Override(GetParamsKey, modules.GetParams), diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 12eaf117c..5623f4ae1 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -32,11 +32,10 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sharedutils" + "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storagemarketadapter" ) type API struct { @@ -75,7 +74,7 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, addr address.Ad if err != nil { return nil, xerrors.Errorf("failed getting miner worker: %w", err) } - providerInfo := storagemarketadapter.NewStorageProviderInfo(miner, mw, 0, pid) + providerInfo := utils.NewStorageProviderInfo(miner, mw, 0, pid) result, err := a.SMDealClient.ProposeStorageDeal( ctx, addr, @@ -83,7 +82,7 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, addr address.Ad data, storagemarket.Epoch(math.MaxUint64), storagemarket.Epoch(blocksDuration), - sharedutils.ToSharedTokenAmount(epochPrice), + utils.ToSharedTokenAmount(epochPrice), tokenamount.Empty) if err != nil { @@ -109,7 +108,7 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { PieceRef: v.Proposal.PieceRef, Size: v.Proposal.PieceSize, - PricePerEpoch: sharedutils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch), + PricePerEpoch: utils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch), Duration: v.Proposal.Duration, } } @@ -129,7 +128,7 @@ func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo, Provider: v.Proposal.Provider, PieceRef: v.Proposal.PieceRef, Size: v.Proposal.PieceSize, - PricePerEpoch: sharedutils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch), + PricePerEpoch: utils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch), Duration: v.Proposal.Duration, }, nil } @@ -163,7 +162,7 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffe out[k] = api.QueryOffer{ Root: root, Size: queryResponse.Size, - MinPrice: sharedutils.FromSharedTokenAmount(queryResponse.PieceRetrievalPrice()), + MinPrice: utils.FromSharedTokenAmount(queryResponse.PieceRetrievalPrice()), Miner: p.Address, // TODO: check MinerPeerID: p.ID, } @@ -294,7 +293,7 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path ctx, order.Root.Bytes(), retrievalmarket.NewParamsV0(types.BigDiv(order.Total, types.NewInt(order.Size)).Int, 0, 0), - sharedutils.ToSharedTokenAmount(order.Total), + utils.ToSharedTokenAmount(order.Total), order.MinerPeerID, order.Client, order.Miner) @@ -321,10 +320,10 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path } func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) { - info := storagemarketadapter.NewStorageProviderInfo(miner, address.Undef, 0, p) + info := utils.NewStorageProviderInfo(miner, address.Undef, 0, p) signedAsk, err := a.SMDealClient.GetAsk(ctx, info) if err != nil { return nil, err } - return sharedutils.FromSignedStorageAsk(signedAsk) + return utils.FromSignedStorageAsk(signedAsk) } diff --git a/node/modules/client.go b/node/modules/client.go index 313672c6b..91796634d 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -5,7 +5,7 @@ import ( "path/filepath" "reflect" - "github.com/filecoin-project/lotus/retrievaladapter" + "github.com/filecoin-project/lotus/markets/retrievaladapter" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 4909a8e67..d74595012 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -40,7 +40,7 @@ import ( "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/retrievaladapter" + "github.com/filecoin-project/lotus/markets/retrievaladapter" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" ) diff --git a/storagemarketadapter/utilities.go b/storagemarketadapter/utilities.go deleted file mode 100644 index 2922f9d1a..000000000 --- a/storagemarketadapter/utilities.go +++ /dev/null @@ -1,80 +0,0 @@ -package storagemarketadapter - -import ( - "bytes" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-fil-markets/storagemarket" - - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/lib/sharedutils" - peer "github.com/libp2p/go-libp2p-peer" -) - -func NewStorageProviderInfo(address address.Address, miner address.Address, sectorSize uint64, peer peer.ID) storagemarket.StorageProviderInfo { - return storagemarket.StorageProviderInfo{ - Address: address, - Worker: miner, - SectorSize: sectorSize, - PeerID: peer, - } -} - -func FromOnChainDeal(deal actors.OnChainDeal) storagemarket.StorageDeal { - return storagemarket.StorageDeal{ - PieceRef: deal.PieceRef, - PieceSize: deal.PieceSize, - Client: deal.Client, - Provider: deal.Provider, - StoragePricePerEpoch: sharedutils.ToSharedTokenAmount(deal.StoragePricePerEpoch), - StorageCollateral: sharedutils.ToSharedTokenAmount(deal.StorageCollateral), - ActivationEpoch: deal.ActivationEpoch, - } -} - -func ToOnChainDeal(deal storagemarket.StorageDeal) actors.OnChainDeal { - return actors.OnChainDeal{ - PieceRef: deal.PieceRef, - PieceSize: deal.PieceSize, - Client: deal.Client, - Provider: deal.Provider, - StoragePricePerEpoch: sharedutils.FromSharedTokenAmount(deal.StoragePricePerEpoch), - StorageCollateral: sharedutils.FromSharedTokenAmount(deal.StorageCollateral), - ActivationEpoch: deal.ActivationEpoch, - } -} - -func ToSharedBalance(balance actors.StorageParticipantBalance) storagemarket.Balance { - return storagemarket.Balance{ - Locked: sharedutils.ToSharedTokenAmount(balance.Locked), - Available: sharedutils.ToSharedTokenAmount(balance.Available), - } -} - -func ToSharedStorageDealProposal(proposal *actors.StorageDealProposal) (*storagemarket.StorageDealProposal, error) { - var encoded bytes.Buffer - err := proposal.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out storagemarket.StorageDealProposal - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -} - -func FromSharedStorageDealProposal(proposal *storagemarket.StorageDealProposal) (*actors.StorageDealProposal, error) { - var encoded bytes.Buffer - err := proposal.MarshalCBOR(&encoded) - if err != nil { - return nil, err - } - var out actors.StorageDealProposal - err = out.UnmarshalCBOR(&encoded) - if err != nil { - return nil, err - } - return &out, nil -}