From 61e2568b8d932fa8b243b8a551d59fee325ad69a Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Tue, 29 Oct 2019 19:42:16 -0700 Subject: [PATCH 1/3] feat(datatransfer): implement and extract feat(datatransfer): setup implementation path Sets up a path to implementation, offering both the dagservice implementation and a future graphsync implement, establishes message interfaces and network layer, and isolates the datatransfer module from the app WIP using CBOR encoding for dataxfermsg * Bring cbor-gen stuff into datatransfer package * make transferRequest private struct * add transferResponse + funcs * Rename VoucherID to VoucherType * more tests passing WIP trying out some stuff * Embed request/response in message so all the interfaces work AND the CBOR unmarshaling works: this is more like the spec anyway * get rid of pb stuff all message tests passing, some others in datatransfer Some cleanup for PR Cleanup for PR, clarifying and additional comments mod tidy Respond to PR comments: * Make DataTransferRequest/Response be returned in from Net * Regenerate cbor_gen and fix the generator caller so it works better * Please the linters Fix tests Initiate push and pull requests (#536) * add issue link for data TransferID generation * comment out failing but not relevant tests * finish voucher rename from Identifier --> Type tests passing cleanup for PR remove unused fmt import in graphsync_test a better reflection send data transfer response other tests passing feat(datatransfer): milestone 2 infrastructure Setup test path for all tickets for milestone 2 responses alert subscribers when request is not accepted (#607) Graphsync response is scheduled when a valid push request is received (#625) fix(datatransfer): fix tests fix an error with read buffers in tests fix(deps): fix go.sum Feat/dt graphsync pullreqs (#627) * graphsync responses to pull requests Feat/dt initiator cleanup (#645) * ChannelID.To --> ChannelID.Initiator * We now store our peer ID (from host.ID()) so it can be used when creating ChannelIDs. * InProgressChannels returns all of impl.channels, currently just for testing * Implements go-data-transfer issue * Some assertions were changed based on the above. * Renamed some variables and added some assertions based on the new understanding * Updated SHA for graphsync module * Updated fakeGraphSync test structs to use new interfaces from new SHA above Techdebt/dt split graphsync impl receiver (#651) * Split up graphsyncImpl and graphsyncReceiver * rename graphsync to utils DTM sends data over graphsync for validated push requests (#665) * create channels when a request is received. register push request hook with graphsync. fix tests. * better NewReaders * use mutex lock around impl.channels access * fix(datatransfer): fix test uncertainty * fix a data race and also don't use random bytes in basic block which can fail * privatize 3 funcs with @hannahhoward Feat/dt gs pullrequests (#693) * Implements DTM Sends Data Over Graphsync For Validated Pull Requests * rename a field in a test struct * refactor a couple of private functions (one was refactored out of existence) Feat/dt subscribe, file Xfer round trip (#720) Implements the rest of Subscriber Is Notified When Request Completed #24: * send a graphsync message within a go func and consume responses until error or transfer is complete. * notify subscribers of results. * Rename datatransfer.Event to EventCode. * datatransfer.Event is now a struct that includes a message and a timestamp as well as the Event.Code int, formerly Event, update all uses * Add extension data to graphsync request hook, gsReq * rename sendRequest to sendDtRequest, to distinguish it from sendGsRequest, where Dt = datatransfer, Gs = graphsync * use a mutex lock for last transfer ID * obey the linter Don't respond with error in gsReqRcdHook when we can't find the datatransfer extension. (#754) * update to correct graphsync version, update tests & code to call the new graphsync hooks * getExtensionData returns an empty struct + nil if we can't find our extension * Don't respond with error when we can't find the extension. * Test for same * mod tidy minor fix to go.sum feat(datatransfer): switch to graphsync implementation Move over to real graphsync implementation of data transfer, add constructors for graphsync instances on client and miner side fix(datatransfer): Fix validators Validators were checking payload cid against commP -- which are not the same any more. Added a payloadCid to client deal to maintain the record, fixed validator logic Feat/dt extraction use go-fil-components/datatransfer (#770) * Initial commit after changing to go-fil-components/datatransfer * blow away the datatransfer dir * use go-fil-components master after its PR #1 was merged * go mod tidy use a package updates after rebase with master --- chain/deals/cbor_gen.go | 22 +++- chain/deals/client.go | 6 +- chain/deals/client_utils.go | 7 +- chain/deals/provider.go | 4 +- chain/deals/provider_utils.go | 7 +- chain/deals/request_validation_test.go | 50 +++---- chain/deals/types.go | 4 +- datatransfer/dagservice_impl.go | 78 ----------- datatransfer/types.go | 173 ------------------------- go.mod | 7 +- go.sum | 44 ++++++- node/builder.go | 4 +- node/modules/client.go | 25 +++- node/modules/dtypes/storage.go | 9 +- node/modules/storageminer.go | 36 ++++- 15 files changed, 153 insertions(+), 323 deletions(-) delete mode 100644 datatransfer/dagservice_impl.go delete mode 100644 datatransfer/types.go diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go index 2a5f1f8a9..bd56e0551 100644 --- a/chain/deals/cbor_gen.go +++ b/chain/deals/cbor_gen.go @@ -526,7 +526,7 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{135}); err != nil { + if _, err := w.Write([]byte{136}); err != nil { return err } @@ -568,6 +568,12 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { return err } + // t.PayloadCid (cid.Cid) (struct) + + if err := cbg.WriteCid(w, t.PayloadCid); err != nil { + return xerrors.Errorf("failed to write cid field t.PayloadCid: %w", err) + } + // t.PublishMessage (types.SignedMessage) (struct) if err := t.PublishMessage.MarshalCBOR(w); err != nil { return err @@ -586,7 +592,7 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 7 { + if extra != 8 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -650,6 +656,18 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.DealID = uint64(extra) + // t.PayloadCid (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PayloadCid: %w", err) + } + + t.PayloadCid = c + + } // t.PublishMessage (types.SignedMessage) (struct) { diff --git a/chain/deals/client.go b/chain/deals/client.go index 709432f63..798296135 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -23,6 +23,7 @@ import ( "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/retrieval/discovery" ) @@ -35,6 +36,7 @@ type ClientDeal struct { Miner peer.ID MinerWorker address.Address DealID uint64 + PayloadCid cid.Cid PublishMessage *types.SignedMessage @@ -244,8 +246,8 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro State: api.DealUnknown, Miner: p.MinerID, MinerWorker: p.MinerWorker, - - s: s, + PayloadCid: p.Data, + s: s, } c.incoming <- deal diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go index 944846ed0..5c875749a 100644 --- a/chain/deals/client_utils.go +++ b/chain/deals/client_utils.go @@ -1,7 +1,6 @@ package deals import ( - "bytes" "context" "runtime" @@ -13,10 +12,10 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" + "github.com/filecoin-project/go-fil-components/datatransfer" cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/datatransfer" "github.com/filecoin-project/lotus/lib/padreader" "github.com/filecoin-project/lotus/node/modules/dtypes" ) @@ -148,7 +147,7 @@ func (c *ClientRequestValidator) ValidatePull( Selector ipld.Node) error { dealVoucher, ok := voucher.(*StorageDataTransferVoucher) if !ok { - return xerrors.Errorf("voucher type %s: %w", voucher.Identifier(), ErrWrongVoucherType) + return xerrors.Errorf("voucher type %s: %w", voucher.Type(), ErrWrongVoucherType) } var deal ClientDeal @@ -159,7 +158,7 @@ func (c *ClientRequestValidator) ValidatePull( if deal.Miner != receiver { return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Miner.String(), receiver.String(), ErrWrongPeer) } - if !bytes.Equal(deal.Proposal.PieceRef, baseCid.Bytes()) { + if !deal.PayloadCid.Equals(baseCid) { return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece) } for _, state := range DataTransferStates { diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 9e45ef8b2..7c0bde0bf 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -14,11 +14,11 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-fil-components/datatransfer" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/datatransfer" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" @@ -225,7 +225,7 @@ func (p *Provider) onDataTransferEvent(event datatransfer.Event, channelState da var next api.DealState var err error var mut func(*MinerDeal) - switch event { + switch event.Code { case datatransfer.Complete: next = api.DealStaged mut = func(deal *MinerDeal) { diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go index e1fcfcf32..35614f32d 100644 --- a/chain/deals/provider_utils.go +++ b/chain/deals/provider_utils.go @@ -1,12 +1,11 @@ package deals import ( - "bytes" "context" "runtime" + "github.com/filecoin-project/go-fil-components/datatransfer" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/datatransfer" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/ipld/go-ipld-prime" @@ -163,7 +162,7 @@ func (m *ProviderRequestValidator) ValidatePush( Selector ipld.Node) error { dealVoucher, ok := voucher.(*StorageDataTransferVoucher) if !ok { - return xerrors.Errorf("voucher type %s: %w", voucher.Identifier(), ErrWrongVoucherType) + return xerrors.Errorf("voucher type %s: %w", voucher.Type(), ErrWrongVoucherType) } var deal MinerDeal @@ -175,7 +174,7 @@ func (m *ProviderRequestValidator) ValidatePush( return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Client.String(), sender.String(), ErrWrongPeer) } - if !bytes.Equal(deal.Proposal.PieceRef, baseCid.Bytes()) { + if !deal.Ref.Equals(baseCid) { return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece) } for _, state := range DataTransferStates { diff --git a/chain/deals/request_validation_test.go b/chain/deals/request_validation_test.go index 6da68d219..f6e6a0e33 100644 --- a/chain/deals/request_validation_test.go +++ b/chain/deals/request_validation_test.go @@ -35,7 +35,7 @@ func (wrongDTType) FromBytes([]byte) error { return fmt.Errorf("not implemented") } -func (wrongDTType) Identifier() string { +func (wrongDTType) Type() string { return "WrongDTTYPE" } @@ -76,6 +76,7 @@ func newClientDeal(minerID peer.ID, state api.DealState) (deals.ClientDeal, erro return deals.ClientDeal{ Proposal: newProposal, ProposalCid: proposalNd.Cid(), + PayloadCid: blockGenerator.Next().Cid(), Miner: minerID, MinerWorker: minerAddr, State: state, @@ -91,10 +92,7 @@ func newMinerDeal(clientID peer.ID, state api.DealState) (deals.MinerDeal, error if err != nil { return deals.MinerDeal{}, err } - ref, err := cid.Cast(newProposal.PieceRef) - if err != nil { - return deals.MinerDeal{}, err - } + ref := blockGenerator.Next().Cid() return deals.MinerDeal{ Proposal: newProposal, @@ -143,11 +141,8 @@ func TestClientRequestValidation(t *testing.T) { if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { t.Fatal("deal tracking failed") } - pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrWrongPeer) { + payloadCid := clientDeal.PayloadCid + if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, payloadCid, nil), deals.ErrWrongPeer) { t.Fatal("Pull should fail if miner address is incorrect") } }) @@ -171,11 +166,8 @@ func TestClientRequestValidation(t *testing.T) { if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { t.Fatal("deal tracking failed") } - pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrInacceptableDealState) { + payloadCid := clientDeal.PayloadCid + if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, payloadCid, nil), deals.ErrInacceptableDealState) { t.Fatal("Pull should fail if deal is in a state that cannot be data transferred") } }) @@ -187,11 +179,8 @@ func TestClientRequestValidation(t *testing.T) { if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { t.Fatal("deal tracking failed") } - pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil) != nil { + payloadCid := clientDeal.PayloadCid + if crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, payloadCid, nil) != nil { t.Fatal("Pull should should succeed when all parameters are correct") } }) @@ -236,11 +225,8 @@ func TestProviderRequestValidation(t *testing.T) { if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { t.Fatal("deal tracking failed") } - pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrWrongPeer) { + ref := minerDeal.Ref + if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, ref, nil), deals.ErrWrongPeer) { t.Fatal("Push should fail if miner address is incorrect") } }) @@ -264,11 +250,8 @@ func TestProviderRequestValidation(t *testing.T) { if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { t.Fatal("deal tracking failed") } - pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrInacceptableDealState) { + ref := minerDeal.Ref + if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, ref, nil), deals.ErrInacceptableDealState) { t.Fatal("Push should fail if deal is in a state that cannot be data transferred") } }) @@ -280,11 +263,8 @@ func TestProviderRequestValidation(t *testing.T) { if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { t.Fatal("deal tracking failed") } - pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil) != nil { + ref := minerDeal.Ref + if mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, ref, nil) != nil { t.Fatal("Push should should succeed when all parameters are correct") } }) diff --git a/chain/deals/types.go b/chain/deals/types.go index 8a368e80d..4ad60a4bc 100644 --- a/chain/deals/types.go +++ b/chain/deals/types.go @@ -108,7 +108,7 @@ func (dv *StorageDataTransferVoucher) FromBytes(raw []byte) error { return dv.UnmarshalCBOR(r) } -// Identifier is the unique string identifier for a StorageDataTransferVoucher -func (dv *StorageDataTransferVoucher) Identifier() string { +// Type is the unique string identifier for a StorageDataTransferVoucher +func (dv *StorageDataTransferVoucher) Type() string { return "StorageDataTransferVoucher" } diff --git a/datatransfer/dagservice_impl.go b/datatransfer/dagservice_impl.go deleted file mode 100644 index 98c15d062..000000000 --- a/datatransfer/dagservice_impl.go +++ /dev/null @@ -1,78 +0,0 @@ -package datatransfer - -import ( - "context" - "reflect" - - "github.com/ipfs/go-cid" - ipldformat "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" - ipld "github.com/ipld/go-ipld-prime" - "github.com/libp2p/go-libp2p-core/peer" - "golang.org/x/xerrors" -) - -// This file implements a VERY simple, incomplete version of the data transfer -// module that allows us to make the necessary insertions of data transfer -// functionality into the storage market -// It does not: -// -- actually validate requests -// -- support Push requests -// -- support multiple subscribers -// -- do any actual network coordination or use Graphsync - -type dagserviceImpl struct { - dag ipldformat.DAGService - subscriber Subscriber -} - -// NewDAGServiceDataTransfer returns a data transfer manager based on -// an IPLD DAGService -func NewDAGServiceDataTransfer(dag ipldformat.DAGService) Manager { - return &dagserviceImpl{dag, nil} -} - -// RegisterVoucherType registers a validator for the given voucher type -// will error if voucher type does not implement voucher -// or if there is a voucher type registered with an identical identifier -func (impl *dagserviceImpl) RegisterVoucherType(voucherType reflect.Type, validator RequestValidator) error { - return nil -} - -// open a data transfer that will send data to the recipient peer and -// transfer parts of the piece that match the selector -func (impl *dagserviceImpl) OpenPushDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, Selector ipld.Node) (ChannelID, error) { - return ChannelID{}, xerrors.Errorf("not implemented") -} - -// open a data transfer that will request data from the sending peer and -// transfer parts of the piece that match the selector -func (impl *dagserviceImpl) OpenPullDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, Selector ipld.Node) (ChannelID, error) { - ctx, cancel := context.WithCancel(ctx) - go func() { - defer cancel() - err := merkledag.FetchGraph(ctx, baseCid, impl.dag) - var event Event - if err != nil { - event = Error - } else { - event = Complete - } - impl.subscriber(event, ChannelState{Channel: Channel{voucher: voucher}}) - }() - return ChannelID{}, nil -} - -// close an open channel (effectively a cancel) -func (impl *dagserviceImpl) CloseDataTransferChannel(x ChannelID) {} - -// get status of a transfer -func (impl *dagserviceImpl) TransferChannelStatus(x ChannelID) Status { return ChannelNotFoundError } - -// get notified when certain types of events happen -func (impl *dagserviceImpl) SubscribeToEvents(subscriber Subscriber) { - impl.subscriber = subscriber -} - -// get all in progress transfers -func (impl *dagserviceImpl) InProgressChannels() map[ChannelID]ChannelState { return nil } diff --git a/datatransfer/types.go b/datatransfer/types.go deleted file mode 100644 index 684b7f7a8..000000000 --- a/datatransfer/types.go +++ /dev/null @@ -1,173 +0,0 @@ -package datatransfer - -import ( - "context" - "reflect" - - "github.com/ipfs/go-cid" - ipld "github.com/ipld/go-ipld-prime" - "github.com/libp2p/go-libp2p-core/peer" -) - -// Voucher is used to validate -// a data transfer request against the underlying storage or retrieval deal -// that precipitated it. The only requirement is a voucher can read and write -// from bytes, and has a string identifier type -type Voucher interface { - // ToBytes converts the Voucher to raw bytes - ToBytes() ([]byte, error) - // FromBytes reads a Voucher from raw bytes - FromBytes([]byte) error - // Identifier is a unique string identifier for this voucher type - Identifier() string -} - -// Status is the status of transfer for a given channel -type Status int - -const ( - // Ongoing means the data transfer is in progress - Ongoing Status = iota - - // Completed means the data transfer is completed successfully - Completed - - // Failed means the data transfer failed - Failed - - // ChannelNotFoundError means the searched for data transfer does not exist - ChannelNotFoundError -) - -// TransferID is an identifier for a data transfer, shared between -// request/responder and unique to the requester -type TransferID uint64 - -// ChannelID is a unique identifier for a channel, distinct by both the other -// party's peer ID + the transfer ID -type ChannelID struct { - to peer.ID - id TransferID -} - -// Channel represents all the parameters for a single data transfer -type Channel struct { - // an identifier for this channel shared by request and responder, set by requester through protocol - transferID TransferID - // base CID for the piece being transferred - baseCid cid.Cid - // portion of Piece to return, specified by an IPLD selector - selector ipld.Node - // used to verify this channel - voucher Voucher - // the party that is sending the data (not who initiated the request) - sender peer.ID - // the party that is receiving the data (not who initiated the request) - recipient peer.ID - // expected amount of data to be transferred - totalSize uint64 -} - -// TransferID returns the transfer id for this channel -func (c Channel) TransferID() TransferID { return c.transferID } - -// BaseCID returns the CID that is at the root of this data transfer -func (c Channel) BaseCID() cid.Cid { return c.baseCid } - -// Selector returns the IPLD selector for this data transfer (represented as -// an IPLD node) -func (c Channel) Selector() ipld.Node { return c.selector } - -// Voucher returns the voucher for this data transfer -func (c Channel) Voucher() Voucher { return c.voucher } - -// Sender returns the peer id for the node that is sending data -func (c Channel) Sender() peer.ID { return c.sender } - -// Recipient returns the peer id for the node that is receiving data -func (c Channel) Recipient() peer.ID { return c.recipient } - -// TotalSize returns the total size for the data being transferred -func (c Channel) TotalSize() uint64 { return c.totalSize } - -// ChannelState is immutable channel data plus mutable state -type ChannelState struct { - Channel - // total bytes sent from this node (0 if receiver) - sent uint64 - // total bytes received by this node (0 if sender) - received uint64 -} - -// Sent returns the number of bytes sent -func (c ChannelState) Sent() uint64 { return c.sent } - -// Received returns the number of bytes received -func (c ChannelState) Received() uint64 { return c.received } - -// Event is a name for an event that occurs on a data transfer channel -type Event int - -const ( - // Open is an event occurs when a channel is first opened - Open Event = iota - - // Progress is an event that gets emitted every time more data is transferred - Progress - - // Error is an event that emits when an error occurs in a data transfer - Error - - // Complete is emitted when a data transfer is complete - Complete -) - -// Subscriber is a callback that is called when events are emitted -type Subscriber func(event Event, channelState ChannelState) - -// RequestValidator is an interface implemented by the client of the -// data transfer module to validate requests -type RequestValidator interface { - // ValidatePush validates a push request received from the peer that will send data - ValidatePush( - sender peer.ID, - voucher Voucher, - baseCid cid.Cid, - selector ipld.Node) error - // ValidatePull validates a pull request received from the peer that will receive data - ValidatePull( - receiver peer.ID, - voucher Voucher, - baseCid cid.Cid, - selector ipld.Node) error -} - -// Manager is the core interface presented by all implementations of -// of the data transfer sub system -type Manager interface { - // RegisterVoucherType registers a validator for the given voucher type - // will error if voucher type does not implement voucher - // or if there is a voucher type registered with an identical identifier - RegisterVoucherType(voucherType reflect.Type, validator RequestValidator) error - - // open a data transfer that will send data to the recipient peer and - // open a data transfer that will send data to the recipient peer and - // transfer parts of the piece that match the selector - OpenPushDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, selector ipld.Node) (ChannelID, error) - - // open a data transfer that will request data from the sending peer and - // transfer parts of the piece that match the selector - OpenPullDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, selector ipld.Node) (ChannelID, error) - - // close an open channel (effectively a cancel) - CloseDataTransferChannel(x ChannelID) - - // get status of a transfer - TransferChannelStatus(x ChannelID) Status - - // get notified when certain types of events happen - SubscribeToEvents(subscriber Subscriber) - - // get all in progress transfers - InProgressChannels() map[ChannelID]ChannelState -} diff --git a/go.mod b/go.mod index 541d7f8d1..1dad59831 100644 --- a/go.mod +++ b/go.mod @@ -15,12 +15,12 @@ require ( github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 + github.com/filecoin-project/go-fil-components v0.0.0-20191209202555-21dd66ba3701 github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107220006-3361d30ea5ab github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-ole/go-ole v1.2.4 // indirect - github.com/google/go-cmp v0.3.1 // indirect github.com/gorilla/mux v1.7.3 github.com/gorilla/websocket v1.4.1 github.com/hashicorp/go-multierror v1.0.0 @@ -35,6 +35,7 @@ require ( github.com/ipfs/go-ds-badger v0.0.7 github.com/ipfs/go-filestore v0.0.2 github.com/ipfs/go-fs-lock v0.0.1 + github.com/ipfs/go-graphsync v0.0.4 github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1 github.com/ipfs/go-ipfs-blockstore v0.1.1 github.com/ipfs/go-ipfs-blocksutil v0.0.1 @@ -50,7 +51,7 @@ require ( github.com/ipfs/go-merkledag v0.2.4 github.com/ipfs/go-path v0.0.7 github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb - github.com/ipld/go-ipld-prime v0.0.2-0.20191025154717-8dff1cbec43b + github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 github.com/libp2p/go-libp2p v0.4.2 github.com/libp2p/go-libp2p-circuit v0.1.4 github.com/libp2p/go-libp2p-connmgr v0.1.0 @@ -79,8 +80,6 @@ require ( github.com/multiformats/go-multiaddr-net v0.1.1 github.com/multiformats/go-multihash v0.0.10 github.com/multiformats/go-varint v0.0.2 - github.com/onsi/ginkgo v1.9.0 // indirect - github.com/onsi/gomega v1.6.0 // indirect github.com/opentracing/opentracing-go v1.1.0 github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a github.com/prometheus/common v0.2.0 diff --git a/go.sum b/go.sum index 32e0a5717..81f67fff2 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,7 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= +github.com/btcsuite/btcd v0.0.0-20190629003639-c26ffa870fd8/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 h1:A/EVblehb75cUgXA5njHPn0kLAsykn6mJGz7rnmW5W0= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -93,6 +94,9 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMX github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= +github.com/filecoin-project/go-fil-components v0.0.0-20191209202555-21dd66ba3701 h1:aDoskukzly9o/fFbHGOZVZDOJc3eeo6IQwSp5bvkfb8= +github.com/filecoin-project/go-fil-components v0.0.0-20191209202555-21dd66ba3701/go.mod h1:VEQZ6UfSBYB3wCl19LZUScGfNaIakrBlMSxiVATWPMg= +github.com/filecoin-project/go-fil-filestore v0.0.0-20191202230242-40c6a5a2306c/go.mod h1:pPW5DjeA3Fk89gLGxMYH7FFd8TzZ4H3NAryc7UWQVCg= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= @@ -126,8 +130,9 @@ github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -209,6 +214,8 @@ github.com/ipfs/go-filestore v0.0.2 h1:pcYwpjtXXwirtbjBXKVJM9CTa9F7/8v1EkfnDaHTO github.com/ipfs/go-filestore v0.0.2/go.mod h1:KnZ41qJsCt2OX2mxZS0xsK3Psr0/oB93HMMssLujjVc= github.com/ipfs/go-fs-lock v0.0.1 h1:XHX8uW4jQBYWHj59XXcjg7BHlHxV9ZOYs6Y43yb7/l0= github.com/ipfs/go-fs-lock v0.0.1/go.mod h1:DNBekbboPKcxs1aukPSaOtFA3QfSdi5C855v0i9XJ8Y= +github.com/ipfs/go-graphsync v0.0.4 h1:iF98+J8pcqvEb48IM0TemqeGARsCDtwQ73P9ejMZIuU= +github.com/ipfs/go-graphsync v0.0.4/go.mod h1:6UACBjfOXEa8rQL3Q/JpZpWS0nZDCLx134WUkjrmFpQ= github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1 h1:0xpCaaXvN8bPyws3ObiCn7G0KIfgbS9E132eL57dHx8= github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1/go.mod h1:8yRx0xLUps1Xq8ZDnIwIVdQRp7JjA55gGvCiRHT91Vk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= @@ -269,8 +276,10 @@ github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb h1:tmWYgjltxwM7PD github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= -github.com/ipld/go-ipld-prime v0.0.2-0.20191025154717-8dff1cbec43b h1:ACSEK4f1SDQC+FJ4B4pqHFW14d7kEW2ufwXA/c7eLP0= -github.com/ipld/go-ipld-prime v0.0.2-0.20191025154717-8dff1cbec43b/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w= +github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 h1:fASnkvtR+SmB2y453RxmDD3Uvd4LonVUgFGk9JoDaZs= +github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w= +github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5 h1:lSip43rAdyGA+yRQuy6ju0ucZkWpYc1F2CTQtZTVW/4= +github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= @@ -281,6 +290,8 @@ github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+ github.com/jbenet/go-cienv v0.0.0-20150120210510-1bb1476777ec/go.mod h1:rGaEvXB4uRSZMmzKNLoXvTu1sfx+1kv/DojUlPrSZGs= github.com/jbenet/go-cienv v0.1.0 h1:Vc/s0QbQtoxX8MwwSLWWh+xNNZvM3Lw7NsTcHrvvhMc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c h1:uUx61FiAa1GI6ZmVd2wf2vULeQZIKG66eybjNXKYCz4= +github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c/go.mod h1:sdx1xVM9UuLw1tXnhJWN3piypTUO3vCIHYmG15KE/dU= github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2 h1:vhC1OXXiT9R2pczegwz6moDvuRpggaroAXhPIseh57A= github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod h1:8GXXJV31xl8whumTzdZsTt3RnUIiPqzkyf7mxToRCMs= github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= @@ -319,6 +330,8 @@ github.com/libp2p/go-conn-security v0.0.1/go.mod h1:bGmu51N0KU9IEjX7kl2PQjgZa40J github.com/libp2p/go-conn-security-multistream v0.0.2/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= github.com/libp2p/go-conn-security-multistream v0.1.0 h1:aqGmto+ttL/uJgX0JtQI0tD21CIEy5eYd1Hlp0juHY0= github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= +github.com/libp2p/go-eventbus v0.0.2/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= +github.com/libp2p/go-eventbus v0.0.3/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= github.com/libp2p/go-eventbus v0.1.0 h1:mlawomSAjjkk97QnYiEmHsLu7E136+2oCWSHRUvMfzQ= github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= github.com/libp2p/go-flow-metrics v0.0.1 h1:0gxuFd2GuK7IIP5pKljLwps6TvcuYgvG7Atqi3INF5s= @@ -326,6 +339,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-libp2p v0.0.30/go.mod h1:XWT8FGHlhptAv1+3V/+J5mEpzyui/5bvFsNuWYs611A= github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM= github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8= +github.com/libp2p/go-libp2p v0.2.1/go.mod h1:HZbtEOrgZN4F1fGZVvkV+930Wx3DkqlpBlO8dIoZWds= +github.com/libp2p/go-libp2p v0.3.0/go.mod h1:J7DPB1+zB5VLc8v/kKSD8+u2cbyIGI0Dh/Pf3Wprt+0= github.com/libp2p/go-libp2p v0.4.2 h1:p0cthB0jDNHO4gH2HzS8/nAMMXbfUlFHs0jwZ4U+F2g= github.com/libp2p/go-libp2p v0.4.2/go.mod h1:MNmgUxUw5pMsdOzMlT0EE7oKjRasl+WyVwM0IBlpKgQ= github.com/libp2p/go-libp2p-autonat v0.0.6/go.mod h1:uZneLdOkZHro35xIhpbtTzLlgYturpu4J5+0cZK3MqE= @@ -334,10 +349,12 @@ github.com/libp2p/go-libp2p-autonat v0.1.1 h1:WLBZcIRsjZlWdAZj9CiBSvU2wQXoUOiS1Z github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE= github.com/libp2p/go-libp2p-blankhost v0.0.1/go.mod h1:Ibpbw/7cPPYwFb7PACIWdvxxv0t0XCCI10t7czjAjTc= github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= +github.com/libp2p/go-libp2p-blankhost v0.1.3/go.mod h1:KML1//wiKR8vuuJO0y3LUd1uLv+tlkGTAr3jC0S5cLg= github.com/libp2p/go-libp2p-blankhost v0.1.4 h1:I96SWjR4rK9irDHcHq3XHN6hawCRTPUADzkJacgZLvk= github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU= github.com/libp2p/go-libp2p-circuit v0.0.9/go.mod h1:uU+IBvEQzCu953/ps7bYzC/D/R0Ho2A9LfKVVCatlqU= github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= +github.com/libp2p/go-libp2p-circuit v0.1.1/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= github.com/libp2p/go-libp2p-circuit v0.1.4 h1:Phzbmrg3BkVzbqd4ZZ149JxCuUWu2wZcXf/Kr6hZJj8= github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU= github.com/libp2p/go-libp2p-connmgr v0.1.0 h1:vp0t0F0EuT3rrlTtnMnIyyzCnly7nIlRoEbhJpgp0qU= @@ -346,6 +363,8 @@ github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGc github.com/libp2p/go-libp2p-core v0.0.2/go.mod h1:9dAcntw/n46XycV4RnlBq3BpgrmyUi9LuoTNdPrbUco= github.com/libp2p/go-libp2p-core v0.0.3/go.mod h1:j+YQMNz9WNSkNezXOsahp9kwZBKBvxLpKD316QWSJXE= github.com/libp2p/go-libp2p-core v0.0.4/go.mod h1:jyuCQP356gzfCFtRKyvAbNkyeuxb7OlyhWZ3nls5d2I= +github.com/libp2p/go-libp2p-core v0.0.6/go.mod h1:0d9xmaYAVY5qmbp/fcgxHT3ZJsLjYeYPMJAUKpaCHrE= +github.com/libp2p/go-libp2p-core v0.0.9/go.mod h1:0d9xmaYAVY5qmbp/fcgxHT3ZJsLjYeYPMJAUKpaCHrE= github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI= github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0= github.com/libp2p/go-libp2p-core v0.2.4 h1:Et6ykkTwI6PU44tr8qUF9k43vP0aduMNniShAbUJJw8= @@ -391,6 +410,8 @@ github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMg github.com/libp2p/go-libp2p-peerstore v0.0.1/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20= github.com/libp2p/go-libp2p-peerstore v0.0.6/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= +github.com/libp2p/go-libp2p-peerstore v0.1.2/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= +github.com/libp2p/go-libp2p-peerstore v0.1.3 h1:wMgajt1uM2tMiqf4M+4qWKVyyFc8SfA+84VV9glZq1M= github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= github.com/libp2p/go-libp2p-peerstore v0.1.4 h1:d23fvq5oYMJ/lkkbO4oTwBp/JP+I/1m5gZJobNXCE/k= github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs= @@ -413,11 +434,15 @@ github.com/libp2p/go-libp2p-routing-helpers v0.1.0 h1:BaFvpyv8TyhCN7TihawTiKuzeu github.com/libp2p/go-libp2p-routing-helpers v0.1.0/go.mod h1:oUs0h39vNwYtYXnQWOTU5BaafbedSyWCCal3gqHuoOQ= github.com/libp2p/go-libp2p-secio v0.0.3/go.mod h1:hS7HQ00MgLhRO/Wyu1bTX6ctJKhVpm+j2/S2A5UqYb0= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= +github.com/libp2p/go-libp2p-secio v0.1.1/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= +github.com/libp2p/go-libp2p-secio v0.2.0 h1:ywzZBsWEEz2KNTn5RtzauEDq5RFEefPsttXYwAWqHng= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= github.com/libp2p/go-libp2p-secio v0.2.1 h1:eNWbJTdyPA7NxhP7J3c5lT97DC5d+u+IldkgCYFTPVA= github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= github.com/libp2p/go-libp2p-swarm v0.0.6/go.mod h1:s5GZvzg9xXe8sbeESuFpjt8CJPTCa8mhEusweJqyFy8= github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= +github.com/libp2p/go-libp2p-swarm v0.1.1/go.mod h1:4NVJaLwq/dr5kEq79Jo6pMin7ZFwLx73ln1FTefR91Q= +github.com/libp2p/go-libp2p-swarm v0.2.0/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+74Jjio7xGvsTgU= github.com/libp2p/go-libp2p-swarm v0.2.2 h1:T4hUpgEs2r371PweU3DuH7EOmBIdTBCwWs+FLcgx3bQ= github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaTNyBcHImCxRpPKU= github.com/libp2p/go-libp2p-testing v0.0.1/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= @@ -461,6 +486,7 @@ github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FW github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= +github.com/libp2p/go-stream-muxer v0.0.1 h1:Ce6e2Pyu+b5MC1k3eeFtAax0pW4gc6MosYSLV05UeLw= github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14= github.com/libp2p/go-stream-muxer v0.1.0/go.mod h1:8JAVsjeRBCWwPoZeH0W1imLOcriqXJyFvB0mR4A04sQ= github.com/libp2p/go-stream-muxer-multistream v0.1.1/go.mod h1:zmGdfkQ1AzOECIAcccoL8L//laqawOsO03zX8Sa+eGw= @@ -528,6 +554,7 @@ github.com/multiformats/go-multiaddr v0.1.1 h1:rVAztJYMhCQ7vEFr8FvxW3mS+HF2eY/oP github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= +github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0 h1:YWJoIDwLePniH7OU5hBnDZV6SWuvJqJ0YtN6pLeH9zA= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q= @@ -541,6 +568,7 @@ github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmr github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= +github.com/multiformats/go-multihash v0.0.6/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM= @@ -607,6 +635,7 @@ github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHei github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= @@ -672,6 +701,8 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7V github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -707,6 +738,8 @@ golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -730,10 +763,13 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -760,6 +796,7 @@ golang.org/x/sys v0.0.0-20190524152521-dbbf3f1254d4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -800,6 +837,7 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/node/builder.go b/node/builder.go index ee34b3600..ee15f8e70 100644 --- a/node/builder.go +++ b/node/builder.go @@ -243,8 +243,9 @@ func Online() Option { Override(new(storage.TicketFn), modules.SealTicketGen), Override(new(*storage.Miner), modules.StorageMiner), + Override(new(dtypes.StagingBlockstore), modules.StagingBlockstore), Override(new(dtypes.StagingDAG), modules.StagingDAG), - + Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync), Override(new(*retrieval.Miner), retrieval.NewMiner), Override(new(dtypes.ProviderDealStore), modules.NewProviderDealStore), Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), @@ -372,6 +373,7 @@ func Repo(r repo.Repo) Option { Override(new(dtypes.ClientFilestore), modules.ClientFstore), Override(new(dtypes.ClientBlockstore), modules.ClientBlockstore), Override(new(dtypes.ClientDAG), modules.ClientDAG), + Override(new(dtypes.ClientGraphsync), modules.ClientGraphsync), Override(new(ci.PrivKey), lp2p.PrivKey), Override(new(ci.PubKey), ci.PrivKey.GetPublic), diff --git a/node/modules/client.go b/node/modules/client.go index 205d39a6d..dc17996c9 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -9,6 +9,10 @@ import ( "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" + graphsync "github.com/ipfs/go-graphsync/impl" + "github.com/ipfs/go-graphsync/ipldbridge" + gsnet "github.com/ipfs/go-graphsync/network" + "github.com/ipfs/go-graphsync/storeutil" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/routing" @@ -20,8 +24,8 @@ import ( "github.com/ipfs/go-merkledag" "go.uber.org/fx" + "github.com/filecoin-project/go-fil-components/datatransfer/impl/graphsync" "github.com/filecoin-project/lotus/chain/deals" - "github.com/filecoin-project/lotus/datatransfer" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" ) @@ -49,15 +53,15 @@ func ClientBlockstore(fstore dtypes.ClientFilestore) dtypes.ClientBlockstore { // request validator with the data transfer module as the validator for // StorageDataTransferVoucher types func RegisterClientValidator(crv *deals.ClientRequestValidator, dtm dtypes.ClientDataTransfer) { - if err := dtm.RegisterVoucherType(reflect.TypeOf(deals.StorageDataTransferVoucher{}), crv); err != nil { + if err := dtm.RegisterVoucherType(reflect.TypeOf(&deals.StorageDataTransferVoucher{}), crv); err != nil { panic(err) } } // NewClientDAGServiceDataTransfer returns a data transfer manager that just // uses the clients's Client DAG service for transfers -func NewClientDAGServiceDataTransfer(dag dtypes.ClientDAG) dtypes.ClientDataTransfer { - return datatransfer.NewDAGServiceDataTransfer(dag) +func NewClientDAGServiceDataTransfer(h host.Host, gs dtypes.ClientGraphsync) dtypes.ClientDataTransfer { + return graphsyncimpl.NewGraphSyncDataTransfer(h, gs) } // NewClientDealStore creates a statestore for the client to store its deals @@ -65,6 +69,7 @@ func NewClientDealStore(ds dtypes.MetadataDS) dtypes.ClientDealStore { return statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))) } +// ClientDAG is a DAGService for the ClientBlockstore func ClientDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.ClientBlockstore, rt routing.Routing, h host.Host) dtypes.ClientDAG { bitswapNetwork := network.NewFromIpfsHost(h, rt) exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, ibs) @@ -80,3 +85,15 @@ func ClientDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.ClientBlocks return dag } + +// ClientGraphsync creates a graphsync instance which reads and writes blocks +// to the ClientBlockstore +func ClientGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.ClientBlockstore, h host.Host) dtypes.ClientGraphsync { + graphsyncNetwork := gsnet.NewFromLibp2pHost(h) + ipldBridge := ipldbridge.NewIPLDBridge() + loader := storeutil.LoaderForBlockstore(ibs) + storer := storeutil.StorerForBlockstore(ibs) + gs := graphsync.New(helpers.LifecycleCtx(mctx, lc), graphsyncNetwork, ipldBridge, loader, storer) + + return gs +} diff --git a/node/modules/dtypes/storage.go b/node/modules/dtypes/storage.go index 32b97767d..4c09a61e5 100644 --- a/node/modules/dtypes/storage.go +++ b/node/modules/dtypes/storage.go @@ -1,15 +1,15 @@ package dtypes import ( + "github.com/filecoin-project/go-fil-components/datatransfer" + "github.com/filecoin-project/go-statestore" bserv "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" "github.com/ipfs/go-filestore" + "github.com/ipfs/go-graphsync" blockstore "github.com/ipfs/go-ipfs-blockstore" exchange "github.com/ipfs/go-ipfs-exchange-interface" ipld "github.com/ipfs/go-ipld-format" - - "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/datatransfer" ) // MetadataDS stores metadata @@ -26,6 +26,7 @@ type ChainBlockService bserv.BlockService type ClientFilestore *filestore.Filestore type ClientBlockstore blockstore.Blockstore type ClientDAG ipld.DAGService +type ClientGraphsync graphsync.GraphExchange type ClientDealStore *statestore.StateStore // ClientDataTransfer is a data transfer manager for the client @@ -37,3 +38,5 @@ type ProviderDealStore *statestore.StateStore type ProviderDataTransfer datatransfer.Manager type StagingDAG ipld.DAGService +type StagingBlockstore blockstore.Blockstore +type StagingGraphsync graphsync.GraphExchange diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 23fb35455..13c8a749e 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -11,6 +11,10 @@ import ( "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" + graphsync "github.com/ipfs/go-graphsync/impl" + "github.com/ipfs/go-graphsync/ipldbridge" + gsnet "github.com/ipfs/go-graphsync/network" + "github.com/ipfs/go-graphsync/storeutil" blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-merkledag" "github.com/libp2p/go-libp2p-core/host" @@ -20,13 +24,13 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + dtgraphsync "github.com/filecoin-project/go-fil-components/datatransfer/impl/graphsync" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/gen" - "github.com/filecoin-project/lotus/datatransfer" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" @@ -142,15 +146,15 @@ func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h *de // request validator with the data transfer module as the validator for // StorageDataTransferVoucher types func RegisterProviderValidator(mrv *deals.ProviderRequestValidator, dtm dtypes.ProviderDataTransfer) { - if err := dtm.RegisterVoucherType(reflect.TypeOf(deals.StorageDataTransferVoucher{}), mrv); err != nil { + if err := dtm.RegisterVoucherType(reflect.TypeOf(&deals.StorageDataTransferVoucher{}), mrv); err != nil { panic(err) } } // NewProviderDAGServiceDataTransfer returns a data transfer manager that just // uses the provider's Staging DAG service for transfers -func NewProviderDAGServiceDataTransfer(dag dtypes.StagingDAG) dtypes.ProviderDataTransfer { - return datatransfer.NewDAGServiceDataTransfer(dag) +func NewProviderDAGServiceDataTransfer(h host.Host, gs dtypes.StagingGraphsync) dtypes.ProviderDataTransfer { + return dtgraphsync.NewGraphSyncDataTransfer(h, gs) } // NewProviderDealStore creates a statestore for the client to store its deals @@ -158,7 +162,9 @@ func NewProviderDealStore(ds dtypes.MetadataDS) dtypes.ProviderDealStore { return statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))) } -func StagingDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, rt routing.Routing, h host.Host) (dtypes.StagingDAG, error) { +// StagingBlockstore creates a blockstore for staging blocks for a miner +// in a storage deal, prior to sealing +func StagingBlockstore(r repo.LockedRepo) (dtypes.StagingBlockstore, error) { stagingds, err := r.Datastore("/staging") if err != nil { return nil, err @@ -167,8 +173,14 @@ func StagingDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, rt bs := blockstore.NewBlockstore(stagingds) ibs := blockstore.NewIdStore(bs) + return ibs, nil +} + +// StagingDAG is a DAGService for the StagingBlockstore +func StagingDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.StagingBlockstore, rt routing.Routing, h host.Host) (dtypes.StagingDAG, error) { + bitswapNetwork := network.NewFromIpfsHost(h, rt) - exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, bs) + exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, ibs) bsvc := blockservice.New(ibs, exch) dag := merkledag.NewDAGService(bsvc) @@ -182,6 +194,18 @@ func StagingDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, rt return dag, nil } +// StagingGraphsync creates a graphsync instance which reads and writes blocks +// to the StagingBlockstore +func StagingGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.StagingBlockstore, h host.Host) dtypes.StagingGraphsync { + graphsyncNetwork := gsnet.NewFromLibp2pHost(h) + ipldBridge := ipldbridge.NewIPLDBridge() + loader := storeutil.LoaderForBlockstore(ibs) + storer := storeutil.StorerForBlockstore(ibs) + gs := graphsync.New(helpers.LifecycleCtx(mctx, lc), graphsyncNetwork, ipldBridge, loader, storer) + + return gs +} + func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api api.FullNode, epp gen.ElectionPoStProver) (*miner.Miner, error) { minerAddr, err := minerAddrFromDS(ds) if err != nil { From 5312340bf9b176403560098fcb17484d6a8b61cb Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Wed, 18 Dec 2019 17:05:07 -0800 Subject: [PATCH 2/3] refactor(datatransfer): use independent repo Use independent repo for go-data-transfer --- chain/deals/client_utils.go | 2 +- chain/deals/provider.go | 2 +- chain/deals/provider_utils.go | 2 +- go.mod | 2 +- go.sum | 20 +++++++++++++++++++- node/modules/client.go | 2 +- node/modules/dtypes/storage.go | 5 +++-- node/modules/storageminer.go | 2 +- 8 files changed, 28 insertions(+), 9 deletions(-) diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go index 5c875749a..f2d150a23 100644 --- a/chain/deals/client_utils.go +++ b/chain/deals/client_utils.go @@ -12,8 +12,8 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" - "github.com/filecoin-project/go-fil-components/datatransfer" cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/lib/padreader" diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 7c0bde0bf..0b1fa3900 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -14,7 +14,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-fil-components/datatransfer" + "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go index 35614f32d..a02c6daf9 100644 --- a/chain/deals/provider_utils.go +++ b/chain/deals/provider_utils.go @@ -4,7 +4,7 @@ import ( "context" "runtime" - "github.com/filecoin-project/go-fil-components/datatransfer" + "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/ipld/go-ipld-prime" diff --git a/go.mod b/go.mod index 1dad59831..068fb7b29 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-fil-components v0.0.0-20191209202555-21dd66ba3701 + github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107220006-3361d30ea5ab github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 diff --git a/go.sum b/go.sum index 81f67fff2..f87498af4 100644 --- a/go.sum +++ b/go.sum @@ -54,7 +54,10 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.2.1-0.20180108230905-e214231b295a/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/daaku/go.zipexe v1.0.0 h1:VSOgZtH418pH9L16hC/JrgSNJbbAL26pj7lmD1+CGdY= @@ -98,6 +101,8 @@ github.com/filecoin-project/go-fil-components v0.0.0-20191209202555-21dd66ba3701 github.com/filecoin-project/go-fil-components v0.0.0-20191209202555-21dd66ba3701/go.mod h1:VEQZ6UfSBYB3wCl19LZUScGfNaIakrBlMSxiVATWPMg= github.com/filecoin-project/go-fil-filestore v0.0.0-20191202230242-40c6a5a2306c/go.mod h1:pPW5DjeA3Fk89gLGxMYH7FFd8TzZ4H3NAryc7UWQVCg= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= +github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY= +github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= @@ -155,6 +160,8 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE= +github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c h1:+MSf4NEnLCYZoAgK6fqwc7NH88nM8haFSxKGUGIG3vA= +github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c/go.mod h1:WVPCl0HO/0RAL5+vBH2GMxBomlxBF70MAS78+Lu1//k= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= @@ -624,10 +631,15 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -660,7 +672,10 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg= +github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= @@ -675,6 +690,7 @@ github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba/go.mod h1:CH github.com/whyrusleeping/cbor-gen v0.0.0-20190910031516-c1cbffdb01bb/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20190917003517-d78d67427694/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191116002219-891f55cd449d/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= +github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0 h1:efb/4CnrubzNGqQOeHErxyQ6rIsJb7GcgeSDF7fqWeI= github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= @@ -751,6 +767,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -819,8 +836,9 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361 h1:RIIXAeV6GvDBuADKumTODatUqANFZ+5BPMnzsy4hulY= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/node/modules/client.go b/node/modules/client.go index dc17996c9..f01bbf8c3 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -24,7 +24,7 @@ import ( "github.com/ipfs/go-merkledag" "go.uber.org/fx" - "github.com/filecoin-project/go-fil-components/datatransfer/impl/graphsync" + "github.com/filecoin-project/go-data-transfer/impl/graphsync" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" diff --git a/node/modules/dtypes/storage.go b/node/modules/dtypes/storage.go index 4c09a61e5..d80f86d24 100644 --- a/node/modules/dtypes/storage.go +++ b/node/modules/dtypes/storage.go @@ -1,8 +1,6 @@ package dtypes import ( - "github.com/filecoin-project/go-fil-components/datatransfer" - "github.com/filecoin-project/go-statestore" bserv "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" "github.com/ipfs/go-filestore" @@ -10,6 +8,9 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" exchange "github.com/ipfs/go-ipfs-exchange-interface" ipld "github.com/ipfs/go-ipld-format" + + "github.com/filecoin-project/go-data-transfer" + "github.com/filecoin-project/go-statestore" ) // MetadataDS stores metadata diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 13c8a749e..9725d3440 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -24,7 +24,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - dtgraphsync "github.com/filecoin-project/go-fil-components/datatransfer/impl/graphsync" + dtgraphsync "github.com/filecoin-project/go-data-transfer/impl/graphsync" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" From bf9aa4465bbafa8305a062583bc5ac6ad41413a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 8 Jan 2020 14:20:43 +0100 Subject: [PATCH 3/3] mod tidy --- go.sum | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/go.sum b/go.sum index f87498af4..3cc9acf70 100644 --- a/go.sum +++ b/go.sum @@ -97,13 +97,10 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMX github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-fil-components v0.0.0-20191209202555-21dd66ba3701 h1:aDoskukzly9o/fFbHGOZVZDOJc3eeo6IQwSp5bvkfb8= -github.com/filecoin-project/go-fil-components v0.0.0-20191209202555-21dd66ba3701/go.mod h1:VEQZ6UfSBYB3wCl19LZUScGfNaIakrBlMSxiVATWPMg= -github.com/filecoin-project/go-fil-filestore v0.0.0-20191202230242-40c6a5a2306c/go.mod h1:pPW5DjeA3Fk89gLGxMYH7FFd8TzZ4H3NAryc7UWQVCg= -github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= +github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107220006-3361d30ea5ab h1:bsrBNO1LwnhOLxPEXlSPal/WuY61mLJUCHYyD0NayHg= @@ -637,9 +634,9 @@ github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=