61e2568b8d
feat(datatransfer): setup implementation path Sets up a path to implementation, offering both the dagservice implementation and a future graphsync implement, establishes message interfaces and network layer, and isolates the datatransfer module from the app WIP using CBOR encoding for dataxfermsg * Bring cbor-gen stuff into datatransfer package * make transferRequest private struct * add transferResponse + funcs * Rename VoucherID to VoucherType * more tests passing WIP trying out some stuff * Embed request/response in message so all the interfaces work AND the CBOR unmarshaling works: this is more like the spec anyway * get rid of pb stuff all message tests passing, some others in datatransfer Some cleanup for PR Cleanup for PR, clarifying and additional comments mod tidy Respond to PR comments: * Make DataTransferRequest/Response be returned in from Net * Regenerate cbor_gen and fix the generator caller so it works better * Please the linters Fix tests Initiate push and pull requests (#536) * add issue link for data TransferID generation * comment out failing but not relevant tests * finish voucher rename from Identifier --> Type tests passing cleanup for PR remove unused fmt import in graphsync_test a better reflection send data transfer response other tests passing feat(datatransfer): milestone 2 infrastructure Setup test path for all tickets for milestone 2 responses alert subscribers when request is not accepted (#607) Graphsync response is scheduled when a valid push request is received (#625) fix(datatransfer): fix tests fix an error with read buffers in tests fix(deps): fix go.sum Feat/dt graphsync pullreqs (#627) * graphsync responses to pull requests Feat/dt initiator cleanup (#645) * ChannelID.To --> ChannelID.Initiator * We now store our peer ID (from host.ID()) so it can be used when creating ChannelIDs. * InProgressChannels returns all of impl.channels, currently just for testing * Implements go-data-transfer issue * Some assertions were changed based on the above. * Renamed some variables and added some assertions based on the new understanding * Updated SHA for graphsync module * Updated fakeGraphSync test structs to use new interfaces from new SHA above Techdebt/dt split graphsync impl receiver (#651) * Split up graphsyncImpl and graphsyncReceiver * rename graphsync to utils DTM sends data over graphsync for validated push requests (#665) * create channels when a request is received. register push request hook with graphsync. fix tests. * better NewReaders * use mutex lock around impl.channels access * fix(datatransfer): fix test uncertainty * fix a data race and also don't use random bytes in basic block which can fail * privatize 3 funcs with @hannahhoward Feat/dt gs pullrequests (#693) * Implements DTM Sends Data Over Graphsync For Validated Pull Requests * rename a field in a test struct * refactor a couple of private functions (one was refactored out of existence) Feat/dt subscribe, file Xfer round trip (#720) Implements the rest of Subscriber Is Notified When Request Completed #24: * send a graphsync message within a go func and consume responses until error or transfer is complete. * notify subscribers of results. * Rename datatransfer.Event to EventCode. * datatransfer.Event is now a struct that includes a message and a timestamp as well as the Event.Code int, formerly Event, update all uses * Add extension data to graphsync request hook, gsReq * rename sendRequest to sendDtRequest, to distinguish it from sendGsRequest, where Dt = datatransfer, Gs = graphsync * use a mutex lock for last transfer ID * obey the linter Don't respond with error in gsReqRcdHook when we can't find the datatransfer extension. (#754) * update to correct graphsync version, update tests & code to call the new graphsync hooks * getExtensionData returns an empty struct + nil if we can't find our extension * Don't respond with error when we can't find the extension. * Test for same * mod tidy minor fix to go.sum feat(datatransfer): switch to graphsync implementation Move over to real graphsync implementation of data transfer, add constructors for graphsync instances on client and miner side fix(datatransfer): Fix validators Validators were checking payload cid against commP -- which are not the same any more. Added a payloadCid to client deal to maintain the record, fixed validator logic Feat/dt extraction use go-fil-components/datatransfer (#770) * Initial commit after changing to go-fil-components/datatransfer * blow away the datatransfer dir * use go-fil-components master after its PR #1 was merged * go mod tidy use a package updates after rebase with master
100 lines
3.5 KiB
Go
100 lines
3.5 KiB
Go
package modules
|
|
|
|
import (
|
|
"context"
|
|
"path/filepath"
|
|
"reflect"
|
|
|
|
"github.com/filecoin-project/go-statestore"
|
|
"github.com/filecoin-project/lotus/node/modules/helpers"
|
|
"github.com/ipfs/go-bitswap"
|
|
"github.com/ipfs/go-bitswap/network"
|
|
graphsync "github.com/ipfs/go-graphsync/impl"
|
|
"github.com/ipfs/go-graphsync/ipldbridge"
|
|
gsnet "github.com/ipfs/go-graphsync/network"
|
|
"github.com/ipfs/go-graphsync/storeutil"
|
|
"github.com/libp2p/go-libp2p-core/host"
|
|
"github.com/libp2p/go-libp2p-core/routing"
|
|
|
|
"github.com/ipfs/go-blockservice"
|
|
"github.com/ipfs/go-datastore"
|
|
"github.com/ipfs/go-datastore/namespace"
|
|
"github.com/ipfs/go-filestore"
|
|
blockstore "github.com/ipfs/go-ipfs-blockstore"
|
|
"github.com/ipfs/go-merkledag"
|
|
"go.uber.org/fx"
|
|
|
|
"github.com/filecoin-project/go-fil-components/datatransfer/impl/graphsync"
|
|
"github.com/filecoin-project/lotus/chain/deals"
|
|
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
|
"github.com/filecoin-project/lotus/node/repo"
|
|
)
|
|
|
|
func ClientFstore(r repo.LockedRepo) (dtypes.ClientFilestore, error) {
|
|
clientds, err := r.Datastore("/client")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
blocks := namespace.Wrap(clientds, datastore.NewKey("blocks"))
|
|
|
|
fm := filestore.NewFileManager(clientds, filepath.Dir(r.Path()))
|
|
fm.AllowFiles = true
|
|
// TODO: fm.AllowUrls (needs more code in client import)
|
|
|
|
bs := blockstore.NewBlockstore(blocks)
|
|
return filestore.NewFilestore(bs, fm), nil
|
|
}
|
|
|
|
func ClientBlockstore(fstore dtypes.ClientFilestore) dtypes.ClientBlockstore {
|
|
return blockstore.NewIdStore((*filestore.Filestore)(fstore))
|
|
}
|
|
|
|
// RegisterClientValidator is an initialization hook that registers the client
|
|
// request validator with the data transfer module as the validator for
|
|
// StorageDataTransferVoucher types
|
|
func RegisterClientValidator(crv *deals.ClientRequestValidator, dtm dtypes.ClientDataTransfer) {
|
|
if err := dtm.RegisterVoucherType(reflect.TypeOf(&deals.StorageDataTransferVoucher{}), crv); err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
// NewClientDAGServiceDataTransfer returns a data transfer manager that just
|
|
// uses the clients's Client DAG service for transfers
|
|
func NewClientDAGServiceDataTransfer(h host.Host, gs dtypes.ClientGraphsync) dtypes.ClientDataTransfer {
|
|
return graphsyncimpl.NewGraphSyncDataTransfer(h, gs)
|
|
}
|
|
|
|
// NewClientDealStore creates a statestore for the client to store its deals
|
|
func NewClientDealStore(ds dtypes.MetadataDS) dtypes.ClientDealStore {
|
|
return statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client")))
|
|
}
|
|
|
|
// ClientDAG is a DAGService for the ClientBlockstore
|
|
func ClientDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.ClientBlockstore, rt routing.Routing, h host.Host) dtypes.ClientDAG {
|
|
bitswapNetwork := network.NewFromIpfsHost(h, rt)
|
|
exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, ibs)
|
|
|
|
bsvc := blockservice.New(ibs, exch)
|
|
dag := merkledag.NewDAGService(bsvc)
|
|
|
|
lc.Append(fx.Hook{
|
|
OnStop: func(_ context.Context) error {
|
|
return bsvc.Close()
|
|
},
|
|
})
|
|
|
|
return dag
|
|
}
|
|
|
|
// ClientGraphsync creates a graphsync instance which reads and writes blocks
|
|
// to the ClientBlockstore
|
|
func ClientGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.ClientBlockstore, h host.Host) dtypes.ClientGraphsync {
|
|
graphsyncNetwork := gsnet.NewFromLibp2pHost(h)
|
|
ipldBridge := ipldbridge.NewIPLDBridge()
|
|
loader := storeutil.LoaderForBlockstore(ibs)
|
|
storer := storeutil.StorerForBlockstore(ibs)
|
|
gs := graphsync.New(helpers.LifecycleCtx(mctx, lc), graphsyncNetwork, ipldBridge, loader, storer)
|
|
|
|
return gs
|
|
}
|