lotus/node/builder.go

452 lines
14 KiB
Go
Raw Normal View History

package node
import (
"context"
2019-07-04 15:50:48 +00:00
"errors"
"time"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
2019-07-08 14:07:09 +00:00
blockstore "github.com/ipfs/go-ipfs-blockstore"
ci "github.com/libp2p/go-libp2p-core/crypto"
2019-07-05 10:06:28 +00:00
"github.com/libp2p/go-libp2p-core/host"
"github.com/libp2p/go-libp2p-core/peer"
2019-07-05 10:06:28 +00:00
"github.com/libp2p/go-libp2p-core/peerstore"
"github.com/libp2p/go-libp2p-core/routing"
"github.com/libp2p/go-libp2p-peerstore/pstoremem"
2019-07-08 14:07:09 +00:00
pubsub "github.com/libp2p/go-libp2p-pubsub"
2019-07-05 10:06:28 +00:00
record "github.com/libp2p/go-libp2p-record"
"github.com/multiformats/go-multiaddr"
"go.uber.org/fx"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain"
2019-11-09 23:00:22 +00:00
"github.com/filecoin-project/lotus/chain/blocksync"
"github.com/filecoin-project/lotus/chain/deals"
2019-11-25 04:45:13 +00:00
"github.com/filecoin-project/lotus/chain/gen"
2019-11-08 20:11:56 +00:00
"github.com/filecoin-project/lotus/chain/market"
2019-12-01 23:11:43 +00:00
"github.com/filecoin-project/lotus/chain/messagepool"
"github.com/filecoin-project/lotus/chain/metrics"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/hello"
"github.com/filecoin-project/lotus/node/impl"
"github.com/filecoin-project/lotus/node/modules"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/modules/helpers"
"github.com/filecoin-project/lotus/node/modules/lp2p"
"github.com/filecoin-project/lotus/node/modules/testing"
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/paych"
2019-10-17 08:57:56 +00:00
"github.com/filecoin-project/lotus/peermgr"
retrievalmarket "github.com/filecoin-project/lotus/retrieval"
"github.com/filecoin-project/lotus/retrieval/discovery"
"github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/sectorblocks"
)
2019-07-04 15:50:48 +00:00
// special is a type used to give keys to modules which
// can't really be identified by the returned type
type special struct{ id int }
//nolint:golint
2019-07-04 15:50:48 +00:00
var (
DefaultTransportsKey = special{0} // Libp2p option
PNetKey = special{1} // Option + multiret
DiscoveryHandlerKey = special{2} // Private type
AddrsFactoryKey = special{3} // Libp2p option
SmuxTransportKey = special{4} // Libp2p option
RelayKey = special{5} // Libp2p option
SecurityKey = special{6} // Libp2p option
BaseRoutingKey = special{7} // fx groups + multiret
NatPortMapKey = special{8} // Libp2p option
ConnectionManagerKey = special{9} // Libp2p option
)
type invoke int
2019-07-08 13:36:43 +00:00
//nolint:golint
2019-07-04 15:50:48 +00:00
const (
2019-07-08 13:36:43 +00:00
// libp2p
2019-07-04 20:06:02 +00:00
2019-07-08 13:36:43 +00:00
PstoreAddSelfKeysKey = invoke(iota)
2019-07-04 15:50:48 +00:00
StartListeningKey
2019-10-11 00:31:06 +00:00
BootstrapKey
2019-07-04 15:50:48 +00:00
2019-07-08 13:36:43 +00:00
// filecoin
2019-07-09 22:58:51 +00:00
SetGenesisKey
2019-07-08 14:07:09 +00:00
2019-07-08 13:36:43 +00:00
RunHelloKey
2019-07-08 14:07:09 +00:00
RunBlockSyncKey
2019-10-17 08:57:56 +00:00
RunPeerMgrKey
2019-07-08 14:07:09 +00:00
HandleIncomingBlocksKey
HandleIncomingMessagesKey
2019-07-08 13:36:43 +00:00
2019-08-01 17:12:41 +00:00
RunDealClientKey
RegisterClientValidatorKey
2019-08-14 20:27:10 +00:00
// storage miner
2019-12-04 19:44:15 +00:00
GetParamsKey
2019-08-02 16:25:10 +00:00
HandleDealsKey
2019-08-26 13:45:36 +00:00
HandleRetrievalKey
2019-08-14 20:27:10 +00:00
RunSectorServiceKey
RegisterProviderValidatorKey
2019-08-01 17:12:41 +00:00
// daemon
2019-07-23 22:34:13 +00:00
ExtractApiKey
2019-10-10 11:07:00 +00:00
HeadMetricsKey
2019-12-17 16:28:02 +00:00
RunPeerTaggerKey
2019-07-23 22:34:13 +00:00
2019-07-10 17:28:49 +00:00
SetApiEndpointKey
2019-07-04 15:50:48 +00:00
_nInvokes // keep this last
)
2019-07-10 13:06:04 +00:00
type Settings struct {
2019-07-04 20:06:02 +00:00
// modules is a map of constructors for DI
//
// In most cases the index will be a reflect. Type of element returned by
// the constructor, but for some 'constructors' it's hard to specify what's
// the return type should be (or the constructor returns fx group)
2019-07-04 15:50:48 +00:00
modules map[interface{}]fx.Option
// invokes are separate from modules as they can't be referenced by return
// type, and must be applied in correct order
invokes []fx.Option
nodeType repo.RepoType
2019-07-19 09:23:24 +00:00
2019-07-10 13:06:04 +00:00
Online bool // Online option applied
Config bool // Config option applied
2019-07-04 15:50:48 +00:00
}
2019-07-09 17:03:36 +00:00
func defaults() []Option {
return []Option{
Override(new(helpers.MetricsCtx), context.Background),
Override(new(record.Validator), modules.RecordValidator),
2019-07-24 21:10:27 +00:00
// Filecoin modules
2019-07-09 17:03:36 +00:00
}
2019-07-04 15:50:48 +00:00
}
2019-07-19 09:23:24 +00:00
func libp2p() Option {
2019-07-04 15:50:48 +00:00
return Options(
Override(new(peerstore.Peerstore), pstoremem.NewPeerstore),
Override(DefaultTransportsKey, lp2p.DefaultTransports),
Override(PNetKey, lp2p.PNet),
Override(new(lp2p.RawHost), lp2p.Host),
Override(new(host.Host), lp2p.RoutedHost),
Override(new(lp2p.BaseIpfsRouting), lp2p.DHTRouting(false)),
Override(DiscoveryHandlerKey, lp2p.DiscoveryHandler),
Override(AddrsFactoryKey, lp2p.AddrsFactory(nil, nil)),
Override(SmuxTransportKey, lp2p.SmuxTransport(true)),
Override(RelayKey, lp2p.Relay(true, false)),
Override(SecurityKey, lp2p.Security(true, false)),
Override(BaseRoutingKey, lp2p.BaseRouting),
Override(new(routing.Routing), lp2p.Routing),
Override(NatPortMapKey, lp2p.NatPortMap),
2019-12-17 06:15:06 +00:00
Override(ConnectionManagerKey, lp2p.ConnectionManager(50, 200, 20*time.Second, nil)),
2019-07-04 15:50:48 +00:00
2019-07-08 14:07:09 +00:00
Override(new(*pubsub.PubSub), lp2p.GossipSub()),
2019-07-04 15:50:48 +00:00
Override(PstoreAddSelfKeysKey, lp2p.PstoreAddSelfKeys),
Override(StartListeningKey, lp2p.StartListening(config.DefaultFullNode().Libp2p.ListenAddresses)),
2019-07-19 09:23:24 +00:00
)
}
func isType(t repo.RepoType) func(s *Settings) bool {
return func(s *Settings) bool { return s.nodeType == t }
}
2019-07-19 09:23:24 +00:00
// Online sets up basic libp2p node
func Online() Option {
return Options(
// make sure that online is applied before Config.
// This is important because Config overrides some of Online units
func(s *Settings) error { s.Online = true; return nil },
ApplyIf(func(s *Settings) bool { return s.Config },
Error(errors.New("the Online option must be set before Config option")),
),
2019-07-08 13:36:43 +00:00
2019-07-19 09:23:24 +00:00
libp2p(),
2019-07-08 13:36:43 +00:00
// common
2019-07-19 09:23:24 +00:00
// Full node
2019-07-08 13:36:43 +00:00
2019-11-12 17:59:38 +00:00
ApplyIf(isType(repo.FullNode),
2019-07-23 22:34:13 +00:00
// TODO: Fix offline mode
Override(new(dtypes.BootstrapPeers), modules.BuiltinBootstrap),
2019-07-23 22:34:13 +00:00
Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages),
2019-07-26 04:54:22 +00:00
Override(new(*store.ChainStore), modules.ChainStore),
Override(new(*stmgr.StateManager), stmgr.NewStateManager),
2019-08-12 23:54:53 +00:00
Override(new(*wallet.Wallet), wallet.NewWallet),
2019-07-23 22:34:13 +00:00
Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker),
Override(new(dtypes.ChainGCBlockstore), modules.ChainGCBlockstore),
Override(new(dtypes.ChainExchange), modules.ChainExchange),
Override(new(dtypes.ChainBlockService), modules.ChainBlockservice),
Override(new(dtypes.ClientDAG), testing.MemoryClientDag),
2019-07-08 13:36:43 +00:00
2019-07-19 09:23:24 +00:00
// Filecoin services
2019-11-15 21:35:29 +00:00
Override(new(*chain.Syncer), modules.NewSyncer),
2019-11-09 23:00:22 +00:00
Override(new(*blocksync.BlockSync), blocksync.NewBlockSyncClient),
2019-12-01 23:11:43 +00:00
Override(new(*messagepool.MessagePool), modules.MessagePool),
2019-07-08 13:36:43 +00:00
2019-07-24 22:49:37 +00:00
Override(new(modules.Genesis), modules.ErrorGenesis),
2019-07-19 09:23:24 +00:00
Override(SetGenesisKey, modules.SetGenesis),
Override(new(*hello.Service), hello.NewHelloService),
2019-11-09 23:00:22 +00:00
Override(new(*blocksync.BlockSyncService), blocksync.NewBlockSyncService),
2019-10-17 08:57:56 +00:00
Override(new(*peermgr.PeerMgr), peermgr.NewPeerMgr),
2019-07-19 09:23:24 +00:00
Override(RunHelloKey, modules.RunHello),
Override(RunBlockSyncKey, modules.RunBlockSync),
2019-10-17 08:57:56 +00:00
Override(RunPeerMgrKey, modules.RunPeerMgr),
2019-07-19 09:23:24 +00:00
Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks),
2019-08-01 17:12:41 +00:00
2019-08-26 13:45:36 +00:00
Override(new(*discovery.Local), discovery.NewLocal),
Override(new(retrievalmarket.PeerResolver), modules.RetrievalResolver),
2019-08-26 13:45:36 +00:00
Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient),
Override(new(dtypes.ClientDealStore), modules.NewClientDealStore),
Override(new(dtypes.ClientDataTransfer), modules.NewClientDAGServiceDataTransfer),
Override(new(*deals.ClientRequestValidator), deals.NewClientRequestValidator),
2019-08-01 17:12:41 +00:00
Override(new(*deals.Client), deals.NewClient),
Override(RegisterClientValidatorKey, modules.RegisterClientValidator),
2019-08-01 17:12:41 +00:00
Override(RunDealClientKey, modules.RunDealClient),
2019-09-16 16:40:26 +00:00
Override(new(*paych.Store), paych.NewStore),
Override(new(*paych.Manager), paych.NewManager),
2019-11-08 17:15:38 +00:00
Override(new(*market.FundMgr), market.NewFundMgr),
2019-07-19 09:23:24 +00:00
),
// Storage miner
2019-11-12 17:59:38 +00:00
ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner },
Override(new(*sectorbuilder.SectorBuilder), modules.SectorBuilder),
Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks),
2019-11-07 18:22:59 +00:00
Override(new(storage.TicketFn), modules.SealTicketGen),
Override(new(*storage.Miner), modules.StorageMiner),
2019-08-02 16:25:10 +00:00
feat(datatransfer): implement and extract feat(datatransfer): setup implementation path Sets up a path to implementation, offering both the dagservice implementation and a future graphsync implement, establishes message interfaces and network layer, and isolates the datatransfer module from the app WIP using CBOR encoding for dataxfermsg * Bring cbor-gen stuff into datatransfer package * make transferRequest private struct * add transferResponse + funcs * Rename VoucherID to VoucherType * more tests passing WIP trying out some stuff * Embed request/response in message so all the interfaces work AND the CBOR unmarshaling works: this is more like the spec anyway * get rid of pb stuff all message tests passing, some others in datatransfer Some cleanup for PR Cleanup for PR, clarifying and additional comments mod tidy Respond to PR comments: * Make DataTransferRequest/Response be returned in from Net * Regenerate cbor_gen and fix the generator caller so it works better * Please the linters Fix tests Initiate push and pull requests (#536) * add issue link for data TransferID generation * comment out failing but not relevant tests * finish voucher rename from Identifier --> Type tests passing cleanup for PR remove unused fmt import in graphsync_test a better reflection send data transfer response other tests passing feat(datatransfer): milestone 2 infrastructure Setup test path for all tickets for milestone 2 responses alert subscribers when request is not accepted (#607) Graphsync response is scheduled when a valid push request is received (#625) fix(datatransfer): fix tests fix an error with read buffers in tests fix(deps): fix go.sum Feat/dt graphsync pullreqs (#627) * graphsync responses to pull requests Feat/dt initiator cleanup (#645) * ChannelID.To --> ChannelID.Initiator * We now store our peer ID (from host.ID()) so it can be used when creating ChannelIDs. * InProgressChannels returns all of impl.channels, currently just for testing * Implements go-data-transfer issue * Some assertions were changed based on the above. * Renamed some variables and added some assertions based on the new understanding * Updated SHA for graphsync module * Updated fakeGraphSync test structs to use new interfaces from new SHA above Techdebt/dt split graphsync impl receiver (#651) * Split up graphsyncImpl and graphsyncReceiver * rename graphsync to utils DTM sends data over graphsync for validated push requests (#665) * create channels when a request is received. register push request hook with graphsync. fix tests. * better NewReaders * use mutex lock around impl.channels access * fix(datatransfer): fix test uncertainty * fix a data race and also don't use random bytes in basic block which can fail * privatize 3 funcs with @hannahhoward Feat/dt gs pullrequests (#693) * Implements DTM Sends Data Over Graphsync For Validated Pull Requests * rename a field in a test struct * refactor a couple of private functions (one was refactored out of existence) Feat/dt subscribe, file Xfer round trip (#720) Implements the rest of Subscriber Is Notified When Request Completed #24: * send a graphsync message within a go func and consume responses until error or transfer is complete. * notify subscribers of results. * Rename datatransfer.Event to EventCode. * datatransfer.Event is now a struct that includes a message and a timestamp as well as the Event.Code int, formerly Event, update all uses * Add extension data to graphsync request hook, gsReq * rename sendRequest to sendDtRequest, to distinguish it from sendGsRequest, where Dt = datatransfer, Gs = graphsync * use a mutex lock for last transfer ID * obey the linter Don't respond with error in gsReqRcdHook when we can't find the datatransfer extension. (#754) * update to correct graphsync version, update tests & code to call the new graphsync hooks * getExtensionData returns an empty struct + nil if we can't find our extension * Don't respond with error when we can't find the extension. * Test for same * mod tidy minor fix to go.sum feat(datatransfer): switch to graphsync implementation Move over to real graphsync implementation of data transfer, add constructors for graphsync instances on client and miner side fix(datatransfer): Fix validators Validators were checking payload cid against commP -- which are not the same any more. Added a payloadCid to client deal to maintain the record, fixed validator logic Feat/dt extraction use go-fil-components/datatransfer (#770) * Initial commit after changing to go-fil-components/datatransfer * blow away the datatransfer dir * use go-fil-components master after its PR #1 was merged * go mod tidy use a package updates after rebase with master
2019-10-30 02:42:16 +00:00
Override(new(dtypes.StagingBlockstore), modules.StagingBlockstore),
2019-08-06 22:04:21 +00:00
Override(new(dtypes.StagingDAG), modules.StagingDAG),
feat(datatransfer): implement and extract feat(datatransfer): setup implementation path Sets up a path to implementation, offering both the dagservice implementation and a future graphsync implement, establishes message interfaces and network layer, and isolates the datatransfer module from the app WIP using CBOR encoding for dataxfermsg * Bring cbor-gen stuff into datatransfer package * make transferRequest private struct * add transferResponse + funcs * Rename VoucherID to VoucherType * more tests passing WIP trying out some stuff * Embed request/response in message so all the interfaces work AND the CBOR unmarshaling works: this is more like the spec anyway * get rid of pb stuff all message tests passing, some others in datatransfer Some cleanup for PR Cleanup for PR, clarifying and additional comments mod tidy Respond to PR comments: * Make DataTransferRequest/Response be returned in from Net * Regenerate cbor_gen and fix the generator caller so it works better * Please the linters Fix tests Initiate push and pull requests (#536) * add issue link for data TransferID generation * comment out failing but not relevant tests * finish voucher rename from Identifier --> Type tests passing cleanup for PR remove unused fmt import in graphsync_test a better reflection send data transfer response other tests passing feat(datatransfer): milestone 2 infrastructure Setup test path for all tickets for milestone 2 responses alert subscribers when request is not accepted (#607) Graphsync response is scheduled when a valid push request is received (#625) fix(datatransfer): fix tests fix an error with read buffers in tests fix(deps): fix go.sum Feat/dt graphsync pullreqs (#627) * graphsync responses to pull requests Feat/dt initiator cleanup (#645) * ChannelID.To --> ChannelID.Initiator * We now store our peer ID (from host.ID()) so it can be used when creating ChannelIDs. * InProgressChannels returns all of impl.channels, currently just for testing * Implements go-data-transfer issue * Some assertions were changed based on the above. * Renamed some variables and added some assertions based on the new understanding * Updated SHA for graphsync module * Updated fakeGraphSync test structs to use new interfaces from new SHA above Techdebt/dt split graphsync impl receiver (#651) * Split up graphsyncImpl and graphsyncReceiver * rename graphsync to utils DTM sends data over graphsync for validated push requests (#665) * create channels when a request is received. register push request hook with graphsync. fix tests. * better NewReaders * use mutex lock around impl.channels access * fix(datatransfer): fix test uncertainty * fix a data race and also don't use random bytes in basic block which can fail * privatize 3 funcs with @hannahhoward Feat/dt gs pullrequests (#693) * Implements DTM Sends Data Over Graphsync For Validated Pull Requests * rename a field in a test struct * refactor a couple of private functions (one was refactored out of existence) Feat/dt subscribe, file Xfer round trip (#720) Implements the rest of Subscriber Is Notified When Request Completed #24: * send a graphsync message within a go func and consume responses until error or transfer is complete. * notify subscribers of results. * Rename datatransfer.Event to EventCode. * datatransfer.Event is now a struct that includes a message and a timestamp as well as the Event.Code int, formerly Event, update all uses * Add extension data to graphsync request hook, gsReq * rename sendRequest to sendDtRequest, to distinguish it from sendGsRequest, where Dt = datatransfer, Gs = graphsync * use a mutex lock for last transfer ID * obey the linter Don't respond with error in gsReqRcdHook when we can't find the datatransfer extension. (#754) * update to correct graphsync version, update tests & code to call the new graphsync hooks * getExtensionData returns an empty struct + nil if we can't find our extension * Don't respond with error when we can't find the extension. * Test for same * mod tidy minor fix to go.sum feat(datatransfer): switch to graphsync implementation Move over to real graphsync implementation of data transfer, add constructors for graphsync instances on client and miner side fix(datatransfer): Fix validators Validators were checking payload cid against commP -- which are not the same any more. Added a payloadCid to client deal to maintain the record, fixed validator logic Feat/dt extraction use go-fil-components/datatransfer (#770) * Initial commit after changing to go-fil-components/datatransfer * blow away the datatransfer dir * use go-fil-components master after its PR #1 was merged * go mod tidy use a package updates after rebase with master
2019-10-30 02:42:16 +00:00
Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync),
Override(new(retrievalmarket.RetrievalProvider), modules.RetrievalProvider),
Override(new(dtypes.ProviderDealStore), modules.NewProviderDealStore),
Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer),
Override(new(*deals.ProviderRequestValidator), deals.NewProviderRequestValidator),
Override(new(*deals.Provider), deals.NewProvider),
Override(RegisterProviderValidatorKey, modules.RegisterProviderValidator),
2019-08-26 13:45:36 +00:00
Override(HandleRetrievalKey, modules.HandleRetrieval),
2019-12-04 19:44:15 +00:00
Override(GetParamsKey, modules.GetParams),
2019-08-02 16:25:10 +00:00
Override(HandleDealsKey, modules.HandleDeals),
2019-11-25 04:45:13 +00:00
Override(new(gen.ElectionPoStProver), storage.NewElectionPoStProver),
Override(new(*miner.Miner), modules.SetupBlockProducer),
),
2019-07-19 09:23:24 +00:00
)
}
2019-07-24 00:58:31 +00:00
func StorageMiner(out *api.StorageMiner) Option {
2019-07-19 09:23:24 +00:00
return Options(
ApplyIf(func(s *Settings) bool { return s.Config },
Error(errors.New("the StorageMiner option must be set before Config option")),
),
ApplyIf(func(s *Settings) bool { return s.Online },
Error(errors.New("the StorageMiner option must be set before Online option")),
),
func(s *Settings) error {
2019-11-12 17:59:38 +00:00
s.nodeType = repo.StorageMiner
2019-07-19 09:23:24 +00:00
return nil
},
2019-07-24 00:58:31 +00:00
func(s *Settings) error {
resAPI := &impl.StorageMinerAPI{}
2019-07-24 00:58:31 +00:00
s.invokes[ExtractApiKey] = fx.Extract(resAPI)
*out = resAPI
return nil
},
2019-07-04 15:50:48 +00:00
)
}
2019-07-10 13:06:04 +00:00
// Config sets up constructors based on the provided Config
func ConfigCommon(cfg *config.Common) Option {
2019-07-04 15:50:48 +00:00
return Options(
2019-07-10 13:06:04 +00:00
func(s *Settings) error { s.Config = true; return nil },
2019-07-04 15:50:48 +00:00
Override(SetApiEndpointKey, func(lr repo.LockedRepo) error {
apima, err := multiaddr.NewMultiaddr(cfg.API.ListenAddress)
if err != nil {
return err
}
return lr.SetAPIEndpoint(apima)
}),
2019-07-10 13:06:04 +00:00
ApplyIf(func(s *Settings) bool { return s.Online },
2019-07-04 15:50:48 +00:00
Override(StartListeningKey, lp2p.StartListening(cfg.Libp2p.ListenAddresses)),
2019-12-17 16:09:43 +00:00
Override(ConnectionManagerKey, lp2p.ConnectionManager(
cfg.Libp2p.ConnMgrLow,
cfg.Libp2p.ConnMgrHigh,
time.Duration(cfg.Libp2p.ConnMgrGrace),
cfg.Libp2p.ProtectedPeers)),
2019-10-11 02:45:45 +00:00
ApplyIf(func(s *Settings) bool { return len(cfg.Libp2p.BootstrapPeers) > 0 },
Override(new(dtypes.BootstrapPeers), modules.ConfigBootstrap(cfg.Libp2p.BootstrapPeers)),
),
2019-07-04 15:50:48 +00:00
),
)
}
2019-11-12 17:59:38 +00:00
func ConfigFullNode(c interface{}) Option {
cfg, ok := c.(*config.FullNode)
if !ok {
return Error(xerrors.Errorf("invalid config from repo, got: %T", c))
2019-07-10 15:38:35 +00:00
}
2019-11-12 17:59:38 +00:00
return Options(
ConfigCommon(&cfg.Common),
2019-12-11 15:08:50 +00:00
If(cfg.Metrics.HeadNotifs,
Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)),
),
2019-11-20 20:31:00 +00:00
If(cfg.Metrics.PubsubTracing,
Override(new(*pubsub.PubSub), lp2p.GossipSub(lp2p.PubsubTracer())),
),
2019-11-12 17:59:38 +00:00
)
}
2019-11-12 17:59:38 +00:00
func ConfigStorageMiner(c interface{}, lr repo.LockedRepo) Option {
cfg, ok := c.(*config.StorageMiner)
if !ok {
return Error(xerrors.Errorf("invalid config from repo, got: %T", c))
2019-07-10 15:38:35 +00:00
}
2019-11-12 17:59:38 +00:00
path := cfg.SectorBuilder.Path
if path == "" {
path = lr.Path()
}
return Options(
ConfigCommon(&cfg.Common),
Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig(path,
cfg.SectorBuilder.WorkerCount,
cfg.SectorBuilder.DisableLocalPreCommit,
cfg.SectorBuilder.DisableLocalCommit)),
2019-11-12 17:59:38 +00:00
)
}
2019-07-10 15:38:35 +00:00
func Repo(r repo.Repo) Option {
return func(settings *Settings) error {
lr, err := r.Lock(settings.nodeType)
if err != nil {
return err
}
c, err := lr.Config()
if err != nil {
return err
}
2019-07-10 17:36:17 +00:00
return Options(
Override(new(repo.LockedRepo), modules.LockedRepo(lr)), // module handles closing
2019-07-16 16:02:51 +00:00
2019-11-12 17:59:38 +00:00
ApplyIf(isType(repo.FullNode), ConfigFullNode(c)),
ApplyIf(isType(repo.StorageMiner), ConfigStorageMiner(c, lr)),
Override(new(dtypes.MetadataDS), modules.Datastore),
Override(new(dtypes.ChainBlockstore), modules.ChainBlockstore),
2019-07-23 20:23:44 +00:00
Override(new(dtypes.ClientFilestore), modules.ClientFstore),
Override(new(dtypes.ClientBlockstore), modules.ClientBlockstore),
Override(new(dtypes.ClientDAG), modules.ClientDAG),
feat(datatransfer): implement and extract feat(datatransfer): setup implementation path Sets up a path to implementation, offering both the dagservice implementation and a future graphsync implement, establishes message interfaces and network layer, and isolates the datatransfer module from the app WIP using CBOR encoding for dataxfermsg * Bring cbor-gen stuff into datatransfer package * make transferRequest private struct * add transferResponse + funcs * Rename VoucherID to VoucherType * more tests passing WIP trying out some stuff * Embed request/response in message so all the interfaces work AND the CBOR unmarshaling works: this is more like the spec anyway * get rid of pb stuff all message tests passing, some others in datatransfer Some cleanup for PR Cleanup for PR, clarifying and additional comments mod tidy Respond to PR comments: * Make DataTransferRequest/Response be returned in from Net * Regenerate cbor_gen and fix the generator caller so it works better * Please the linters Fix tests Initiate push and pull requests (#536) * add issue link for data TransferID generation * comment out failing but not relevant tests * finish voucher rename from Identifier --> Type tests passing cleanup for PR remove unused fmt import in graphsync_test a better reflection send data transfer response other tests passing feat(datatransfer): milestone 2 infrastructure Setup test path for all tickets for milestone 2 responses alert subscribers when request is not accepted (#607) Graphsync response is scheduled when a valid push request is received (#625) fix(datatransfer): fix tests fix an error with read buffers in tests fix(deps): fix go.sum Feat/dt graphsync pullreqs (#627) * graphsync responses to pull requests Feat/dt initiator cleanup (#645) * ChannelID.To --> ChannelID.Initiator * We now store our peer ID (from host.ID()) so it can be used when creating ChannelIDs. * InProgressChannels returns all of impl.channels, currently just for testing * Implements go-data-transfer issue * Some assertions were changed based on the above. * Renamed some variables and added some assertions based on the new understanding * Updated SHA for graphsync module * Updated fakeGraphSync test structs to use new interfaces from new SHA above Techdebt/dt split graphsync impl receiver (#651) * Split up graphsyncImpl and graphsyncReceiver * rename graphsync to utils DTM sends data over graphsync for validated push requests (#665) * create channels when a request is received. register push request hook with graphsync. fix tests. * better NewReaders * use mutex lock around impl.channels access * fix(datatransfer): fix test uncertainty * fix a data race and also don't use random bytes in basic block which can fail * privatize 3 funcs with @hannahhoward Feat/dt gs pullrequests (#693) * Implements DTM Sends Data Over Graphsync For Validated Pull Requests * rename a field in a test struct * refactor a couple of private functions (one was refactored out of existence) Feat/dt subscribe, file Xfer round trip (#720) Implements the rest of Subscriber Is Notified When Request Completed #24: * send a graphsync message within a go func and consume responses until error or transfer is complete. * notify subscribers of results. * Rename datatransfer.Event to EventCode. * datatransfer.Event is now a struct that includes a message and a timestamp as well as the Event.Code int, formerly Event, update all uses * Add extension data to graphsync request hook, gsReq * rename sendRequest to sendDtRequest, to distinguish it from sendGsRequest, where Dt = datatransfer, Gs = graphsync * use a mutex lock for last transfer ID * obey the linter Don't respond with error in gsReqRcdHook when we can't find the datatransfer extension. (#754) * update to correct graphsync version, update tests & code to call the new graphsync hooks * getExtensionData returns an empty struct + nil if we can't find our extension * Don't respond with error when we can't find the extension. * Test for same * mod tidy minor fix to go.sum feat(datatransfer): switch to graphsync implementation Move over to real graphsync implementation of data transfer, add constructors for graphsync instances on client and miner side fix(datatransfer): Fix validators Validators were checking payload cid against commP -- which are not the same any more. Added a payloadCid to client deal to maintain the record, fixed validator logic Feat/dt extraction use go-fil-components/datatransfer (#770) * Initial commit after changing to go-fil-components/datatransfer * blow away the datatransfer dir * use go-fil-components master after its PR #1 was merged * go mod tidy use a package updates after rebase with master
2019-10-30 02:42:16 +00:00
Override(new(dtypes.ClientGraphsync), modules.ClientGraphsync),
Override(new(ci.PrivKey), lp2p.PrivKey),
Override(new(ci.PubKey), ci.PrivKey.GetPublic),
Override(new(peer.ID), peer.IDFromPublicKey),
2019-07-23 20:23:44 +00:00
Override(new(types.KeyStore), modules.KeyStore),
Override(new(*dtypes.APIAlg), modules.APISecret),
)(settings)
}
2019-07-10 15:38:35 +00:00
}
2019-07-24 00:09:34 +00:00
func FullAPI(out *api.FullNode) Option {
2019-07-23 22:34:13 +00:00
return func(s *Settings) error {
resAPI := &impl.FullNodeAPI{}
2019-07-23 22:34:13 +00:00
s.invokes[ExtractApiKey] = fx.Extract(resAPI)
*out = resAPI
return nil
}
}
2019-09-17 14:23:08 +00:00
type StopFunc func(context.Context) error
2019-07-02 13:05:43 +00:00
// New builds and starts new Filecoin node
2019-09-17 14:23:08 +00:00
func New(ctx context.Context, opts ...Option) (StopFunc, error) {
2019-07-10 13:06:04 +00:00
settings := Settings{
modules: map[interface{}]fx.Option{},
invokes: make([]fx.Option, _nInvokes),
2019-11-12 17:59:38 +00:00
nodeType: repo.FullNode,
2019-07-04 15:50:48 +00:00
}
2019-07-04 20:06:02 +00:00
// apply module options in the right order
2019-07-09 17:03:36 +00:00
if err := Options(Options(defaults()...), Options(opts...))(&settings); err != nil {
2019-09-17 14:23:08 +00:00
return nil, err
2019-07-04 15:50:48 +00:00
}
2019-07-04 20:06:02 +00:00
// gather constructors for fx.Options
2019-07-04 15:50:48 +00:00
ctors := make([]fx.Option, 0, len(settings.modules))
for _, opt := range settings.modules {
ctors = append(ctors, opt)
}
2019-07-04 20:06:02 +00:00
// fill holes in invokes for use in fx.Options
2019-07-04 15:50:48 +00:00
for i, opt := range settings.invokes {
if opt == nil {
settings.invokes[i] = fx.Options()
}
}
app := fx.New(
2019-07-04 15:50:48 +00:00
fx.Options(ctors...),
fx.Options(settings.invokes...),
2019-07-09 17:03:36 +00:00
fx.NopLogger,
)
2019-07-04 20:06:02 +00:00
// TODO: we probably should have a 'firewall' for Closing signal
// on this context, and implement closing logic through lifecycles
// correctly
if err := app.Start(ctx); err != nil {
// comment fx.NopLogger few lines above for easier debugging
2019-09-17 14:23:08 +00:00
return nil, err
}
2019-09-17 14:23:08 +00:00
return app.Stop, nil
}
// In-memory / testing
2019-10-23 11:02:00 +00:00
func Test() Option {
2019-07-04 15:50:48 +00:00
return Options(
2019-10-23 11:02:00 +00:00
Unset(RunPeerMgrKey),
Unset(new(*peermgr.PeerMgr)),
)
}