diff --git a/node/builder.go b/node/builder.go index 4466b39c2..aaaa1247e 100644 --- a/node/builder.go +++ b/node/builder.go @@ -8,14 +8,6 @@ import ( metricsi "github.com/ipfs/go-metrics-interface" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/chain" - "github.com/filecoin-project/lotus/chain/exchange" - rpcstmgr "github.com/filecoin-project/lotus/chain/stmgr/rpc" - "github.com/filecoin-project/lotus/chain/store" - "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/node/hello" "github.com/filecoin-project/lotus/system" logging "github.com/ipfs/go-log/v2" @@ -33,52 +25,21 @@ import ( "go.uber.org/fx" "golang.org/x/xerrors" - "github.com/filecoin-project/go-fil-markets/discovery" - discoveryimpl "github.com/filecoin-project/go-fil-markets/discovery/impl" - "github.com/filecoin-project/go-fil-markets/retrievalmarket" - "github.com/filecoin-project/go-fil-markets/storagemarket" - "github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask" - - storage2 "github.com/filecoin-project/specs-storage/storage" - - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/beacon" - "github.com/filecoin-project/lotus/chain/gen" - "github.com/filecoin-project/lotus/chain/gen/slashfilter" - "github.com/filecoin-project/lotus/chain/market" - "github.com/filecoin-project/lotus/chain/messagepool" - "github.com/filecoin-project/lotus/chain/messagesigner" - "github.com/filecoin-project/lotus/chain/metrics" - "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" - ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" - "github.com/filecoin-project/lotus/chain/wallet/remotewallet" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/peermgr" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" - "github.com/filecoin-project/lotus/markets/dealfilter" "github.com/filecoin-project/lotus/markets/storageadapter" - "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/config" - "github.com/filecoin-project/lotus/node/impl" - "github.com/filecoin-project/lotus/node/impl/common" - "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/modules/lp2p" "github.com/filecoin-project/lotus/node/modules/testing" "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/paychmgr" - "github.com/filecoin-project/lotus/paychmgr/settler" - "github.com/filecoin-project/lotus/storage" - "github.com/filecoin-project/lotus/storage/sectorblocks" ) //nolint:deadcode,varcheck @@ -246,199 +207,6 @@ func isFullOrLiteNode(s *Settings) bool { return s.nodeType == repo.FullNode } func isFullNode(s *Settings) bool { return s.nodeType == repo.FullNode && !s.Lite } func isLiteNode(s *Settings) bool { return s.nodeType == repo.FullNode && s.Lite } -// Chain node provides access to the Filecoin blockchain, by setting up a full -// validator node, or by delegating some actions to other nodes (lite mode) -var ChainNode = Options( - // Full node or lite node - // TODO: Fix offline mode - - // Consensus settings - Override(new(dtypes.DrandSchedule), modules.BuiltinDrandConfig), - Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()), - Override(new(dtypes.NetworkName), modules.NetworkName), - Override(new(modules.Genesis), modules.ErrorGenesis), - Override(new(dtypes.AfterGenesisSet), modules.SetGenesis), - Override(SetGenesisKey, modules.DoSetGenesis), - Override(new(beacon.Schedule), modules.RandomSchedule), - - // Network bootstrap - Override(new(dtypes.BootstrapPeers), modules.BuiltinBootstrap), - Override(new(dtypes.DrandBootstrap), modules.DrandBootstrap), - - // Consensus: crypto dependencies - Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), - - // Consensus: VM - Override(new(vm.SyscallBuilder), vm.Syscalls), - - // Consensus: Chain storage/access - Override(new(*store.ChainStore), modules.ChainStore), - Override(new(*stmgr.StateManager), modules.StateManager), - Override(new(dtypes.ChainBitswap), modules.ChainBitswap), - Override(new(dtypes.ChainBlockService), modules.ChainBlockService), // todo: unused - - // Consensus: Chain sync - - // We don't want the SyncManagerCtor to be used as an fx constructor, but rather as a value. - // It will be called implicitly by the Syncer constructor. - Override(new(chain.SyncManagerCtor), func() chain.SyncManagerCtor { return chain.NewSyncManager }), - Override(new(*chain.Syncer), modules.NewSyncer), - Override(new(exchange.Client), exchange.NewClient), - - // Chain networking - Override(new(*hello.Service), hello.NewHelloService), - Override(new(exchange.Server), exchange.NewServer), - Override(new(*peermgr.PeerMgr), peermgr.NewPeerMgr), - - // Chain mining API dependencies - Override(new(*slashfilter.SlashFilter), modules.NewSlashFilter), - - // Service: Message Pool - Override(new(dtypes.DefaultMaxFeeFunc), modules.NewDefaultMaxFeeFunc), - Override(new(*messagepool.MessagePool), modules.MessagePool), - Override(new(*dtypes.MpoolLocker), new(dtypes.MpoolLocker)), - - // Shared graphsync (markets, serving chain) - Override(new(dtypes.Graphsync), modules.Graphsync(config.DefaultFullNode().Client.SimultaneousTransfers)), - - // Service: Wallet - Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), - Override(new(*wallet.LocalWallet), wallet.NewWallet), - Override(new(wallet.Default), From(new(*wallet.LocalWallet))), - Override(new(api.Wallet), From(new(wallet.MultiWallet))), - - // Service: Payment channels - Override(new(paychmgr.PaychAPI), From(new(modules.PaychAPI))), - Override(new(*paychmgr.Store), modules.NewPaychStore), - Override(new(*paychmgr.Manager), modules.NewManager), - Override(HandlePaymentChannelManagerKey, modules.HandlePaychManager), - Override(SettlePaymentChannelsKey, settler.SettlePaymentChannels), - - // Markets (common) - Override(new(*discoveryimpl.Local), modules.NewLocalDiscovery), - - // Markets (retrieval) - Override(new(discovery.PeerResolver), modules.RetrievalResolver), - Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient), - Override(new(dtypes.ClientDataTransfer), modules.NewClientGraphsyncDataTransfer), - - // Markets (storage) - Override(new(*market.FundManager), market.NewFundManager), - Override(new(dtypes.ClientDatastore), modules.NewClientDatastore), - Override(new(storagemarket.StorageClient), modules.StorageClient), - Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter), - Override(HandleMigrateClientFundsKey, modules.HandleMigrateClientFunds), - - Override(new(*full.GasPriceCache), full.NewGasPriceCache), - - // Lite node API - ApplyIf(isLiteNode, - Override(new(messagepool.Provider), messagepool.NewProviderLite), - Override(new(messagesigner.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))), - Override(new(full.ChainModuleAPI), From(new(api.Gateway))), - Override(new(full.GasModuleAPI), From(new(api.Gateway))), - Override(new(full.MpoolModuleAPI), From(new(api.Gateway))), - Override(new(full.StateModuleAPI), From(new(api.Gateway))), - Override(new(stmgr.StateManagerAPI), rpcstmgr.NewRPCStateManager), - ), - - // Full node API / service startup - ApplyIf(isFullNode, - Override(new(messagepool.Provider), messagepool.NewProvider), - Override(new(messagesigner.MpoolNonceAPI), From(new(*messagepool.MessagePool))), - Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), - Override(new(full.GasModuleAPI), From(new(full.GasModule))), - Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))), - Override(new(full.StateModuleAPI), From(new(full.StateModule))), - Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), - - Override(RunHelloKey, modules.RunHello), - Override(RunChainExchangeKey, modules.RunChainExchange), - Override(RunPeerMgrKey, modules.RunPeerMgr), - Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages), - Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), - ), -) - -var MinerNode = Options( - // API dependencies - Override(new(api.Common), From(new(common.CommonAPI))), - Override(new(sectorstorage.StorageAuth), modules.StorageAuth), - - // Actor config - Override(new(dtypes.MinerAddress), modules.MinerAddress), - Override(new(dtypes.MinerID), modules.MinerID), - Override(new(abi.RegisteredSealProof), modules.SealProofType), - Override(new(dtypes.NetworkName), modules.StorageNetworkName), - - // Sector storage - Override(new(*stores.Index), stores.NewIndex), - Override(new(stores.SectorIndex), From(new(*stores.Index))), - Override(new(stores.LocalStorage), From(new(repo.LockedRepo))), - Override(new(*sectorstorage.Manager), modules.SectorStorage), - Override(new(sectorstorage.SectorManager), From(new(*sectorstorage.Manager))), - Override(new(storiface.WorkerReturn), From(new(sectorstorage.SectorManager))), - - // Sector storage: Proofs - Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), - Override(new(storage2.Prover), From(new(sectorstorage.SectorManager))), - - // Sealing - Override(new(sealing.SectorIDCounter), modules.SectorIDCounter), - Override(GetParamsKey, modules.GetParams), - - // Mining / proving - Override(new(*slashfilter.SlashFilter), modules.NewSlashFilter), - Override(new(*storage.Miner), modules.StorageMiner(config.DefaultStorageMiner().Fees)), - Override(new(*miner.Miner), modules.SetupBlockProducer), - Override(new(gen.WinningPoStProver), storage.NewWinningPoStProver), - - Override(new(*storage.AddressSelector), modules.AddressSelector(nil)), - - // Markets - Override(new(dtypes.StagingMultiDstore), modules.StagingMultiDatastore), - Override(new(dtypes.StagingBlockstore), modules.StagingBlockstore), - Override(new(dtypes.StagingDAG), modules.StagingDAG), - Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync), - Override(new(dtypes.ProviderPieceStore), modules.NewProviderPieceStore), - Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), - - // Markets (retrieval) - Override(new(retrievalmarket.RetrievalProvider), modules.RetrievalProvider), - Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(nil)), - Override(HandleRetrievalKey, modules.HandleRetrieval), - - // Markets (storage) - Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), - Override(new(*storedask.StoredAsk), modules.NewStorageAsk), - Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(nil)), - Override(new(storagemarket.StorageProvider), modules.StorageProvider), - Override(new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{})), - Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(nil, nil)), - Override(HandleMigrateProviderFundsKey, modules.HandleMigrateProviderFunds), - Override(HandleDealsKey, modules.HandleDeals), - - // Config (todo: get a real property system) - Override(new(dtypes.ConsiderOnlineStorageDealsConfigFunc), modules.NewConsiderOnlineStorageDealsConfigFunc), - Override(new(dtypes.SetConsiderOnlineStorageDealsConfigFunc), modules.NewSetConsideringOnlineStorageDealsFunc), - Override(new(dtypes.ConsiderOnlineRetrievalDealsConfigFunc), modules.NewConsiderOnlineRetrievalDealsConfigFunc), - Override(new(dtypes.SetConsiderOnlineRetrievalDealsConfigFunc), modules.NewSetConsiderOnlineRetrievalDealsConfigFunc), - Override(new(dtypes.StorageDealPieceCidBlocklistConfigFunc), modules.NewStorageDealPieceCidBlocklistConfigFunc), - Override(new(dtypes.SetStorageDealPieceCidBlocklistConfigFunc), modules.NewSetStorageDealPieceCidBlocklistConfigFunc), - Override(new(dtypes.ConsiderOfflineStorageDealsConfigFunc), modules.NewConsiderOfflineStorageDealsConfigFunc), - Override(new(dtypes.SetConsiderOfflineStorageDealsConfigFunc), modules.NewSetConsideringOfflineStorageDealsFunc), - Override(new(dtypes.ConsiderOfflineRetrievalDealsConfigFunc), modules.NewConsiderOfflineRetrievalDealsConfigFunc), - Override(new(dtypes.SetConsiderOfflineRetrievalDealsConfigFunc), modules.NewSetConsiderOfflineRetrievalDealsConfigFunc), - Override(new(dtypes.ConsiderVerifiedStorageDealsConfigFunc), modules.NewConsiderVerifiedStorageDealsConfigFunc), - Override(new(dtypes.SetConsiderVerifiedStorageDealsConfigFunc), modules.NewSetConsideringVerifiedStorageDealsFunc), - Override(new(dtypes.ConsiderUnverifiedStorageDealsConfigFunc), modules.NewConsiderUnverifiedStorageDealsConfigFunc), - Override(new(dtypes.SetConsiderUnverifiedStorageDealsConfigFunc), modules.NewSetConsideringUnverifiedStorageDealsFunc), - Override(new(dtypes.SetSealingConfigFunc), modules.NewSetSealConfigFunc), - Override(new(dtypes.GetSealingConfigFunc), modules.NewGetSealConfigFunc), - Override(new(dtypes.SetExpectedSealDurationFunc), modules.NewSetExpectedSealDurationFunc), - Override(new(dtypes.GetExpectedSealDurationFunc), modules.NewGetExpectedSealDurationFunc), -) - // Online sets up basic libp2p node func Online() Option { @@ -457,29 +225,6 @@ func Online() Option { ) } -func StorageMiner(out *api.StorageMiner) Option { - return Options( - ApplyIf(func(s *Settings) bool { return s.Config }, - Error(errors.New("the StorageMiner option must be set before Config option")), - ), - ApplyIf(func(s *Settings) bool { return s.Online }, - Error(errors.New("the StorageMiner option must be set before Online option")), - ), - - func(s *Settings) error { - s.nodeType = repo.StorageMiner - return nil - }, - - func(s *Settings) error { - resAPI := &impl.StorageMinerAPI{} - s.invokes[ExtractApiKey] = fx.Populate(resAPI) - *out = resAPI - return nil - }, - ) -} - // Config sets up constructors based on the provided Config func ConfigCommon(cfg *config.Common) Option { return Options( @@ -518,70 +263,6 @@ func ConfigCommon(cfg *config.Common) Option { ) } -func ConfigFullNode(c interface{}) Option { - cfg, ok := c.(*config.FullNode) - if !ok { - return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) - } - - ipfsMaddr := cfg.Client.IpfsMAddr - return Options( - ConfigCommon(&cfg.Common), - - If(cfg.Client.UseIpfs, - Override(new(dtypes.ClientBlockstore), modules.IpfsClientBlockstore(ipfsMaddr, cfg.Client.IpfsOnlineMode)), - If(cfg.Client.IpfsUseForRetrieval, - Override(new(dtypes.ClientRetrievalStoreManager), modules.ClientBlockstoreRetrievalStoreManager), - ), - ), - Override(new(dtypes.Graphsync), modules.Graphsync(cfg.Client.SimultaneousTransfers)), - - If(cfg.Metrics.HeadNotifs, - Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)), - ), - - If(cfg.Wallet.RemoteBackend != "", - Override(new(*remotewallet.RemoteWallet), remotewallet.SetupRemoteWallet(cfg.Wallet.RemoteBackend)), - ), - If(cfg.Wallet.EnableLedger, - Override(new(*ledgerwallet.LedgerWallet), ledgerwallet.NewWallet), - ), - If(cfg.Wallet.DisableLocal, - Unset(new(*wallet.LocalWallet)), - Override(new(wallet.Default), wallet.NilDefault), - ), - ) -} - -func ConfigStorageMiner(c interface{}) Option { - cfg, ok := c.(*config.StorageMiner) - if !ok { - return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) - } - - return Options( - ConfigCommon(&cfg.Common), - - If(cfg.Dealmaking.Filter != "", - Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(dealfilter.CliStorageDealFilter(cfg.Dealmaking.Filter))), - ), - - If(cfg.Dealmaking.RetrievalFilter != "", - Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(dealfilter.CliRetrievalDealFilter(cfg.Dealmaking.RetrievalFilter))), - ), - - Override(new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(&cfg.Fees, storageadapter.PublishMsgConfig{ - Period: time.Duration(cfg.Dealmaking.PublishMsgPeriod), - MaxDealsPerMsg: cfg.Dealmaking.MaxDealsPerPublishMsg, - })), - Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(&cfg.Fees, &cfg.Dealmaking)), - - Override(new(sectorstorage.SealerConfig), cfg.Storage), - Override(new(*storage.AddressSelector), modules.AddressSelector(&cfg.Addresses)), - Override(new(*storage.Miner), modules.StorageMiner(cfg.Fees)), - ) -} - func Repo(r repo.Repo) Option { return func(settings *Settings) error { lr, err := r.Lock(settings.nodeType) @@ -654,31 +335,6 @@ func Repo(r repo.Repo) Option { } } -type FullOption = Option - -func Lite(enable bool) FullOption { - return func(s *Settings) error { - s.Lite = enable - return nil - } -} - -func FullAPI(out *api.FullNode, fopts ...FullOption) Option { - return Options( - func(s *Settings) error { - s.nodeType = repo.FullNode - return nil - }, - Options(fopts...), - func(s *Settings) error { - resAPI := &impl.FullNodeAPI{} - s.invokes[ExtractApiKey] = fx.Populate(resAPI) - *out = resAPI - return nil - }, - ) -} - type StopFunc func(context.Context) error // New builds and starts new Filecoin node @@ -710,7 +366,7 @@ func New(ctx context.Context, opts ...Option) (StopFunc, error) { fx.Options(ctors...), fx.Options(settings.invokes...), - fx.NopLogger, + //fx.NopLogger, ) // TODO: we probably should have a 'firewall' for Closing signal diff --git a/node/builder_chain.go b/node/builder_chain.go new file mode 100644 index 000000000..6fb5b540e --- /dev/null +++ b/node/builder_chain.go @@ -0,0 +1,212 @@ +package node + +import ( + "go.uber.org/fx" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-fil-markets/discovery" + discoveryimpl "github.com/filecoin-project/go-fil-markets/discovery/impl" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/storagemarket" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain" + "github.com/filecoin-project/lotus/chain/beacon" + "github.com/filecoin-project/lotus/chain/exchange" + "github.com/filecoin-project/lotus/chain/gen/slashfilter" + "github.com/filecoin-project/lotus/chain/market" + "github.com/filecoin-project/lotus/chain/messagepool" + "github.com/filecoin-project/lotus/chain/messagesigner" + "github.com/filecoin-project/lotus/chain/metrics" + "github.com/filecoin-project/lotus/chain/stmgr" + rpcstmgr "github.com/filecoin-project/lotus/chain/stmgr/rpc" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/chain/wallet" + ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" + "github.com/filecoin-project/lotus/chain/wallet/remotewallet" + "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/lib/peermgr" + "github.com/filecoin-project/lotus/markets/storageadapter" + "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/node/hello" + "github.com/filecoin-project/lotus/node/impl" + "github.com/filecoin-project/lotus/node/impl/full" + "github.com/filecoin-project/lotus/node/modules" + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/paychmgr" + "github.com/filecoin-project/lotus/paychmgr/settler" +) + +// Chain node provides access to the Filecoin blockchain, by setting up a full +// validator node, or by delegating some actions to other nodes (lite mode) +var ChainNode = Options( + // Full node or lite node + // TODO: Fix offline mode + + // Consensus settings + Override(new(dtypes.DrandSchedule), modules.BuiltinDrandConfig), + Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()), + Override(new(dtypes.NetworkName), modules.NetworkName), + Override(new(modules.Genesis), modules.ErrorGenesis), + Override(new(dtypes.AfterGenesisSet), modules.SetGenesis), + Override(SetGenesisKey, modules.DoSetGenesis), + Override(new(beacon.Schedule), modules.RandomSchedule), + + // Network bootstrap + Override(new(dtypes.BootstrapPeers), modules.BuiltinBootstrap), + Override(new(dtypes.DrandBootstrap), modules.DrandBootstrap), + + // Consensus: crypto dependencies + Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), + + // Consensus: VM + Override(new(vm.SyscallBuilder), vm.Syscalls), + + // Consensus: Chain storage/access + Override(new(*store.ChainStore), modules.ChainStore), + Override(new(*stmgr.StateManager), modules.StateManager), + Override(new(dtypes.ChainBitswap), modules.ChainBitswap), + Override(new(dtypes.ChainBlockService), modules.ChainBlockService), // todo: unused + + // Consensus: Chain sync + + // We don't want the SyncManagerCtor to be used as an fx constructor, but rather as a value. + // It will be called implicitly by the Syncer constructor. + Override(new(chain.SyncManagerCtor), func() chain.SyncManagerCtor { return chain.NewSyncManager }), + Override(new(*chain.Syncer), modules.NewSyncer), + Override(new(exchange.Client), exchange.NewClient), + + // Chain networking + Override(new(*hello.Service), hello.NewHelloService), + Override(new(exchange.Server), exchange.NewServer), + Override(new(*peermgr.PeerMgr), peermgr.NewPeerMgr), + + // Chain mining API dependencies + Override(new(*slashfilter.SlashFilter), modules.NewSlashFilter), + + // Service: Message Pool + Override(new(dtypes.DefaultMaxFeeFunc), modules.NewDefaultMaxFeeFunc), + Override(new(*messagepool.MessagePool), modules.MessagePool), + Override(new(*dtypes.MpoolLocker), new(dtypes.MpoolLocker)), + + // Shared graphsync (markets, serving chain) + Override(new(dtypes.Graphsync), modules.Graphsync(config.DefaultFullNode().Client.SimultaneousTransfers)), + + // Service: Wallet + Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), + Override(new(*wallet.LocalWallet), wallet.NewWallet), + Override(new(wallet.Default), From(new(*wallet.LocalWallet))), + Override(new(api.Wallet), From(new(wallet.MultiWallet))), + + // Service: Payment channels + Override(new(paychmgr.PaychAPI), From(new(modules.PaychAPI))), + Override(new(*paychmgr.Store), modules.NewPaychStore), + Override(new(*paychmgr.Manager), modules.NewManager), + Override(HandlePaymentChannelManagerKey, modules.HandlePaychManager), + Override(SettlePaymentChannelsKey, settler.SettlePaymentChannels), + + // Markets (common) + Override(new(*discoveryimpl.Local), modules.NewLocalDiscovery), + + // Markets (retrieval) + Override(new(discovery.PeerResolver), modules.RetrievalResolver), + Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient), + Override(new(dtypes.ClientDataTransfer), modules.NewClientGraphsyncDataTransfer), + + // Markets (storage) + Override(new(*market.FundManager), market.NewFundManager), + Override(new(dtypes.ClientDatastore), modules.NewClientDatastore), + Override(new(storagemarket.StorageClient), modules.StorageClient), + Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter), + Override(HandleMigrateClientFundsKey, modules.HandleMigrateClientFunds), + + Override(new(*full.GasPriceCache), full.NewGasPriceCache), + + // Lite node API + ApplyIf(isLiteNode, + Override(new(messagesigner.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))), + Override(new(full.ChainModuleAPI), From(new(api.Gateway))), + Override(new(full.GasModuleAPI), From(new(api.Gateway))), + Override(new(full.MpoolModuleAPI), From(new(api.Gateway))), + Override(new(full.StateModuleAPI), From(new(api.Gateway))), + Override(new(stmgr.StateManagerAPI), rpcstmgr.NewRPCStateManager), + ), + + // Full node API / service startup + ApplyIf(isFullNode, + Override(new(messagesigner.MpoolNonceAPI), From(new(*messagepool.MessagePool))), + Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), + Override(new(full.GasModuleAPI), From(new(full.GasModule))), + Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))), + Override(new(full.StateModuleAPI), From(new(full.StateModule))), + Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), + + Override(RunHelloKey, modules.RunHello), + Override(RunChainExchangeKey, modules.RunChainExchange), + Override(RunPeerMgrKey, modules.RunPeerMgr), + Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages), + Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), + ), +) + +func ConfigFullNode(c interface{}) Option { + cfg, ok := c.(*config.FullNode) + if !ok { + return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) + } + + ipfsMaddr := cfg.Client.IpfsMAddr + return Options( + ConfigCommon(&cfg.Common), + + If(cfg.Client.UseIpfs, + Override(new(dtypes.ClientBlockstore), modules.IpfsClientBlockstore(ipfsMaddr, cfg.Client.IpfsOnlineMode)), + If(cfg.Client.IpfsUseForRetrieval, + Override(new(dtypes.ClientRetrievalStoreManager), modules.ClientBlockstoreRetrievalStoreManager), + ), + ), + Override(new(dtypes.Graphsync), modules.Graphsync(cfg.Client.SimultaneousTransfers)), + + If(cfg.Metrics.HeadNotifs, + Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)), + ), + + If(cfg.Wallet.RemoteBackend != "", + Override(new(*remotewallet.RemoteWallet), remotewallet.SetupRemoteWallet(cfg.Wallet.RemoteBackend)), + ), + If(cfg.Wallet.EnableLedger, + Override(new(*ledgerwallet.LedgerWallet), ledgerwallet.NewWallet), + ), + If(cfg.Wallet.DisableLocal, + Unset(new(*wallet.LocalWallet)), + Override(new(wallet.Default), wallet.NilDefault), + ), + ) +} + +type FullOption = Option + +func Lite(enable bool) FullOption { + return func(s *Settings) error { + s.Lite = enable + return nil + } +} + +func FullAPI(out *api.FullNode, fopts ...FullOption) Option { + return Options( + func(s *Settings) error { + s.nodeType = repo.FullNode + return nil + }, + Options(fopts...), + func(s *Settings) error { + resAPI := &impl.FullNodeAPI{} + s.invokes[ExtractApiKey] = fx.Populate(resAPI) + *out = resAPI + return nil + }, + ) +} diff --git a/node/builder_miner.go b/node/builder_miner.go new file mode 100644 index 000000000..eeb33749c --- /dev/null +++ b/node/builder_miner.go @@ -0,0 +1,202 @@ +package node + +import ( + "errors" + "time" + + "go.uber.org/fx" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/markets/retrievaladapter" + storage2 "github.com/filecoin-project/specs-storage/storage" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/gen" + "github.com/filecoin-project/lotus/chain/gen/slashfilter" + sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" + "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/extern/sector-storage/stores" + "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + "github.com/filecoin-project/lotus/markets/dealfilter" + "github.com/filecoin-project/lotus/markets/storageadapter" + "github.com/filecoin-project/lotus/miner" + "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/node/impl" + "github.com/filecoin-project/lotus/node/impl/common" + "github.com/filecoin-project/lotus/node/modules" + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/lotus/storage/sectorblocks" +) + +var MinerNode = Options( + // API dependencies + Override(new(api.Common), From(new(common.CommonAPI))), + Override(new(sectorstorage.StorageAuth), modules.StorageAuth), + + // Actor config + Override(new(dtypes.MinerAddress), modules.MinerAddress), + Override(new(dtypes.MinerID), modules.MinerID), + Override(new(abi.RegisteredSealProof), modules.SealProofType), + Override(new(dtypes.NetworkName), modules.StorageNetworkName), + + // Mining / proving + Override(new(*storage.AddressSelector), modules.AddressSelector(nil)), +) + +func ConfigStorageMiner(c interface{}) Option { + cfg, ok := c.(*config.StorageMiner) + if !ok { + return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) + } + + return Options( + ConfigCommon(&cfg.Common), + + Override(new(stores.LocalStorage), From(new(repo.LockedRepo))), + Override(new(*stores.Local), modules.LocalStorage), + Override(new(*stores.Remote), modules.RemoteStorage), + + If(!cfg.Subsystems.EnableMining, + If(cfg.Subsystems.EnableSealing, Error(xerrors.Errorf("sealing can only be enabled on a mining node"))), + If(cfg.Subsystems.EnableSectorStorage, Error(xerrors.Errorf("sealing can only be enabled on a mining node"))), + ), + If(cfg.Subsystems.EnableMining, + If(!cfg.Subsystems.EnableSealing, Error(xerrors.Errorf("sealing can't be disabled on a mining node yet"))), + If(!cfg.Subsystems.EnableSectorStorage, Error(xerrors.Errorf("sealing can't be disabled on a mining node yet"))), + + // Sector storage: Proofs + Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), + Override(new(storage2.Prover), From(new(sectorstorage.SectorManager))), + + // Sealing (todo should be under EnableSealing, but storagefsm is currently bundled with storage.Miner) + Override(new(sealing.SectorIDCounter), modules.SectorIDCounter), + Override(GetParamsKey, modules.GetParams), + + Override(new(dtypes.SetSealingConfigFunc), modules.NewSetSealConfigFunc), + Override(new(dtypes.GetSealingConfigFunc), modules.NewGetSealConfigFunc), + + // Mining / proving + Override(new(*slashfilter.SlashFilter), modules.NewSlashFilter), + Override(new(*storage.Miner), modules.StorageMiner(config.DefaultStorageMiner().Fees)), + Override(new(*miner.Miner), modules.SetupBlockProducer), + Override(new(gen.WinningPoStProver), storage.NewWinningPoStProver), + Override(new(*storage.Miner), modules.StorageMiner(cfg.Fees)), + Override(new(sectorblocks.SectorBuilder), From(new(*storage.Miner))), + ), + + If(cfg.Subsystems.EnableSectorStorage, + // Sector storage + Override(new(*stores.Index), stores.NewIndex), + Override(new(stores.SectorIndex), From(new(*stores.Index))), + Override(new(*sectorstorage.Manager), modules.SectorStorage), + Override(new(sectorstorage.Unsealer), From(new(*sectorstorage.Manager))), + Override(new(sectorstorage.SectorManager), From(new(*sectorstorage.Manager))), + Override(new(storiface.WorkerReturn), From(new(sectorstorage.SectorManager))), + ), + + If(!cfg.Subsystems.EnableSectorStorage, + Override(new(modules.MinerStorageService), modules.ConnectStorageService(cfg.Subsystems.SectorIndexApiInfo)), + Override(new(sectorstorage.Unsealer), From(new(modules.MinerStorageService))), + Override(new(sectorblocks.SectorBuilder), From(new(modules.MinerStorageService))), + ), + If(!cfg.Subsystems.EnableSealing, + Override(new(modules.MinerSealingService), modules.ConnectSealingService(cfg.Subsystems.SealerApiInfo)), + Override(new(stores.SectorIndex), From(new(modules.MinerSealingService))), + ), + + If(cfg.Subsystems.EnableStorageMarket, + // Markets + Override(new(dtypes.StagingMultiDstore), modules.StagingMultiDatastore), + Override(new(dtypes.StagingBlockstore), modules.StagingBlockstore), + Override(new(dtypes.StagingDAG), modules.StagingDAG), + Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync), + Override(new(dtypes.ProviderPieceStore), modules.NewProviderPieceStore), + Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), + + // Markets (retrieval deps) + Override(new(*sectorstorage.PieceProvider), sectorstorage.NewPieceProvider), + + // Markets (retrieval) + + Override(new(retrievalmarket.RetrievalProviderNode), retrievaladapter.NewRetrievalProviderNode), + Override(new(rmnet.RetrievalMarketNetwork), modules.RetrievalNetwork), + Override(new(retrievalmarket.RetrievalProvider), modules.RetrievalProvider), + Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(nil)), + Override(HandleRetrievalKey, modules.HandleRetrieval), + + // Markets (storage) + Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), + Override(new(*storedask.StoredAsk), modules.NewStorageAsk), + Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(nil)), + Override(new(storagemarket.StorageProvider), modules.StorageProvider), + Override(new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{})), + Override(HandleMigrateProviderFundsKey, modules.HandleMigrateProviderFunds), + Override(HandleDealsKey, modules.HandleDeals), + + // Config (todo: get a real property system) + Override(new(dtypes.ConsiderOnlineStorageDealsConfigFunc), modules.NewConsiderOnlineStorageDealsConfigFunc), + Override(new(dtypes.SetConsiderOnlineStorageDealsConfigFunc), modules.NewSetConsideringOnlineStorageDealsFunc), + Override(new(dtypes.ConsiderOnlineRetrievalDealsConfigFunc), modules.NewConsiderOnlineRetrievalDealsConfigFunc), + Override(new(dtypes.SetConsiderOnlineRetrievalDealsConfigFunc), modules.NewSetConsiderOnlineRetrievalDealsConfigFunc), + Override(new(dtypes.StorageDealPieceCidBlocklistConfigFunc), modules.NewStorageDealPieceCidBlocklistConfigFunc), + Override(new(dtypes.SetStorageDealPieceCidBlocklistConfigFunc), modules.NewSetStorageDealPieceCidBlocklistConfigFunc), + Override(new(dtypes.ConsiderOfflineStorageDealsConfigFunc), modules.NewConsiderOfflineStorageDealsConfigFunc), + Override(new(dtypes.SetConsiderOfflineStorageDealsConfigFunc), modules.NewSetConsideringOfflineStorageDealsFunc), + Override(new(dtypes.ConsiderOfflineRetrievalDealsConfigFunc), modules.NewConsiderOfflineRetrievalDealsConfigFunc), + Override(new(dtypes.SetConsiderOfflineRetrievalDealsConfigFunc), modules.NewSetConsiderOfflineRetrievalDealsConfigFunc), + Override(new(dtypes.ConsiderVerifiedStorageDealsConfigFunc), modules.NewConsiderVerifiedStorageDealsConfigFunc), + Override(new(dtypes.SetConsiderVerifiedStorageDealsConfigFunc), modules.NewSetConsideringVerifiedStorageDealsFunc), + Override(new(dtypes.ConsiderUnverifiedStorageDealsConfigFunc), modules.NewConsiderUnverifiedStorageDealsConfigFunc), + Override(new(dtypes.SetConsiderUnverifiedStorageDealsConfigFunc), modules.NewSetConsideringUnverifiedStorageDealsFunc), + Override(new(dtypes.SetExpectedSealDurationFunc), modules.NewSetExpectedSealDurationFunc), + Override(new(dtypes.GetExpectedSealDurationFunc), modules.NewGetExpectedSealDurationFunc), + + If(cfg.Dealmaking.Filter != "", + Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(dealfilter.CliStorageDealFilter(cfg.Dealmaking.Filter))), + ), + + If(cfg.Dealmaking.RetrievalFilter != "", + Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(dealfilter.CliRetrievalDealFilter(cfg.Dealmaking.RetrievalFilter))), + ), + Override(new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(&cfg.Fees, storageadapter.PublishMsgConfig{ + Period: time.Duration(cfg.Dealmaking.PublishMsgPeriod), + MaxDealsPerMsg: cfg.Dealmaking.MaxDealsPerPublishMsg, + })), + Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(&cfg.Fees, &cfg.Dealmaking)), + ), + + Override(new(sectorstorage.SealerConfig), cfg.Storage), + Override(new(*storage.AddressSelector), modules.AddressSelector(&cfg.Addresses)), + ) +} + +func StorageMiner(out *api.StorageMiner) Option { + return Options( + ApplyIf(func(s *Settings) bool { return s.Config }, + Error(errors.New("the StorageMiner option must be set before Config option")), + ), + ApplyIf(func(s *Settings) bool { return s.Online }, + Error(errors.New("the StorageMiner option must be set before Online option")), + ), + + func(s *Settings) error { + s.nodeType = repo.StorageMiner + return nil + }, + + func(s *Settings) error { + resAPI := &impl.StorageMinerAPI{} + s.invokes[ExtractApiKey] = fx.Populate(resAPI) + *out = resAPI + return nil + }, + ) +} diff --git a/node/config/def.go b/node/config/def.go index b4cf5e2fa..4013bde6e 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -38,6 +38,7 @@ type Backup struct { type StorageMiner struct { Common + Subsystems MinerSubsystemConfig Dealmaking DealmakingConfig Sealing SealingConfig Storage sectorstorage.SealerConfig @@ -45,6 +46,16 @@ type StorageMiner struct { Addresses MinerAddressConfig } +type MinerSubsystemConfig struct { + EnableMining bool + EnableSealing bool + EnableSectorStorage bool + EnableStorageMarket bool + + SealerApiInfo string // if EnableSealing == false + SectorIndexApiInfo string // if EnableSectorStorage == false +} + type DealmakingConfig struct { ConsiderOnlineStorageDeals bool ConsiderOfflineStorageDeals bool diff --git a/node/modules/storageminer_svc.go b/node/modules/storageminer_svc.go new file mode 100644 index 000000000..8718d236c --- /dev/null +++ b/node/modules/storageminer_svc.go @@ -0,0 +1,71 @@ +package modules + +import ( + "context" + + "github.com/filecoin-project/lotus/storage/sectorblocks" + + "go.uber.org/fx" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/client" + cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/node/modules/helpers" +) + +type MinerSealingService api.StorageMiner +type MinerStorageService api.StorageMiner + +var _ sectorblocks.SectorBuilder = *new(MinerSealingService) + +func connectMinerService(apiInfo string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (api.StorageMiner, error) { + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (api.StorageMiner, error) { + ctx := helpers.LifecycleCtx(mctx, lc) + info := cliutil.ParseApiInfo(apiInfo) + addr, err := info.DialArgs("v0") + if err != nil { + return nil, xerrors.Errorf("could not get DialArgs: %w", err) + } + + log.Infof("Checking api version of %s", addr) + + mapi, closer, err := client.NewStorageMinerRPCV0(ctx, addr, info.AuthHeader()) + if err != nil { + return nil, err + } + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + v, err := mapi.Version(ctx) + if err != nil { + return xerrors.Errorf("checking version: %w", err) + } + + if !v.APIVersion.EqMajorMinor(api.MinerAPIVersion0) { + // TODO(anteva): Is MinerAPIVersion0 correct??? + // we should probably bump it up + return xerrors.Errorf("remote service API version didn't match (expected %s, remote %s)", api.MinerAPIVersion0, v.APIVersion) + } + + return nil + }, + OnStop: func(context.Context) error { + closer() + return nil + }}) + + return mapi, nil + } +} + +func ConnectSealingService(apiInfo string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (MinerSealingService, error) { + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (MinerSealingService, error) { + return connectMinerService(apiInfo)(mctx, lc) + } +} + +func ConnectStorageService(apiInfo string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (MinerStorageService, error) { + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (MinerStorageService, error) { + return connectMinerService(apiInfo)(mctx, lc) + } +}