cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

refactor indexprovider libp2p host connection to fullnode with meshcreator

Browse Source
This commit is contained in:
Anton Evangelatov
2022-02-03 15:44:18 +01:00
parent 20fc5ffa38
commit 595ad44ee7
16 changed files with 122 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/builder_miner.go
+4 -2
View File
@@ -26,6 +26,7 @@ import (
sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
"github.com/filecoin-project/lotus/markets/dagstore"
"github.com/filecoin-project/lotus/markets/dealfilter"
"github.com/filecoin-project/lotus/markets/idxprov"
"github.com/filecoin-project/lotus/markets/retrievaladapter"
"github.com/filecoin-project/lotus/markets/sectoraccessor"
"github.com/filecoin-project/lotus/markets/storageadapter"
@@ -168,8 +169,9 @@ func ConfigStorageMiner(c interface{}) Option {
Override(new(dtypes.ProviderTransferNetwork), modules.NewProviderTransferNetwork),
Override(new(dtypes.ProviderTransport), modules.NewProviderTransport),
Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDataTransfer),
Override(new(*modules.IdxProvHost), modules.IndexerProviderHost(cfg.IndexerProvider)),
Override(new(provider.Interface), modules.IndexerProvider(cfg.IndexerProvider)),
Override(new(idxprov.MeshCreator), idxprov.NewMeshCreator),
Override(new(idxprov.Host), modules.IndexProviderHost(cfg.IndexProvider)),
Override(new(provider.Interface), modules.IndexProvider(cfg.IndexProvider)),
Override(new(*storedask.StoredAsk), modules.NewStorageAsk),
Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(cfg.Dealmaking, nil)),
Override(new(storagemarket.StorageProvider), modules.StorageProvider),
node/config/def.go
+2 -2
View File
@@ -181,7 +181,7 @@ func DefaultStorageMiner() *StorageMiner {
},
},
IndexerProvider: IndexerProviderConfig{
IndexProvider: IndexProviderConfig{
ListenAddresses: []string{
"/ip4/0.0.0.0/tcp/0",
"/ip6/::/tcp/0",
@@ -234,7 +234,7 @@ func DefaultStorageMiner() *StorageMiner {
// TODO: Remove hardcoded defaults once provider library exposes them.
// See: https://github.com/filecoin-project/index-provider/issues/108
cfg.IndexerProvider.Ingest = ipconfig.NewIngest()
cfg.IndexProvider.Ingest = ipconfig.NewIngest()
cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http"
cfg.Common.API.RemoteListenAddress = "127.0.0.1:2345"
node/config/def_test.go
+1 -1
View File
@@ -55,5 +55,5 @@ func TestDefaultMinerRoundtrip(t *testing.T) {
func TestDefaultStorageMiner_SetsIndexIngestTopic(t *testing.T) {
subject := DefaultStorageMiner()
require.Equal(t, "/indexer/ingest/mainnet", subject.IndexerProvider.PubSubTopic)
require.Equal(t, "/indexer/ingest/mainnet", subject.IndexProvider.PubSubTopic)
}
node/config/doc_gen.go
+3 -3
View File
@@ -358,7 +358,7 @@ see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-f
Comment: ``,
},
},
"IndexerProviderConfig": []DocField{
"IndexProviderConfig": []DocField{
{
Name: "ListenAddresses",
Type: "[]string",
@@ -858,8 +858,8 @@ Default is 20 (about once a week).`,
Comment: ``,
},
{
Name: "IndexerProvider",
Type: "IndexerProviderConfig",
Name: "IndexProvider",
Type: "IndexProviderConfig",
Comment: ``,
},
node/config/types.go
+9 -9
View File
@@ -45,14 +45,14 @@ type Backup struct {
type StorageMiner struct {
Common
Subsystems MinerSubsystemConfig
Dealmaking DealmakingConfig
IndexerProvider IndexerProviderConfig
Sealing SealingConfig
Storage sectorstorage.SealerConfig
Fees MinerFeeConfig
Addresses MinerAddressConfig
DAGStore DAGStoreConfig
Subsystems MinerSubsystemConfig
Dealmaking DealmakingConfig
IndexProvider IndexProviderConfig
Sealing SealingConfig
Storage sectorstorage.SealerConfig
Fees MinerFeeConfig
Addresses MinerAddressConfig
DAGStore DAGStoreConfig
}
type DAGStoreConfig struct {
@@ -161,7 +161,7 @@ type DealmakingConfig struct {
RetrievalPricing *RetrievalPricing
}
type IndexerProviderConfig struct {
type IndexProviderConfig struct {
config.Ingest
// Binding address for the libp2p host - 0 means random port.
node/modules/storageminer.go
+5 -4
View File
@@ -62,6 +62,7 @@ import (
"github.com/filecoin-project/lotus/journal"
"github.com/filecoin-project/lotus/markets"
"github.com/filecoin-project/lotus/markets/dagstore"
"github.com/filecoin-project/lotus/markets/idxprov"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
"github.com/filecoin-project/lotus/markets/pricing"
lotusminer "github.com/filecoin-project/lotus/miner"
@@ -590,8 +591,7 @@ func StorageProvider(minerAddress dtypes.MinerAddress,
spn storagemarket.StorageProviderNode,
df dtypes.StorageDealFilter,
dsw *dagstore.Wrapper,
fullnodeApi v1api.FullNode,
idxProvHost *IdxProvHost,
meshCreator idxprov.MeshCreator,
) (storagemarket.StorageProvider, error) {
net := smnet.NewFromLibp2pHost(h)
@@ -622,8 +622,9 @@ func StorageProvider(minerAddress dtypes.MinerAddress,
spn,
address.Address(minerAddress),
storedAsk,
fullnodeApi,
idxProvHost.Host,
meshCreator,
//fullnodeApi,
//idxProvHost,
opt,
)
}
node/modules/storageminer_idxprov.go
+12 -15
View File
@@ -29,6 +29,7 @@ import (
"github.com/libp2p/go-libp2p-core/host"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/markets/idxprov"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/modules/dtypes"
@@ -48,28 +49,24 @@ type IdxProv struct {
peerstore.Peerstore
}
type IdxProvHost struct {
host.Host
}
func IndexerProviderHost(cfg config.IndexerProviderConfig) func(IdxProv) (*IdxProvHost, error) {
return func(args IdxProv) (*IdxProvHost, error) {
func IndexProviderHost(cfg config.IndexProviderConfig) func(IdxProv) (idxprov.Host, error) {
return func(args IdxProv) (idxprov.Host, error) {
pkey := args.Peerstore.PrivKey(args.PeerID)
if pkey == nil {
return nil, fmt.Errorf("missing private key for node ID: %s", args.PeerID.Pretty())
}
h, err := createIndexerProviderHost(args.MetricsCtx, args.Lifecycle, pkey, args.Peerstore, cfg.ListenAddresses, cfg.AnnounceAddresses)
h, err := createIndexProviderHost(args.MetricsCtx, args.Lifecycle, pkey, args.Peerstore, cfg.ListenAddresses, cfg.AnnounceAddresses)
if err != nil {
return nil, xerrors.Errorf("creating indexer provider host: %w", err)
}
return &IdxProvHost{h}, nil
return h, nil
}
}
func IndexerProvider(cfg config.IndexerProviderConfig) func(params IdxProv, marketHost host.Host, h *IdxProvHost) (provider.Interface, error) {
return func(args IdxProv, marketHost host.Host, h *IdxProvHost) (provider.Interface, error) {
func IndexProvider(cfg config.IndexProviderConfig) func(params IdxProv, marketHost host.Host, h idxprov.Host) (provider.Interface, error) {
return func(args IdxProv, marketHost host.Host, h idxprov.Host) (provider.Interface, error) {
ipds := namespace.Wrap(args.Datastore, datastore.NewKey("/indexer-provider"))
pkey := args.Peerstore.PrivKey(args.PeerID)
@@ -77,7 +74,7 @@ func IndexerProvider(cfg config.IndexerProviderConfig) func(params IdxProv, mark
return nil, fmt.Errorf("missing private key for node ID: %s", args.PeerID.Pretty())
}
dt, err := newIndexerProviderDataTransfer(cfg, args.MetricsCtx, args.Lifecycle, args.Repo, h, ipds)
dt, err := newIndexProviderDataTransfer(cfg, args.MetricsCtx, args.Lifecycle, args.Repo, h, ipds)
if err != nil {
return nil, err
}
@@ -111,7 +108,7 @@ func IndexerProvider(cfg config.IndexerProviderConfig) func(params IdxProv, mark
}
}
func createIndexerProviderHost(mctx helpers.MetricsCtx, lc fx.Lifecycle, pkey ci.PrivKey, pstore peerstore.Peerstore, listenAddrs []string, announceAddrs []string) (host.Host, error) {
func createIndexProviderHost(mctx helpers.MetricsCtx, lc fx.Lifecycle, pkey ci.PrivKey, pstore peerstore.Peerstore, listenAddrs []string, announceAddrs []string) (host.Host, error) {
addrsFactory, err := lp2p.MakeAddrsFactory(announceAddrs, nil)
if err != nil {
return nil, err
@@ -140,11 +137,11 @@ func createIndexerProviderHost(mctx helpers.MetricsCtx, lc fx.Lifecycle, pkey ci
return h, nil
}
func newIndexerProviderDataTransfer(cfg config.IndexerProviderConfig, mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, h host.Host, ds datastore.Batching) (datatransfer.Manager, error) {
func newIndexProviderDataTransfer(cfg config.IndexProviderConfig, mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, h host.Host, ds datastore.Batching) (datatransfer.Manager, error) {
net := dtnet.NewFromLibp2pHost(h)
// Set up graphsync
gs := newIndexerProviderGraphsync(cfg, mctx, lc, h)
gs := newIndexProviderGraphsync(cfg, mctx, lc, h)
// Set up data transfer
dtDs := namespace.Wrap(ds, datastore.NewKey("/datatransfer/transfers"))
@@ -168,7 +165,7 @@ func newIndexerProviderDataTransfer(cfg config.IndexerProviderConfig, mctx helpe
return dt, nil
}
func newIndexerProviderGraphsync(cfg config.IndexerProviderConfig, mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host) graphsync.GraphExchange {
func newIndexProviderGraphsync(cfg config.IndexProviderConfig, mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host) graphsync.GraphExchange {
graphsyncNetwork := gsnet.NewFromLibp2pHost(h)
return graphsyncimpl.New(helpers.LifecycleCtx(mctx, lc),
graphsyncNetwork,