From 235377585d45c3e1b478aa4518da3f81dc1ea79f Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Thu, 3 Feb 2022 11:51:01 +0000 Subject: [PATCH 1/3] Clarify the purpose of `MaxSimultaneousTransfers` config Clarify the documentation of `MaxSimultaneousTransfers` to state that the "transfer" refers to multihash syncing only and not the data itself. --- documentation/en/default-lotus-miner-config.toml | 4 ++-- node/config/doc_gen.go | 3 +-- node/config/types.go | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/documentation/en/default-lotus-miner-config.toml b/documentation/en/default-lotus-miner-config.toml index 345a2b91c..07328bb77 100644 --- a/documentation/en/default-lotus-miner-config.toml +++ b/documentation/en/default-lotus-miner-config.toml @@ -285,8 +285,8 @@ # env var: LOTUS_INDEXERPROVIDER_ANNOUNCEADDRESSES #AnnounceAddresses = [] - # The maximum number of simultaneous data transfers between the indexers - # and the indexer provider + # The maximum number of simultaneous requests syncing the list of advertised multihashes between + # the indexers and the index provider. # # type: uint64 # env var: LOTUS_INDEXERPROVIDER_MAXSIMULTANEOUSTRANSFERS diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 1b7c20b63..8e7a1a963 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -378,8 +378,7 @@ Format: multiaddress`, Name: "MaxSimultaneousTransfers", Type: "uint64", - Comment: `The maximum number of simultaneous data transfers between the indexers -and the indexer provider`, + Comment: `The maximum number of simultaneous requests syncing the list of advertised multihashes between the indexers and the index provider.`, }, }, "Libp2p": []DocField{ diff --git a/node/config/types.go b/node/config/types.go index 428f1d59b..18f27feea 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -173,8 +173,8 @@ type IndexerProviderConfig struct { // Format: multiaddress AnnounceAddresses []string - // The maximum number of simultaneous data transfers between the indexers - // and the indexer provider + // The maximum number of simultaneous requests syncing the list of advertised multihashes + // between the indexers and the index provider. MaxSimultaneousTransfers uint64 } From 575e6add435b3eac6daab80522cba44b5e827a59 Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Thu, 3 Feb 2022 11:51:01 +0000 Subject: [PATCH 2/3] Rename `IndexerProvider` to `IndexProvider` and add more docs Rename the config section corresponding to indexing to `IndexProvider` for better readability. Update existing docs for better clarity and add docs for config parameters embedded from `index-provider` `Ingest` config library. --- .../en/default-lotus-miner-config.toml | 45 +++++++++++++------ itests/deals_concurrent_test.go | 10 ++--- itests/kit/deals.go | 6 +-- node/builder_miner.go | 2 +- node/config/def.go | 4 +- node/config/def_test.go | 2 +- node/config/doc_gen.go | 21 +++++---- node/config/types.go | 34 +++++++------- node/modules/storageminer_idxprov.go | 14 +++--- 9 files changed, 81 insertions(+), 57 deletions(-) diff --git a/documentation/en/default-lotus-miner-config.toml b/documentation/en/default-lotus-miner-config.toml index 07328bb77..79b4efb87 100644 --- a/documentation/en/default-lotus-miner-config.toml +++ b/documentation/en/default-lotus-miner-config.toml @@ -257,39 +257,58 @@ #Path = "" -[IndexerProvider] - # env var: LOTUS_INDEXERPROVIDER_LINKCACHESIZE +[IndexProvider] + # The maximum number of multihash chunk links that index provider cash can store before + # LRU eviction. If chunks belonging to a single advertisement are larger than the cache can + # hold, the cache is resized to be able to hold all links. The actual disk usaed depends on + # LinkedChunkSize and the length of multihashes. For example, for 128-bit long multihashes + # with the default LinkedChunkSize, and LinkCacheSize the cache size can grow to 256MiB. + # + # type int + # env var: LOTUS_INDEXPROVIDER_LINKCACHESIZE #LinkCacheSize = 1024 - # env var: LOTUS_INDEXERPROVIDER_LINKEDCHUNKSIZE + # The number of multihashes in each chunk of the + # advertised multihash entries linked list. If multihashes are 128-bit, then + # setting LinkedChunkSize = 16384 will result in blocks of 0.25MiB when + # full. + # + # type int + # env var: LOTUS_INDEXPROVIDER_LINKEDCHUNKSIZE #LinkedChunkSize = 16384 - # env var: LOTUS_INDEXERPROVIDER_PUBSUBTOPIC + # The gossipsub topic name used to publish change to the advertised content. + # + # env var: LOTUS_INDEXPROVIDER_PUBSUBTOPIC #PubSubTopic = "/indexer/ingest/mainnet" - # env var: LOTUS_INDEXERPROVIDER_PURGELINKCACHE + # Whether to purge all cached entries on start-up. + # + # env var: LOTUS_INDEXPROVIDER_PURGELINKCACHE #PurgeLinkCache = false - # Binding address for the libp2p host - 0 means random port. - # Format: multiaddress; see https://multiformats.io/multiaddr/ + # Binding address for the libp2p host contacted by indexer nodes to sync the list of advertised + # multihashes. Note that when port is set to 0 a random port is generated at runtime and may be + # different on every restart. The format of the strings specified must conform to multiaddress; + # see https://multiformats.io/multiaddr/ # # type: []string - # env var: LOTUS_INDEXERPROVIDER_LISTENADDRESSES + # env var: LOTUS_INDEXPROVIDER_LISTENADDRESSES #ListenAddresses = ["/ip4/0.0.0.0/tcp/0", "/ip6/::/tcp/0"] - # Addresses to explicitly announce to other peers. If not specified, - # all interface addresses are announced - # Format: multiaddress + # The address the endpoints at which the data associated to the advertised + # multihashes can be retrieved. If not specified, the ListenAddresses are used instead. The format + # of the strings specified must conform to multiaddress; see https://multiformats.io/multiaddr/ # # type: []string - # env var: LOTUS_INDEXERPROVIDER_ANNOUNCEADDRESSES + # env var: LOTUS_INDEXPROVIDER_ANNOUNCEADDRESSES #AnnounceAddresses = [] # The maximum number of simultaneous requests syncing the list of advertised multihashes between # the indexers and the index provider. # # type: uint64 - # env var: LOTUS_INDEXERPROVIDER_MAXSIMULTANEOUSTRANSFERS + # env var: LOTUS_INDEXPROVIDER_MAXSIMULTANEOUSTRANSFERS #MaxSimultaneousTransfers = 20 diff --git a/itests/deals_concurrent_test.go b/itests/deals_concurrent_test.go index 3bc5e4813..624e253b4 100644 --- a/itests/deals_concurrent_test.go +++ b/itests/deals_concurrent_test.go @@ -55,11 +55,11 @@ func TestDealWithMarketAndMinerNode(t *testing.T) { dh := kit.NewDealHarness(t, client, main, market) dh.RunConcurrentDeals(kit.RunConcurrentDealsOpts{ - N: n, - FastRetrieval: fastRetrieval, - CarExport: carExport, - StartEpoch: startEpoch, - IndexerProvider: idxProv, + N: n, + FastRetrieval: fastRetrieval, + CarExport: carExport, + StartEpoch: startEpoch, + IndexProvider: idxProv, }) } diff --git a/itests/kit/deals.go b/itests/kit/deals.go index ca3848c58..44f785d3d 100644 --- a/itests/kit/deals.go +++ b/itests/kit/deals.go @@ -406,7 +406,7 @@ type RunConcurrentDealsOpts struct { CarExport bool StartEpoch abi.ChainEpoch UseCARFileForStorageDeal bool - IndexerProvider *shared_testutil.MockIndexProvider + IndexProvider *shared_testutil.MockIndexProvider } func (dh *DealHarness) RunConcurrentDeals(opts RunConcurrentDealsOpts) { @@ -434,8 +434,8 @@ func (dh *DealHarness) RunConcurrentDeals(opts RunConcurrentDealsOpts) { }) // Check that the storage provider announced the deal to indexers - if opts.IndexerProvider != nil { - notifs := opts.IndexerProvider.GetNotifs() + if opts.IndexProvider != nil { + notifs := opts.IndexProvider.GetNotifs() _, ok := notifs[string(deal.Bytes())] require.True(dh.t, ok) } diff --git a/node/builder_miner.go b/node/builder_miner.go index de22ef8aa..afac1e828 100644 --- a/node/builder_miner.go +++ b/node/builder_miner.go @@ -168,7 +168,7 @@ func ConfigStorageMiner(c interface{}) Option { Override(new(dtypes.ProviderTransferNetwork), modules.NewProviderTransferNetwork), Override(new(dtypes.ProviderTransport), modules.NewProviderTransport), Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDataTransfer), - Override(new(provider.Interface), modules.IndexerProvider(cfg.IndexerProvider)), + Override(new(provider.Interface), modules.IndexProvider(cfg.IndexProvider)), Override(new(*storedask.StoredAsk), modules.NewStorageAsk), Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(cfg.Dealmaking, nil)), Override(new(storagemarket.StorageProvider), modules.StorageProvider), diff --git a/node/config/def.go b/node/config/def.go index d6fedc753..75e2a08a3 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -181,7 +181,7 @@ func DefaultStorageMiner() *StorageMiner { }, }, - IndexerProvider: IndexerProviderConfig{ + IndexProvider: IndexProviderConfig{ ListenAddresses: []string{ "/ip4/0.0.0.0/tcp/0", "/ip6/::/tcp/0", @@ -234,7 +234,7 @@ func DefaultStorageMiner() *StorageMiner { // TODO: Remove hardcoded defaults once provider library exposes them. // See: https://github.com/filecoin-project/index-provider/issues/108 - cfg.IndexerProvider.Ingest = ipconfig.NewIngest() + cfg.IndexProvider.Ingest = ipconfig.NewIngest() cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" cfg.Common.API.RemoteListenAddress = "127.0.0.1:2345" diff --git a/node/config/def_test.go b/node/config/def_test.go index e6ad1a687..c25b255bf 100644 --- a/node/config/def_test.go +++ b/node/config/def_test.go @@ -55,5 +55,5 @@ func TestDefaultMinerRoundtrip(t *testing.T) { func TestDefaultStorageMiner_SetsIndexIngestTopic(t *testing.T) { subject := DefaultStorageMiner() - require.Equal(t, "/indexer/ingest/mainnet", subject.IndexerProvider.PubSubTopic) + require.Equal(t, "/indexer/ingest/mainnet", subject.IndexProvider.PubSubTopic) } diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 8e7a1a963..d7c500e67 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -358,27 +358,30 @@ see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-f Comment: ``, }, }, - "IndexerProviderConfig": []DocField{ + "IndexProviderConfig": []DocField{ { Name: "ListenAddresses", Type: "[]string", - Comment: `Binding address for the libp2p host - 0 means random port. -Format: multiaddress; see https://multiformats.io/multiaddr/`, + Comment: `Binding address for the libp2p host contacted by indexer nodes to sync the list of advertised +multihashes. Note that when port is set to 0 a random port is generated at runtime and may be +different on every restart. The format of the strings specified must conform to multiaddress; +see https://multiformats.io/multiaddr/`, }, { Name: "AnnounceAddresses", Type: "[]string", - Comment: `Addresses to explicitly announce to other peers. If not specified, -all interface addresses are announced -Format: multiaddress`, + Comment: `The address the endpoints at which the data associated to the advertised +multihashes can be retrieved. If not specified, the ListenAddresses are used instead. The format +of the strings specified must conform to multiaddress; see https://multiformats.io/multiaddr/`, }, { Name: "MaxSimultaneousTransfers", Type: "uint64", - Comment: `The maximum number of simultaneous requests syncing the list of advertised multihashes between the indexers and the index provider.`, + Comment: `The maximum number of simultaneous requests syncing the list of advertised multihashes between +the indexers and the index provider.`, }, }, "Libp2p": []DocField{ @@ -857,8 +860,8 @@ Default is 20 (about once a week).`, Comment: ``, }, { - Name: "IndexerProvider", - Type: "IndexerProviderConfig", + Name: "IndexProvider", + Type: "IndexProviderConfig", Comment: ``, }, diff --git a/node/config/types.go b/node/config/types.go index 18f27feea..b73e957f6 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -45,14 +45,14 @@ type Backup struct { type StorageMiner struct { Common - Subsystems MinerSubsystemConfig - Dealmaking DealmakingConfig - IndexerProvider IndexerProviderConfig - Sealing SealingConfig - Storage sectorstorage.SealerConfig - Fees MinerFeeConfig - Addresses MinerAddressConfig - DAGStore DAGStoreConfig + Subsystems MinerSubsystemConfig + Dealmaking DealmakingConfig + IndexProvider IndexProviderConfig + Sealing SealingConfig + Storage sectorstorage.SealerConfig + Fees MinerFeeConfig + Addresses MinerAddressConfig + DAGStore DAGStoreConfig } type DAGStoreConfig struct { @@ -161,20 +161,22 @@ type DealmakingConfig struct { RetrievalPricing *RetrievalPricing } -type IndexerProviderConfig struct { +type IndexProviderConfig struct { config.Ingest - // Binding address for the libp2p host - 0 means random port. - // Format: multiaddress; see https://multiformats.io/multiaddr/ + // Binding address for the libp2p host contacted by indexer nodes to sync the list of advertised + // multihashes. Note that when port is set to 0 a random port is generated at runtime and may be + // different on every restart. The format of the strings specified must conform to multiaddress; + // see https://multiformats.io/multiaddr/ ListenAddresses []string - // Addresses to explicitly announce to other peers. If not specified, - // all interface addresses are announced - // Format: multiaddress + // The address the endpoints at which the data associated to the advertised + // multihashes can be retrieved. If not specified, the ListenAddresses are used instead. The format + // of the strings specified must conform to multiaddress; see https://multiformats.io/multiaddr/ AnnounceAddresses []string - // The maximum number of simultaneous requests syncing the list of advertised multihashes - // between the indexers and the index provider. + // The maximum number of simultaneous requests syncing the list of advertised multihashes between + // the indexers and the index provider. MaxSimultaneousTransfers uint64 } diff --git a/node/modules/storageminer_idxprov.go b/node/modules/storageminer_idxprov.go index 8a0d21057..b5caf3e59 100644 --- a/node/modules/storageminer_idxprov.go +++ b/node/modules/storageminer_idxprov.go @@ -48,7 +48,7 @@ type IdxProv struct { peerstore.Peerstore } -func IndexerProvider(cfg config.IndexerProviderConfig) func(params IdxProv, marketHost host.Host) (provider.Interface, error) { +func IndexProvider(cfg config.IndexProviderConfig) func(params IdxProv, marketHost host.Host) (provider.Interface, error) { return func(args IdxProv, marketHost host.Host) (provider.Interface, error) { ipds := namespace.Wrap(args.Datastore, datastore.NewKey("/indexer-provider")) @@ -57,12 +57,12 @@ func IndexerProvider(cfg config.IndexerProviderConfig) func(params IdxProv, mark return nil, fmt.Errorf("missing private key for node ID: %s", args.PeerID.Pretty()) } - h, err := createIndexerProviderHost(args.MetricsCtx, args.Lifecycle, pkey, args.Peerstore, cfg.ListenAddresses, cfg.AnnounceAddresses) + h, err := createIndexProviderHost(args.MetricsCtx, args.Lifecycle, pkey, args.Peerstore, cfg.ListenAddresses, cfg.AnnounceAddresses) if err != nil { return nil, xerrors.Errorf("creating indexer provider host: %w", err) } - dt, err := newIndexerProviderDataTransfer(cfg, args.MetricsCtx, args.Lifecycle, args.Repo, h, ipds) + dt, err := newIndexProviderDataTransfer(cfg, args.MetricsCtx, args.Lifecycle, args.Repo, h, ipds) if err != nil { return nil, err } @@ -96,7 +96,7 @@ func IndexerProvider(cfg config.IndexerProviderConfig) func(params IdxProv, mark } } -func createIndexerProviderHost(mctx helpers.MetricsCtx, lc fx.Lifecycle, pkey ci.PrivKey, pstore peerstore.Peerstore, listenAddrs []string, announceAddrs []string) (host.Host, error) { +func createIndexProviderHost(mctx helpers.MetricsCtx, lc fx.Lifecycle, pkey ci.PrivKey, pstore peerstore.Peerstore, listenAddrs []string, announceAddrs []string) (host.Host, error) { addrsFactory, err := lp2p.MakeAddrsFactory(announceAddrs, nil) if err != nil { return nil, err @@ -125,11 +125,11 @@ func createIndexerProviderHost(mctx helpers.MetricsCtx, lc fx.Lifecycle, pkey ci return h, nil } -func newIndexerProviderDataTransfer(cfg config.IndexerProviderConfig, mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, h host.Host, ds datastore.Batching) (datatransfer.Manager, error) { +func newIndexProviderDataTransfer(cfg config.IndexProviderConfig, mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, h host.Host, ds datastore.Batching) (datatransfer.Manager, error) { net := dtnet.NewFromLibp2pHost(h) // Set up graphsync - gs := newIndexerProviderGraphsync(cfg, mctx, lc, h) + gs := newIndexProviderGraphsync(cfg, mctx, lc, h) // Set up data transfer dtDs := namespace.Wrap(ds, datastore.NewKey("/datatransfer/transfers")) @@ -153,7 +153,7 @@ func newIndexerProviderDataTransfer(cfg config.IndexerProviderConfig, mctx helpe return dt, nil } -func newIndexerProviderGraphsync(cfg config.IndexerProviderConfig, mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host) graphsync.GraphExchange { +func newIndexProviderGraphsync(cfg config.IndexProviderConfig, mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host) graphsync.GraphExchange { graphsyncNetwork := gsnet.NewFromLibp2pHost(h) return graphsyncimpl.New(helpers.LifecycleCtx(mctx, lc), graphsyncNetwork, From 6ef0c382c3445595406b14e22f56d1ec4c4056b3 Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Thu, 3 Feb 2022 13:24:46 +0000 Subject: [PATCH 3/3] Fix typos in default miner config --- documentation/en/default-lotus-miner-config.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/en/default-lotus-miner-config.toml b/documentation/en/default-lotus-miner-config.toml index 79b4efb87..6fad2700c 100644 --- a/documentation/en/default-lotus-miner-config.toml +++ b/documentation/en/default-lotus-miner-config.toml @@ -258,9 +258,9 @@ [IndexProvider] - # The maximum number of multihash chunk links that index provider cash can store before + # The maximum number of multihash chunk links that index provider cache can store before # LRU eviction. If chunks belonging to a single advertisement are larger than the cache can - # hold, the cache is resized to be able to hold all links. The actual disk usaed depends on + # hold, the cache is resized to be able to hold all links. The actual disk usage depends on # LinkedChunkSize and the length of multihashes. For example, for 128-bit long multihashes # with the default LinkedChunkSize, and LinkCacheSize the cache size can grow to 256MiB. #