cerc-io/lotus
15
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Rename IndexerProvider to IndexProvider and add more docs

Browse Source 
Rename the config section corresponding to indexing to `IndexProvider`
for better readability.

Update existing docs for better clarity and add docs for config
parameters embedded from `index-provider` `Ingest` config library.
This commit is contained in:
Masih H. Derkani 2022-02-03 11:51:01 +00:00
parent 235377585d
commit 575e6add43
No known key found for this signature in database
GPG Key ID: B320AFAA5B736DB6
9 changed files with 81 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
documentation/en/default-lotus-miner-config.toml
View File

@ -257,39 +257,58 @@
#Path = ""
[IndexerProvider]
# env var: LOTUS_INDEXERPROVIDER_LINKCACHESIZE
[IndexProvider]
# The maximum number of multihash chunk links that index provider cash can store before
# LRU eviction. If chunks belonging to a single advertisement are larger than the cache can
# hold, the cache is resized to be able to hold all links. The actual disk usaed depends on
# LinkedChunkSize and the length of multihashes. For example, for 128-bit long multihashes
# with the default LinkedChunkSize, and LinkCacheSize the cache size can grow to 256MiB.
#
# type int
# env var: LOTUS_INDEXPROVIDER_LINKCACHESIZE
#LinkCacheSize = 1024
# env var: LOTUS_INDEXERPROVIDER_LINKEDCHUNKSIZE
# The number of multihashes in each chunk of the
# advertised multihash entries linked list. If multihashes are 128-bit, then
# setting LinkedChunkSize = 16384 will result in blocks of 0.25MiB when
# full.
#
# type int
# env var: LOTUS_INDEXPROVIDER_LINKEDCHUNKSIZE
#LinkedChunkSize = 16384
# env var: LOTUS_INDEXERPROVIDER_PUBSUBTOPIC
# The gossipsub topic name used to publish change to the advertised content.
#
# env var: LOTUS_INDEXPROVIDER_PUBSUBTOPIC
#PubSubTopic = "/indexer/ingest/mainnet"
# env var: LOTUS_INDEXERPROVIDER_PURGELINKCACHE
# Whether to purge all cached entries on start-up.
#
# env var: LOTUS_INDEXPROVIDER_PURGELINKCACHE
#PurgeLinkCache = false
# Binding address for the libp2p host - 0 means random port.
# Format: multiaddress; see https://multiformats.io/multiaddr/
# Binding address for the libp2p host contacted by indexer nodes to sync the list of advertised
# multihashes. Note that when port is set to 0 a random port is generated at runtime and may be
# different on every restart. The format of the strings specified must conform to multiaddress;
# see https://multiformats.io/multiaddr/
#
# type: []string
# env var: LOTUS_INDEXERPROVIDER_LISTENADDRESSES
# env var: LOTUS_INDEXPROVIDER_LISTENADDRESSES
#ListenAddresses = ["/ip4/0.0.0.0/tcp/0", "/ip6/::/tcp/0"]
# Addresses to explicitly announce to other peers. If not specified,
# all interface addresses are announced
# Format: multiaddress
# The address the endpoints at which the data associated to the advertised
# multihashes can be retrieved. If not specified, the ListenAddresses are used instead. The format
# of the strings specified must conform to multiaddress; see https://multiformats.io/multiaddr/
#
# type: []string
# env var: LOTUS_INDEXERPROVIDER_ANNOUNCEADDRESSES
# env var: LOTUS_INDEXPROVIDER_ANNOUNCEADDRESSES
#AnnounceAddresses = []
# The maximum number of simultaneous requests syncing the list of advertised multihashes between
# the indexers and the index provider.
#
# type: uint64
# env var: LOTUS_INDEXERPROVIDER_MAXSIMULTANEOUSTRANSFERS
# env var: LOTUS_INDEXPROVIDER_MAXSIMULTANEOUSTRANSFERS
#MaxSimultaneousTransfers = 20

10
itests/deals_concurrent_test.go
View File

@ -55,11 +55,11 @@ func TestDealWithMarketAndMinerNode(t *testing.T) {
dh := kit.NewDealHarness(t, client, main, market)
dh.RunConcurrentDeals(kit.RunConcurrentDealsOpts{
N: n,
FastRetrieval: fastRetrieval,
CarExport: carExport,
StartEpoch: startEpoch,
IndexerProvider: idxProv,
N: n,
FastRetrieval: fastRetrieval,
CarExport: carExport,
StartEpoch: startEpoch,
IndexProvider: idxProv,
})
}

6
itests/kit/deals.go
View File

@ -406,7 +406,7 @@ type RunConcurrentDealsOpts struct {
CarExport bool
StartEpoch abi.ChainEpoch
UseCARFileForStorageDeal bool
IndexerProvider *shared_testutil.MockIndexProvider
IndexProvider *shared_testutil.MockIndexProvider
}
func (dh *DealHarness) RunConcurrentDeals(opts RunConcurrentDealsOpts) {
@ -434,8 +434,8 @@ func (dh *DealHarness) RunConcurrentDeals(opts RunConcurrentDealsOpts) {
})
// Check that the storage provider announced the deal to indexers
if opts.IndexerProvider != nil {
notifs := opts.IndexerProvider.GetNotifs()
if opts.IndexProvider != nil {
notifs := opts.IndexProvider.GetNotifs()
_, ok := notifs[string(deal.Bytes())]
require.True(dh.t, ok)
}

2
node/builder_miner.go
View File

@ -168,7 +168,7 @@ func ConfigStorageMiner(c interface{}) Option {
Override(new(dtypes.ProviderTransferNetwork), modules.NewProviderTransferNetwork),
Override(new(dtypes.ProviderTransport), modules.NewProviderTransport),
Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDataTransfer),
Override(new(provider.Interface), modules.IndexerProvider(cfg.IndexerProvider)),
Override(new(provider.Interface), modules.IndexProvider(cfg.IndexProvider)),
Override(new(*storedask.StoredAsk), modules.NewStorageAsk),
Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(cfg.Dealmaking, nil)),
Override(new(storagemarket.StorageProvider), modules.StorageProvider),

4
node/config/def.go
View File

@ -181,7 +181,7 @@ func DefaultStorageMiner() *StorageMiner {
},
},
IndexerProvider: IndexerProviderConfig{
IndexProvider: IndexProviderConfig{
ListenAddresses: []string{
"/ip4/0.0.0.0/tcp/0",
"/ip6/::/tcp/0",
@ -234,7 +234,7 @@ func DefaultStorageMiner() *StorageMiner {
// TODO: Remove hardcoded defaults once provider library exposes them.
// See: https://github.com/filecoin-project/index-provider/issues/108
cfg.IndexerProvider.Ingest = ipconfig.NewIngest()
cfg.IndexProvider.Ingest = ipconfig.NewIngest()
cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http"
cfg.Common.API.RemoteListenAddress = "127.0.0.1:2345"

2
node/config/def_test.go
View File

@ -55,5 +55,5 @@ func TestDefaultMinerRoundtrip(t *testing.T) {
func TestDefaultStorageMiner_SetsIndexIngestTopic(t *testing.T) {
subject := DefaultStorageMiner()
require.Equal(t, "/indexer/ingest/mainnet", subject.IndexerProvider.PubSubTopic)
require.Equal(t, "/indexer/ingest/mainnet", subject.IndexProvider.PubSubTopic)
}

21
node/config/doc_gen.go
View File

@ -358,27 +358,30 @@ see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-f
Comment: ``,
},
},
"IndexerProviderConfig": []DocField{
"IndexProviderConfig": []DocField{
{
Name: "ListenAddresses",
Type: "[]string",
Comment: `Binding address for the libp2p host - 0 means random port.
Format: multiaddress; see https://multiformats.io/multiaddr/`,
Comment: `Binding address for the libp2p host contacted by indexer nodes to sync the list of advertised
multihashes. Note that when port is set to 0 a random port is generated at runtime and may be
different on every restart. The format of the strings specified must conform to multiaddress;
see https://multiformats.io/multiaddr/`,
},
{
Name: "AnnounceAddresses",
Type: "[]string",
Comment: `Addresses to explicitly announce to other peers. If not specified,
all interface addresses are announced
Format: multiaddress`,
Comment: `The address the endpoints at which the data associated to the advertised
multihashes can be retrieved. If not specified, the ListenAddresses are used instead. The format
of the strings specified must conform to multiaddress; see https://multiformats.io/multiaddr/`,
},
{
Name: "MaxSimultaneousTransfers",
Type: "uint64",
Comment: `The maximum number of simultaneous requests syncing the list of advertised multihashes between the indexers and the index provider.`,
Comment: `The maximum number of simultaneous requests syncing the list of advertised multihashes between
the indexers and the index provider.`,
},
},
"Libp2p": []DocField{
@ -857,8 +860,8 @@ Default is 20 (about once a week).`,
Comment: ``,
},
{
Name: "IndexerProvider",
Type: "IndexerProviderConfig",
Name: "IndexProvider",
Type: "IndexProviderConfig",
Comment: ``,
},

34
node/config/types.go
View File

@ -45,14 +45,14 @@ type Backup struct {
type StorageMiner struct {
Common
Subsystems MinerSubsystemConfig
Dealmaking DealmakingConfig
IndexerProvider IndexerProviderConfig
Sealing SealingConfig
Storage sectorstorage.SealerConfig
Fees MinerFeeConfig
Addresses MinerAddressConfig
DAGStore DAGStoreConfig
Subsystems MinerSubsystemConfig
Dealmaking DealmakingConfig
IndexProvider IndexProviderConfig
Sealing SealingConfig
Storage sectorstorage.SealerConfig
Fees MinerFeeConfig
Addresses MinerAddressConfig
DAGStore DAGStoreConfig
}
type DAGStoreConfig struct {
@ -161,20 +161,22 @@ type DealmakingConfig struct {
RetrievalPricing *RetrievalPricing
}
type IndexerProviderConfig struct {
type IndexProviderConfig struct {
config.Ingest
// Binding address for the libp2p host - 0 means random port.
// Format: multiaddress; see https://multiformats.io/multiaddr/
// Binding address for the libp2p host contacted by indexer nodes to sync the list of advertised
// multihashes. Note that when port is set to 0 a random port is generated at runtime and may be
// different on every restart. The format of the strings specified must conform to multiaddress;
// see https://multiformats.io/multiaddr/
ListenAddresses []string
// Addresses to explicitly announce to other peers. If not specified,
// all interface addresses are announced
// Format: multiaddress
// The address the endpoints at which the data associated to the advertised
// multihashes can be retrieved. If not specified, the ListenAddresses are used instead. The format
// of the strings specified must conform to multiaddress; see https://multiformats.io/multiaddr/
AnnounceAddresses []string
// The maximum number of simultaneous requests syncing the list of advertised multihashes
// between the indexers and the index provider.
// The maximum number of simultaneous requests syncing the list of advertised multihashes between
// the indexers and the index provider.
MaxSimultaneousTransfers uint64
}

14
node/modules/storageminer_idxprov.go
View File

@ -48,7 +48,7 @@ type IdxProv struct {
peerstore.Peerstore
}
func IndexerProvider(cfg config.IndexerProviderConfig) func(params IdxProv, marketHost host.Host) (provider.Interface, error) {
func IndexProvider(cfg config.IndexProviderConfig) func(params IdxProv, marketHost host.Host) (provider.Interface, error) {
return func(args IdxProv, marketHost host.Host) (provider.Interface, error) {
ipds := namespace.Wrap(args.Datastore, datastore.NewKey("/indexer-provider"))
@ -57,12 +57,12 @@ func IndexerProvider(cfg config.IndexerProviderConfig) func(params IdxProv, mark
return nil, fmt.Errorf("missing private key for node ID: %s", args.PeerID.Pretty())
}
h, err := createIndexerProviderHost(args.MetricsCtx, args.Lifecycle, pkey, args.Peerstore, cfg.ListenAddresses, cfg.AnnounceAddresses)
h, err := createIndexProviderHost(args.MetricsCtx, args.Lifecycle, pkey, args.Peerstore, cfg.ListenAddresses, cfg.AnnounceAddresses)
if err != nil {
return nil, xerrors.Errorf("creating indexer provider host: %w", err)
}
dt, err := newIndexerProviderDataTransfer(cfg, args.MetricsCtx, args.Lifecycle, args.Repo, h, ipds)
dt, err := newIndexProviderDataTransfer(cfg, args.MetricsCtx, args.Lifecycle, args.Repo, h, ipds)
if err != nil {
return nil, err
}
@ -96,7 +96,7 @@ func IndexerProvider(cfg config.IndexerProviderConfig) func(params IdxProv, mark
}
}
func createIndexerProviderHost(mctx helpers.MetricsCtx, lc fx.Lifecycle, pkey ci.PrivKey, pstore peerstore.Peerstore, listenAddrs []string, announceAddrs []string) (host.Host, error) {
func createIndexProviderHost(mctx helpers.MetricsCtx, lc fx.Lifecycle, pkey ci.PrivKey, pstore peerstore.Peerstore, listenAddrs []string, announceAddrs []string) (host.Host, error) {
addrsFactory, err := lp2p.MakeAddrsFactory(announceAddrs, nil)
if err != nil {
return nil, err
@ -125,11 +125,11 @@ func createIndexerProviderHost(mctx helpers.MetricsCtx, lc fx.Lifecycle, pkey ci
return h, nil
}
func newIndexerProviderDataTransfer(cfg config.IndexerProviderConfig, mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, h host.Host, ds datastore.Batching) (datatransfer.Manager, error) {
func newIndexProviderDataTransfer(cfg config.IndexProviderConfig, mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, h host.Host, ds datastore.Batching) (datatransfer.Manager, error) {
net := dtnet.NewFromLibp2pHost(h)
// Set up graphsync
gs := newIndexerProviderGraphsync(cfg, mctx, lc, h)
gs := newIndexProviderGraphsync(cfg, mctx, lc, h)
// Set up data transfer
dtDs := namespace.Wrap(ds, datastore.NewKey("/datatransfer/transfers"))
@ -153,7 +153,7 @@ func newIndexerProviderDataTransfer(cfg config.IndexerProviderConfig, mctx helpe
return dt, nil
}
func newIndexerProviderGraphsync(cfg config.IndexerProviderConfig, mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host) graphsync.GraphExchange {
func newIndexProviderGraphsync(cfg config.IndexProviderConfig, mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host) graphsync.GraphExchange {
graphsyncNetwork := gsnet.NewFromLibp2pHost(h)
return graphsyncimpl.New(helpers.LifecycleCtx(mctx, lc),
graphsyncNetwork,