cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Rename IndexerProvider to IndexProvider and add more docs

Browse Source 
Rename the config section corresponding to indexing to `IndexProvider`
for better readability.

Update existing docs for better clarity and add docs for config
parameters embedded from `index-provider` `Ingest` config library.
This commit is contained in:
Masih H. Derkani
2022-02-03 13:10:47 +00:00
parent 235377585d
commit 575e6add43
9 changed files with 81 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/builder_miner.go
+1 -1
View File
@@ -168,7 +168,7 @@ func ConfigStorageMiner(c interface{}) Option {
Override(new(dtypes.ProviderTransferNetwork), modules.NewProviderTransferNetwork),
Override(new(dtypes.ProviderTransport), modules.NewProviderTransport),
Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDataTransfer),
Override(new(provider.Interface), modules.IndexerProvider(cfg.IndexerProvider)),
Override(new(provider.Interface), modules.IndexProvider(cfg.IndexProvider)),
Override(new(*storedask.StoredAsk), modules.NewStorageAsk),
Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(cfg.Dealmaking, nil)),
Override(new(storagemarket.StorageProvider), modules.StorageProvider),
node/config/def.go
+2 -2
View File
@@ -181,7 +181,7 @@ func DefaultStorageMiner() *StorageMiner {
},
},
IndexerProvider: IndexerProviderConfig{
IndexProvider: IndexProviderConfig{
ListenAddresses: []string{
"/ip4/0.0.0.0/tcp/0",
"/ip6/::/tcp/0",
@@ -234,7 +234,7 @@ func DefaultStorageMiner() *StorageMiner {
// TODO: Remove hardcoded defaults once provider library exposes them.
// See: https://github.com/filecoin-project/index-provider/issues/108
cfg.IndexerProvider.Ingest = ipconfig.NewIngest()
cfg.IndexProvider.Ingest = ipconfig.NewIngest()
cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http"
cfg.Common.API.RemoteListenAddress = "127.0.0.1:2345"
node/config/def_test.go
+1 -1
View File
@@ -55,5 +55,5 @@ func TestDefaultMinerRoundtrip(t *testing.T) {
func TestDefaultStorageMiner_SetsIndexIngestTopic(t *testing.T) {
subject := DefaultStorageMiner()
require.Equal(t, "/indexer/ingest/mainnet", subject.IndexerProvider.PubSubTopic)
require.Equal(t, "/indexer/ingest/mainnet", subject.IndexProvider.PubSubTopic)
}
node/config/doc_gen.go
+12 -9
View File
@@ -358,27 +358,30 @@ see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-f
Comment: ``,
},
},
"IndexerProviderConfig": []DocField{
"IndexProviderConfig": []DocField{
{
Name: "ListenAddresses",
Type: "[]string",
Comment: `Binding address for the libp2p host - 0 means random port.
Format: multiaddress; see https://multiformats.io/multiaddr/`,
Comment: `Binding address for the libp2p host contacted by indexer nodes to sync the list of advertised
multihashes. Note that when port is set to 0 a random port is generated at runtime and may be
different on every restart. The format of the strings specified must conform to multiaddress;
see https://multiformats.io/multiaddr/`,
},
{
Name: "AnnounceAddresses",
Type: "[]string",
Comment: `Addresses to explicitly announce to other peers. If not specified,
all interface addresses are announced
Format: multiaddress`,
Comment: `The address the endpoints at which the data associated to the advertised
multihashes can be retrieved. If not specified, the ListenAddresses are used instead. The format
of the strings specified must conform to multiaddress; see https://multiformats.io/multiaddr/`,
},
{
Name: "MaxSimultaneousTransfers",
Type: "uint64",
Comment: `The maximum number of simultaneous requests syncing the list of advertised multihashes between the indexers and the index provider.`,
Comment: `The maximum number of simultaneous requests syncing the list of advertised multihashes between
the indexers and the index provider.`,
},
},
"Libp2p": []DocField{
@@ -857,8 +860,8 @@ Default is 20 (about once a week).`,
Comment: ``,
},
{
Name: "IndexerProvider",
Type: "IndexerProviderConfig",
Name: "IndexProvider",
Type: "IndexProviderConfig",
Comment: ``,
},
node/config/types.go
+18 -16
View File
@@ -45,14 +45,14 @@ type Backup struct {
type StorageMiner struct {
Common
Subsystems MinerSubsystemConfig
Dealmaking DealmakingConfig
IndexerProvider IndexerProviderConfig
Sealing SealingConfig
Storage sectorstorage.SealerConfig
Fees MinerFeeConfig
Addresses MinerAddressConfig
DAGStore DAGStoreConfig
Subsystems MinerSubsystemConfig
Dealmaking DealmakingConfig
IndexProvider IndexProviderConfig
Sealing SealingConfig
Storage sectorstorage.SealerConfig
Fees MinerFeeConfig
Addresses MinerAddressConfig
DAGStore DAGStoreConfig
}
type DAGStoreConfig struct {
@@ -161,20 +161,22 @@ type DealmakingConfig struct {
RetrievalPricing *RetrievalPricing
}
type IndexerProviderConfig struct {
type IndexProviderConfig struct {
config.Ingest
// Binding address for the libp2p host - 0 means random port.
// Format: multiaddress; see https://multiformats.io/multiaddr/
// Binding address for the libp2p host contacted by indexer nodes to sync the list of advertised
// multihashes. Note that when port is set to 0 a random port is generated at runtime and may be
// different on every restart. The format of the strings specified must conform to multiaddress;
// see https://multiformats.io/multiaddr/
ListenAddresses []string
// Addresses to explicitly announce to other peers. If not specified,
// all interface addresses are announced
// Format: multiaddress
// The address the endpoints at which the data associated to the advertised
// multihashes can be retrieved. If not specified, the ListenAddresses are used instead. The format
// of the strings specified must conform to multiaddress; see https://multiformats.io/multiaddr/
AnnounceAddresses []string
// The maximum number of simultaneous requests syncing the list of advertised multihashes
// between the indexers and the index provider.
// The maximum number of simultaneous requests syncing the list of advertised multihashes between
// the indexers and the index provider.
MaxSimultaneousTransfers uint64
}
node/modules/storageminer_idxprov.go
+7 -7
View File
@@ -48,7 +48,7 @@ type IdxProv struct {
peerstore.Peerstore
}
func IndexerProvider(cfg config.IndexerProviderConfig) func(params IdxProv, marketHost host.Host) (provider.Interface, error) {
func IndexProvider(cfg config.IndexProviderConfig) func(params IdxProv, marketHost host.Host) (provider.Interface, error) {
return func(args IdxProv, marketHost host.Host) (provider.Interface, error) {
ipds := namespace.Wrap(args.Datastore, datastore.NewKey("/indexer-provider"))
@@ -57,12 +57,12 @@ func IndexerProvider(cfg config.IndexerProviderConfig) func(params IdxProv, mark
return nil, fmt.Errorf("missing private key for node ID: %s", args.PeerID.Pretty())
}
h, err := createIndexerProviderHost(args.MetricsCtx, args.Lifecycle, pkey, args.Peerstore, cfg.ListenAddresses, cfg.AnnounceAddresses)
h, err := createIndexProviderHost(args.MetricsCtx, args.Lifecycle, pkey, args.Peerstore, cfg.ListenAddresses, cfg.AnnounceAddresses)
if err != nil {
return nil, xerrors.Errorf("creating indexer provider host: %w", err)
}
dt, err := newIndexerProviderDataTransfer(cfg, args.MetricsCtx, args.Lifecycle, args.Repo, h, ipds)
dt, err := newIndexProviderDataTransfer(cfg, args.MetricsCtx, args.Lifecycle, args.Repo, h, ipds)
if err != nil {
return nil, err
}
@@ -96,7 +96,7 @@ func IndexerProvider(cfg config.IndexerProviderConfig) func(params IdxProv, mark
}
}
func createIndexerProviderHost(mctx helpers.MetricsCtx, lc fx.Lifecycle, pkey ci.PrivKey, pstore peerstore.Peerstore, listenAddrs []string, announceAddrs []string) (host.Host, error) {
func createIndexProviderHost(mctx helpers.MetricsCtx, lc fx.Lifecycle, pkey ci.PrivKey, pstore peerstore.Peerstore, listenAddrs []string, announceAddrs []string) (host.Host, error) {
addrsFactory, err := lp2p.MakeAddrsFactory(announceAddrs, nil)
if err != nil {
return nil, err
@@ -125,11 +125,11 @@ func createIndexerProviderHost(mctx helpers.MetricsCtx, lc fx.Lifecycle, pkey ci
return h, nil
}
func newIndexerProviderDataTransfer(cfg config.IndexerProviderConfig, mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, h host.Host, ds datastore.Batching) (datatransfer.Manager, error) {
func newIndexProviderDataTransfer(cfg config.IndexProviderConfig, mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, h host.Host, ds datastore.Batching) (datatransfer.Manager, error) {
net := dtnet.NewFromLibp2pHost(h)
// Set up graphsync
gs := newIndexerProviderGraphsync(cfg, mctx, lc, h)
gs := newIndexProviderGraphsync(cfg, mctx, lc, h)
// Set up data transfer
dtDs := namespace.Wrap(ds, datastore.NewKey("/datatransfer/transfers"))
@@ -153,7 +153,7 @@ func newIndexerProviderDataTransfer(cfg config.IndexerProviderConfig, mctx helpe
return dt, nil
}
func newIndexerProviderGraphsync(cfg config.IndexerProviderConfig, mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host) graphsync.GraphExchange {
func newIndexProviderGraphsync(cfg config.IndexProviderConfig, mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host) graphsync.GraphExchange {
graphsyncNetwork := gsnet.NewFromLibp2pHost(h)
return graphsyncimpl.New(helpers.LifecycleCtx(mctx, lc),
graphsyncNetwork,