Integrate the latest index-provider with config to disable announcements

Integrate the latest `index-provider` and reflect the changes to engine
configuration. Note that this commit disables announcements of indices
on the network by default as requested for initial merge to master.

Introduce dedicated index provider configuration parameters with
documentation and defaults that match the defaults in index-provider.

Re-generate code as needed.

node/config/def.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
-	ipconfig "github.com/filecoin-project/index-provider/config"
 	miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
 
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -185,7 +184,11 @@ func DefaultStorageMiner() *StorageMiner {
 		},
 
 		IndexProvider: IndexProviderConfig{
-			Ingest: ipconfig.NewIngest(),
+			Enable:               false,
+			EntriesCacheCapacity: 1024,
+			EntriesChunkSize:     16384,
+			TopicName:            "/indexer/ingest/mainnet",
+			PurgeCacheOnStart:    false,
 		},
 
 		Subsystems: MinerSubsystemConfig{

node/config/doc_gen.go

@@ -366,6 +366,49 @@ see https://docs.filecoin.io/mine/lotus/miner-configuration/#using-filters-for-f
 			Comment: ``,
 		},
 	},
+	"IndexProviderConfig": []DocField{
+		{
+			Name: "Enable",
+			Type: "bool",
+
+			Comment: `Enable set whether to enable indexing announcement to the network and expose endpoints that
+allow indexer nodes to process announcements. Disabled by default.`,
+		},
+		{
+			Name: "EntriesCacheCapacity",
+			Type: "int",
+
+			Comment: `EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement
+entries. Defaults to 1024 if not specified. The cache is evicted using LRU policy. The
+maximum storage used by the cache is a factor of EntriesCacheCapacity, EntriesChunkSize and
+the length of multihashes being advertised. For example, advertising 128-bit long multihashes
+with the default EntriesCacheCapacity, and EntriesChunkSize means the cache size can grow to
+256MiB when full.`,
+		},
+		{
+			Name: "EntriesChunkSize",
+			Type: "int",
+
+			Comment: `EntriesChunkSize sets the maximum number of multihashes to include in a single entries chunk.
+Defaults to 16384 if not specified. Note that chunks are chained together for indexing
+advertisements that include more multihashes than the configured EntriesChunkSize.`,
+		},
+		{
+			Name: "TopicName",
+			Type: "string",
+
+			Comment: `TopicName sets the topic name on which the changes to the advertised content are announced.
+Defaults to '/indexer/ingest/mainnet' if not specified.`,
+		},
+		{
+			Name: "PurgeCacheOnStart",
+			Type: "bool",
+
+			Comment: `PurgeCacheOnStart sets whether to clear any cached entries chunks when the provider engine
+starts. By default, the cache is rehydrated from previously cached entries stored in
+datastore if any is present.`,
+		},
+	},
 	"Libp2p": []DocField{
 		{
 			Name: "ListenAddresses",

node/config/types.go

@@ -3,8 +3,6 @@ package config
 import (
 	"github.com/ipfs/go-cid"
 
-	"github.com/filecoin-project/index-provider/config"
-
 	"github.com/filecoin-project/lotus/chain/types"
 	sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
 )
@@ -166,7 +164,32 @@ type DealmakingConfig struct {
 }
 
 type IndexProviderConfig struct {
-	config.Ingest
+
+	// Enable set whether to enable indexing announcement to the network and expose endpoints that
+	// allow indexer nodes to process announcements. Disabled by default.
+	Enable bool
+
+	// EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement
+	// entries. Defaults to 1024 if not specified. The cache is evicted using LRU policy. The
+	// maximum storage used by the cache is a factor of EntriesCacheCapacity, EntriesChunkSize and
+	// the length of multihashes being advertised. For example, advertising 128-bit long multihashes
+	// with the default EntriesCacheCapacity, and EntriesChunkSize means the cache size can grow to
+	// 256MiB when full.
+	EntriesCacheCapacity int
+
+	// EntriesChunkSize sets the maximum number of multihashes to include in a single entries chunk.
+	// Defaults to 16384 if not specified. Note that chunks are chained together for indexing
+	// advertisements that include more multihashes than the configured EntriesChunkSize.
+	EntriesChunkSize int
+
+	// TopicName sets the topic name on which the changes to the advertised content are announced.
+	// Defaults to '/indexer/ingest/mainnet' if not specified.
+	TopicName string
+
+	// PurgeCacheOnStart sets whether to clear any cached entries chunks when the provider engine
+	// starts. By default, the cache is rehydrated from previously cached entries stored in
+	// datastore if any is present.
+	PurgeCacheOnStart bool
 }
 
 type RetrievalPricing struct {

node/modules/storageminer_idxprov.go

@@ -2,10 +2,7 @@ package modules
 
 import (
 	"context"
-
 	"github.com/ipfs/go-datastore"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"github.com/libp2p/go-libp2p-core/peerstore"
 	"go.uber.org/fx"
 
 	"github.com/filecoin-project/go-address"
@@ -24,33 +21,38 @@ type IdxProv struct {
 
 	fx.Lifecycle
 	Datastore dtypes.MetadataDS
-	PeerID    peer.ID
-	peerstore.Peerstore
 }
 
 func IndexProvider(cfg config.IndexProviderConfig) func(params IdxProv, marketHost host.Host, dt dtypes.ProviderDataTransfer, maddr dtypes.MinerAddress) (provider.Interface, error) {
 	return func(args IdxProv, marketHost host.Host, dt dtypes.ProviderDataTransfer, maddr dtypes.MinerAddress) (provider.Interface, error) {
 		ipds := namespace.Wrap(args.Datastore, datastore.NewKey("/index-provider"))
-
-		pkey := args.Peerstore.PrivKey(args.PeerID)
-		if pkey == nil {
-			return nil, xerrors.Errorf("missing private key for node ID: %s", args.PeerID)
+		var opts = []engine.Option{
+			engine.WithDatastore(ipds),
+			engine.WithHost(marketHost),
+			engine.WithRetrievalAddrs(marketHost.Addrs()...),
 		}
 
-		var retAdds []string
-		for _, a := range marketHost.Addrs() {
-			retAdds = append(retAdds, a.String())
+		llog := log.With("idxProvEnabled", cfg.Enable, "pid", marketHost.ID(), "retAddrs", marketHost.Addrs())
+		// If announcements to the network are enabled, then set options for datatransfer publisher.
+		if cfg.Enable {
+			// Get the miner ID and set as extra gossip data.
+			// The extra data is required by the lotus-specific index-provider gossip message validators.
+			ma := address.Address(maddr)
+			opts = append(opts,
+				engine.WithPublisherKind(engine.DataTransferPublisher),
+				engine.WithDataTransfer(dt),
+				engine.WithExtraGossipData(ma.Bytes()))
+			llog = llog.With("extraGossipData", ma)
+		} else {
+			opts = append(opts, engine.WithPublisherKind(engine.NoPublisher))
 		}
 
-		// Get the miner ID and set as extra gossip data.
-		// The extra data is required by the lotus-specific index-provider gossip message validators.
-		ma := address.Address(maddr)
-		log.Infof("Using extra gossip data in index provider engine: %s", ma)
-
-		e, err := engine.New(cfg.Ingest, pkey, dt, marketHost, ipds, retAdds, engine.WithExtraGossipData(ma.Bytes()))
+		// Instantiate the index provider engine.
+		e, err := engine.New(opts...)
 		if err != nil {
 			return nil, xerrors.Errorf("creating indexer provider engine: %w", err)
 		}
+		llog.Info("Instantiated index provider engine")
 
 		args.Lifecycle.Append(fx.Hook{
 			OnStart: func(ctx context.Context) error {
@@ -60,6 +62,7 @@ func IndexProvider(cfg config.IndexProviderConfig) func(params IdxProv, marketHo
 				if err := e.Start(ctx); err != nil {
 					return xerrors.Errorf("starting indexer provider engine: %w", err)
 				}
+				log.Infof("Started index provider engine")
 				return nil
 			},
 			OnStop: func(_ context.Context) error {