feat: CLI command to announce deal to indexers

api/api_storage.go

@@ -210,6 +210,10 @@ type StorageMiner interface {
 	// DagstoreGC runs garbage collection on the DAG store.
 	DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin
 
+	// IndexerAnnounceDeal informs indexer nodes that a new deal was received,
+	// so they can download its index
+	IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error
+
 	// RuntimeSubsystems returns the subsystems that are enabled
 	// in this instance.
 	RuntimeSubsystems(ctx context.Context) (MinerSubsystems, error) //perm:read

api/proxy_gen.go

@@ -657,6 +657,8 @@ type StorageMinerStruct struct {
 
 		DealsSetPieceCidBlocklist func(p0 context.Context, p1 []cid.Cid) error `perm:"admin"`
 
+		IndexerAnnounceDeal func(p0 context.Context, p1 cid.Cid) error ``
+
 		MarketCancelDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"`
 
 		MarketDataTransferUpdates func(p0 context.Context) (<-chan DataTransferChannel, error) `perm:"write"`
@@ -3879,6 +3881,17 @@ func (s *StorageMinerStub) DealsSetPieceCidBlocklist(p0 context.Context, p1 []ci
 	return ErrNotSupported
 }
 
+func (s *StorageMinerStruct) IndexerAnnounceDeal(p0 context.Context, p1 cid.Cid) error {
+	if s.Internal.IndexerAnnounceDeal == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.IndexerAnnounceDeal(p0, p1)
+}
+
+func (s *StorageMinerStub) IndexerAnnounceDeal(p0 context.Context, p1 cid.Cid) error {
+	return ErrNotSupported
+}
+
 func (s *StorageMinerStruct) MarketCancelDataTransfer(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error {
 	if s.Internal.MarketCancelDataTransfer == nil {
 		return ErrNotSupported

cmd/lotus-miner/index_provider.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/ipfs/go-cid"
+
+	"github.com/fatih/color"
+	"github.com/urfave/cli/v2"
+
+	lcli "github.com/filecoin-project/lotus/cli"
+)
+
+var indexProvCmd = &cli.Command{
+	Name:  "index",
+	Usage: "Manage the index provider on the markets subsystem",
+	Subcommands: []*cli.Command{
+		indexProvAnnounceCmd,
+	},
+}
+
+var indexProvAnnounceCmd = &cli.Command{
+	Name:      "announce",
+	ArgsUsage: "<deal proposal cid>",
+	Usage:     "Announce a deal to indexers so they can download its index",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "color",
+			Usage:       "use color in display output",
+			DefaultText: "depends on output being a TTY",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.IsSet("color") {
+			color.NoColor = !cctx.Bool("color")
+		}
+
+		if cctx.NArg() != 1 {
+			return fmt.Errorf("must provide the deal proposal CID")
+		}
+
+		proposalCidStr := cctx.Args().First()
+		proposalCid, err := cid.Parse(proposalCidStr)
+		if err != nil {
+			return fmt.Errorf("invalid deal proposal CID: %w", proposalCid)
+		}
+
+		marketsApi, closer, err := lcli.GetMarketsAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		return marketsApi.IndexerAnnounceDeal(ctx, proposalCid)
+	},
+}

cmd/lotus-miner/main.go

@@ -49,6 +49,7 @@ func main() {
 		lcli.WithCategory("market", retrievalDealsCmd),
 		lcli.WithCategory("market", dataTransfersCmd),
 		lcli.WithCategory("market", dagstoreCmd),
+		lcli.WithCategory("market", indexProvCmd),
 		lcli.WithCategory("storage", sectorsCmd),
 		lcli.WithCategory("storage", provingCmd),
 		lcli.WithCategory("storage", storageCmd),

documentation/en/api-v0-methods-miner.md

@@ -43,6 +43,8 @@
   * [DealsSetPieceCidBlocklist](#DealsSetPieceCidBlocklist)
 * [I](#I)
   * [ID](#ID)
+* [Indexer](#Indexer)
+  * [IndexerAnnounceDeal](#IndexerAnnounceDeal)
 * [Log](#Log)
   * [LogAlerts](#LogAlerts)
   * [LogList](#LogList)
@@ -663,6 +665,27 @@ Inputs: `null`
 
 Response: `"12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"`
 
+## Indexer
+
+
+### IndexerAnnounceDeal
+IndexerAnnounceDeal informs indexer nodes that a new deal was received,
+so they can download its index
+
+
+Perms: 
+
+Inputs:
+```json
+[
+  {
+    "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+  }
+]
+```
+
+Response: `{}`
+
 ## Log
 
 

documentation/en/cli-lotus-miner.md

@@ -30,6 +30,7 @@ COMMANDS:
      retrieval-deals  Manage retrieval deals and related configuration
      data-transfers   Manage data transfers
      dagstore         Manage the dagstore on the markets subsystem
+     index            Manage the index provider on the markets subsystem
    NETWORK:
      net  Manage P2P Network
    RETRIEVAL:
@@ -1114,6 +1115,37 @@ OPTIONS:
    
 ```
 
+## lotus-miner index
+```
+NAME:
+   lotus-miner index - Manage the index provider on the markets subsystem
+
+USAGE:
+   lotus-miner index command [command options] [arguments...]
+
+COMMANDS:
+   announce  Announce a deal to indexers so they can download its index
+   help, h   Shows a list of commands or help for one command
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
+```
+
+### lotus-miner index announce
+```
+NAME:
+   lotus-miner index announce - Announce a deal to indexers so they can download its index
+
+USAGE:
+   lotus-miner index announce [command options] <deal proposal cid>
+
+OPTIONS:
+   --color     use color in display output (default: depends on output being a TTY)
+   --help, -h  show help (default: false)
+   
+```
+
 ## lotus-miner net
 ```
 NAME:

documentation/en/default-lotus-miner-config.toml

@@ -246,6 +246,34 @@
       #Path = ""
 
 
+[IndexerProvider]
+  # env var: LOTUS_INDEXERPROVIDER_LINKCACHESIZE
+  #LinkCacheSize = 0
+
+  # env var: LOTUS_INDEXERPROVIDER_LINKEDCHUNKSIZE
+  #LinkedChunkSize = 0
+
+  # env var: LOTUS_INDEXERPROVIDER_PUBSUBTOPIC
+  #PubSubTopic = ""
+
+  # env var: LOTUS_INDEXERPROVIDER_PURGELINKCACHE
+  #PurgeLinkCache = false
+
+  # Binding address for the libp2p host - 0 means random port.
+  # Format: multiaddress; see https://multiformats.io/multiaddr/
+  #
+  # type: []string
+  # env var: LOTUS_INDEXERPROVIDER_LISTENADDRESSES
+  #ListenAddresses = ["/ip4/0.0.0.0/tcp/0", "/ip6/::/tcp/0"]
+
+  # The maximum number of simultaneous data transfers between the indexers
+  # and the indexer provider
+  #
+  # type: uint64
+  # env var: LOTUS_INDEXERPROVIDER_MAXSIMULTANEOUSTRANSFERS
+  #MaxSimultaneousTransfers = 20
+
+
 [Sealing]
   # Upper bound on how many sectors can be waiting for more deals to be packed in it before it begins sealing at any given time.
   # If the miner is accepting multiple deals in parallel, up to MaxWaitDealsSectors of new sectors will be created.

node/impl/storminer.go

@@ -832,6 +832,10 @@ func (sm *StorageMinerAPI) DagstoreGC(ctx context.Context) ([]api.DagstoreShardR
 	return ret, nil
 }
 
+func (sm *StorageMinerAPI) IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error {
+	return sm.StorageProvider.AnnounceDealToIndexer(ctx, proposalCid)
+}
+
 func (sm *StorageMinerAPI) DealsList(ctx context.Context) ([]api.MarketDeal, error) {
 	return sm.listDeals(ctx)
 }