Merge pull request #10202 from filecoin-project/feat/faster-sectors-list

sp cli: make sectors list much faster

cmd/lotus-miner/sectors.go

@@ -9,6 +9,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/docker/go-units"
@@ -33,11 +34,14 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
 	cliutil "github.com/filecoin-project/lotus/cli/util"
+	"github.com/filecoin-project/lotus/lib/result"
 	"github.com/filecoin-project/lotus/lib/strle"
 	"github.com/filecoin-project/lotus/lib/tablewriter"
 	sealing "github.com/filecoin-project/lotus/storage/pipeline"
 )
 
+const parallelSectorChecks = 300
+
 var sectorsCmd = &cli.Command{
 	Name:  "sectors",
 	Usage: "interact with sector store",
@@ -306,9 +310,15 @@ var sectorsListCmd = &cli.Command{
 			Usage:   "only show sectors which aren't in the 'Proving' state",
 			Aliases: []string{"u"},
 		},
+		&cli.Int64Flag{
+			Name:  "check-parallelism",
+			Usage: "number of parallel requests to make for checking sector states",
+			Value: parallelSectorChecks,
+		},
 	},
 	Action: func(cctx *cli.Context) error {
-		minerApi, closer, err := lcli.GetStorageMinerAPI(cctx)
+		// http mode allows for parallel json decoding/encoding, which was a bottleneck here
+		minerApi, closer, err := lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp)
 		if err != nil {
 			return err
 		}
@@ -407,16 +417,37 @@ var sectorsListCmd = &cli.Command{
 
 		fast := cctx.Bool("fast")
 
-		for _, s := range list {
-			st, err := minerApi.SectorsStatus(ctx, s, !fast)
-			if err != nil {
+		throttle := make(chan struct{}, cctx.Int64("check-parallelism"))
+
+		slist := make([]result.Result[api.SectorInfo], len(list))
+		var wg sync.WaitGroup
+		for i, s := range list {
+			throttle <- struct{}{}
+			wg.Add(1)
+			go func(i int, s abi.SectorNumber) {
+				defer wg.Done()
+				defer func() { <-throttle }()
+				r := result.Wrap(minerApi.SectorsStatus(ctx, s, !fast))
+				if r.Error != nil {
+					r.Value.SectorID = s
+				}
+				slist[i] = r
+			}(i, s)
+		}
+		wg.Wait()
+
+		for _, rsn := range slist {
+			if rsn.Error != nil {
 				tw.Write(map[string]interface{}{
-					"ID":    s,
+					"ID":    rsn.Value.SectorID,
 					"Error": err,
 				})
 				continue
 			}
 
+			st := rsn.Value
+			s := st.SectorID
+
 			if !showRemoved && st.State == api.SectorState(sealing.Removed) {
 				continue
 			}

documentation/en/cli-lotus-miner.md

@@ -1717,6 +1717,7 @@ USAGE:
    lotus-miner sectors list [command options] [arguments...]
 
 OPTIONS:
+   --check-parallelism value  number of parallel requests to make for checking sector states (default: 300)
    --events, -e               display number of events the sector has received (default: false)
    --fast, -f                 don't show on-chain info for better performance (default: false)
    --initial-pledge, -p       display initial pledge (default: false)