From 81e056be1a3a8cdcab3b4bf7e8ab611c6d06cae2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 7 Feb 2023 11:51:04 +0100 Subject: [PATCH 1/3] sp cli: make sectors list much faster --- cmd/lotus-miner/sectors.go | 39 ++++++++++++++++++++++++++--- documentation/en/cli-lotus-miner.md | 15 +++++------ 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/cmd/lotus-miner/sectors.go b/cmd/lotus-miner/sectors.go index da07901f7..a5c88d15c 100644 --- a/cmd/lotus-miner/sectors.go +++ b/cmd/lotus-miner/sectors.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "github.com/docker/go-units" @@ -33,11 +34,14 @@ import ( "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/lib/result" "github.com/filecoin-project/lotus/lib/strle" "github.com/filecoin-project/lotus/lib/tablewriter" sealing "github.com/filecoin-project/lotus/storage/pipeline" ) +const parallelSectorChecks = 300 + var sectorsCmd = &cli.Command{ Name: "sectors", Usage: "interact with sector store", @@ -306,9 +310,14 @@ var sectorsListCmd = &cli.Command{ Usage: "only show sectors which aren't in the 'Proving' state", Aliases: []string{"u"}, }, + &cli.Int64Flag{ + Name: "check-parallelism", + Usage: "number of parallel requests to make for checking sector states", + Value: parallelSectorChecks, + }, }, Action: func(cctx *cli.Context) error { - minerApi, closer, err := lcli.GetStorageMinerAPI(cctx) + minerApi, closer, err := lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp) if err != nil { return err } @@ -407,16 +416,38 @@ var sectorsListCmd = &cli.Command{ fast := cctx.Bool("fast") + throttle := make(chan struct{}, cctx.Int64("check-parallelism")) + + resCh := make(chan result.Result[api.SectorInfo], len(list)) + var wg sync.WaitGroup for _, s := range list { - st, err := minerApi.SectorsStatus(ctx, s, !fast) - if err != nil { + throttle <- struct{}{} + wg.Add(1) + go func(s abi.SectorNumber) { + defer wg.Done() + defer func() { <-throttle }() + r := result.Wrap(minerApi.SectorsStatus(ctx, s, !fast)) + if r.Error != nil { + r.Value.SectorID = s + } + resCh <- r + }(s) + } + wg.Wait() + close(resCh) + + for rsn := range resCh { + if rsn.Error != nil { tw.Write(map[string]interface{}{ - "ID": s, + "ID": rsn.Value.SectorID, "Error": err, }) continue } + st := rsn.Value + s := st.SectorID + if !showRemoved && st.State == api.SectorState(sealing.Removed) { continue } diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index 0e8e17773..a02315636 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -1717,13 +1717,14 @@ USAGE: lotus-miner sectors list [command options] [arguments...] OPTIONS: - --events, -e display number of events the sector has received (default: false) - --fast, -f don't show on-chain info for better performance (default: false) - --initial-pledge, -p display initial pledge (default: false) - --seal-time, -t display how long it took for the sector to be sealed (default: false) - --show-removed, -r show removed sectors (default: false) - --states value filter sectors by a comma-separated list of states - --unproven, -u only show sectors which aren't in the 'Proving' state (default: false) + --check-parallelism value number of parallel requests to make for checking sector states (default: 300) + --events, -e display number of events the sector has received (default: false) + --fast, -f don't show on-chain info for better performance (default: false) + --initial-pledge, -p display initial pledge (default: false) + --seal-time, -t display how long it took for the sector to be sealed (default: false) + --show-removed, -r show removed sectors (default: false) + --states value filter sectors by a comma-separated list of states + --unproven, -u only show sectors which aren't in the 'Proving' state (default: false) ``` From 17adcb77c3758d5493e73d12f71eb2418662d960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 9 Feb 2023 17:02:41 +0100 Subject: [PATCH 2/3] sp cli: correct order in parallel sectors list --- cmd/lotus-miner/sectors.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cmd/lotus-miner/sectors.go b/cmd/lotus-miner/sectors.go index a5c88d15c..2dad7fc0f 100644 --- a/cmd/lotus-miner/sectors.go +++ b/cmd/lotus-miner/sectors.go @@ -418,25 +418,24 @@ var sectorsListCmd = &cli.Command{ throttle := make(chan struct{}, cctx.Int64("check-parallelism")) - resCh := make(chan result.Result[api.SectorInfo], len(list)) + slist := make([]result.Result[api.SectorInfo], len(list)) var wg sync.WaitGroup - for _, s := range list { + for i, s := range list { throttle <- struct{}{} wg.Add(1) - go func(s abi.SectorNumber) { + go func(i int, s abi.SectorNumber) { defer wg.Done() defer func() { <-throttle }() r := result.Wrap(minerApi.SectorsStatus(ctx, s, !fast)) if r.Error != nil { r.Value.SectorID = s } - resCh <- r - }(s) + slist[i] = r + }(i, s) } wg.Wait() - close(resCh) - for rsn := range resCh { + for _, rsn := range slist { if rsn.Error != nil { tw.Write(map[string]interface{}{ "ID": rsn.Value.SectorID, From 6028d7ba1882aa82dad6a61630f9fb7c91e0ff73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 15 Feb 2023 19:41:46 +0100 Subject: [PATCH 3/3] sectors list: Explain http mode --- cmd/lotus-miner/sectors.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/lotus-miner/sectors.go b/cmd/lotus-miner/sectors.go index 2dad7fc0f..6ff84b3fa 100644 --- a/cmd/lotus-miner/sectors.go +++ b/cmd/lotus-miner/sectors.go @@ -317,6 +317,7 @@ var sectorsListCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { + // http mode allows for parallel json decoding/encoding, which was a bottleneck here minerApi, closer, err := lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp) if err != nil { return err