From 39cc536115fb07d0e65675cb5e72b2c34c01e797 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 29 Oct 2020 04:51:08 +0100 Subject: [PATCH 1/5] Add sectors vis Signed-off-by: Jakub Sztandera --- cmd/lotus-shed/sectors.go | 198 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) diff --git a/cmd/lotus-shed/sectors.go b/cmd/lotus-shed/sectors.go index cf40e1152..815b63315 100644 --- a/cmd/lotus-shed/sectors.go +++ b/cmd/lotus-shed/sectors.go @@ -1,8 +1,16 @@ package main import ( + "bytes" + "encoding/base64" "fmt" + "image" + "image/color" + "image/png" + "os" + "sort" "strconv" + "sync" "golang.org/x/xerrors" @@ -10,6 +18,7 @@ import ( "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/ipfs/go-cid" "github.com/urfave/cli/v2" miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" @@ -18,6 +27,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/parmap" ) var sectorsCmd = &cli.Command{ @@ -27,6 +37,7 @@ var sectorsCmd = &cli.Command{ Subcommands: []*cli.Command{ terminateSectorCmd, terminateSectorPenaltyEstimationCmd, + visAllocatedSectorsCmd, }, } @@ -263,3 +274,190 @@ var terminateSectorPenaltyEstimationCmd = &cli.Command{ return nil }, } + +var visAllocatedSectorsCmd = &cli.Command{ + Name: "vis-allocated", + Usage: "Produces a html with visualisation of allocated sectors", + Action: func(cctx *cli.Context) error { + api, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := lcli.ReqContext(cctx) + var miners []address.Address + if cctx.NArg() == 0 { + miners, err = api.StateListMiners(ctx, types.EmptyTSK) + if err != nil { + return err + } + powCache := make(map[address.Address]types.BigInt) + var lk sync.Mutex + parmap.Par(32, miners, func(a address.Address) { + pow, err := api.StateMinerPower(ctx, a, types.EmptyTSK) + if err != nil { + } + + lk.Lock() + if err == nil { + powCache[a] = pow.MinerPower.QualityAdjPower + } else { + powCache[a] = types.NewInt(0) + } + lk.Unlock() + }) + sort.Slice(miners, func(i, j int) bool { + return powCache[miners[i]].GreaterThan(powCache[miners[j]]) + }) + n := sort.Search(len(miners), func(i int) bool { + pow := powCache[miners[i]] + log.Infof("pow @%d = %s", i, pow) + return pow.IsZero() + }) + miners = miners[:n] + } else { + for _, mS := range cctx.Args().Slice() { + mA, err := address.NewFromString(mS) + if err != nil { + return xerrors.Errorf("parsing address '%s': %w", mS, err) + } + miners = append(miners, mA) + } + } + + pngs := make([][]byte, len(miners)) + for i := 0; i < len(miners); i++ { + func() { + state, err := api.StateReadState(ctx, miners[i], types.EmptyTSK) + if err != nil { + log.Errorf("getting state: %+v", err) + return + } + allocSString := state.State.(map[string]interface{})["AllocatedSectors"].(map[string]interface{})["/"].(string) + + allocCid, err := cid.Decode(allocSString) + if err != nil { + log.Errorf("decoding cid: %+v", err) + return + } + rle, err := api.ChainReadObj(ctx, allocCid) + if err != nil { + log.Errorf("reading AllocatedSectors: %+v", err) + return + } + png, err := rleToPng(rle) + if err != nil { + log.Errorf("converting to png: %+v", err) + return + } + pngs[i] = png + encoded := base64.StdEncoding.EncodeToString(pngs[i]) + fmt.Printf(`%s:

`+"\n", miners[i], encoded) + _ = os.Stdout.Sync() + }() + } + + return nil + }, +} + +func rleToPng(rleBytes []byte) ([]byte, error) { + var bf bitfield.BitField + err := bf.UnmarshalCBOR(bytes.NewReader(rleBytes)) + if err != nil { + return nil, xerrors.Errorf("decoding bitfield: %w", err) + } + { + last, err := bf.Last() + if err != nil { + return nil, xerrors.Errorf("getting last: %w", err) + } + if last == 0 { + return nil, nil + } + } + ri, err := bf.RunIterator() + if err != nil { + return nil, xerrors.Errorf("creating interator: %w", err) + } + + const width = 1024 + const skipTh = 128 + const skipSize = 64 + + var size uint64 + for ri.HasNext() { + run, err := ri.NextRun() + if err != nil { + return nil, xerrors.Errorf("getting next run: %w", err) + } + if !run.Val && run.Len > skipTh*width { + size += run.Len%(2*width) + skipSize*width + } else { + size += run.Len + } + } + + img := image.NewRGBA(image.Rect(0, 0, width, int((size+width-1)/width))) + for i := range img.Pix { + img.Pix[i] = 255 + } + + ri, err = bf.RunIterator() + if err != nil { + return nil, xerrors.Errorf("creating interator: %w", err) + } + + const shade = 15 + idx := uint64(0) + realIdx := uint64(0) + for ri.HasNext() { + run, err := ri.NextRun() + if err != nil { + return nil, xerrors.Errorf("getting next run: %w", err) + } + var cut = false + var oldLen uint64 + if !run.Val && run.Len > skipTh*width { + oldLen = run.Len + run.Len = run.Len%(2*width) + skipSize*width + cut = true + } + for i := uint64(0); i < run.Len; i++ { + col := color.Gray{0} + stripe := (realIdx+i)/width%256 >= 128 + if cut && i > skipSize*width/2 { + stripe = (realIdx+i+(skipSize/2*width))/width%256 >= 128 + } + if !run.Val { + col.Y = 255 + if stripe { + col.Y -= shade + } + } else if stripe { + col.Y += shade + } + img.Set(int((idx+i)%width), int((idx+i)/width), col) + } + if cut { + i := (idx + run.Len/2 + width) &^ (width - 1) + iend := i + width + col := color.RGBA{255, 0, 0, 255} + for ; i < iend; i++ { + img.Set(int(i)%width, int(i)/width, col) + } + realIdx += oldLen + idx += run.Len + } else { + realIdx += run.Len + idx += run.Len + } + } + buf := &bytes.Buffer{} + err = png.Encode(buf, img) + if err != nil { + return nil, xerrors.Errorf("encoding png: %w", err) + } + + return buf.Bytes(), nil +} From e615eecbea1e41146dc292eb067e8aa33373a2c9 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 29 Oct 2020 20:33:58 +0100 Subject: [PATCH 2/5] Skip also set sectors Signed-off-by: Jakub Sztandera --- cmd/lotus-shed/sectors.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/lotus-shed/sectors.go b/cmd/lotus-shed/sectors.go index 815b63315..bafbfddb7 100644 --- a/cmd/lotus-shed/sectors.go +++ b/cmd/lotus-shed/sectors.go @@ -382,8 +382,8 @@ func rleToPng(rleBytes []byte) ([]byte, error) { } const width = 1024 - const skipTh = 128 - const skipSize = 64 + const skipTh = 64 + const skipSize = 32 var size uint64 for ri.HasNext() { @@ -391,7 +391,7 @@ func rleToPng(rleBytes []byte) ([]byte, error) { if err != nil { return nil, xerrors.Errorf("getting next run: %w", err) } - if !run.Val && run.Len > skipTh*width { + if run.Len > skipTh*width { size += run.Len%(2*width) + skipSize*width } else { size += run.Len @@ -418,7 +418,7 @@ func rleToPng(rleBytes []byte) ([]byte, error) { } var cut = false var oldLen uint64 - if !run.Val && run.Len > skipTh*width { + if run.Len > skipTh*width { oldLen = run.Len run.Len = run.Len%(2*width) + skipSize*width cut = true From 7832545b7aaf2ce06681a21acf5d11f0b818189b Mon Sep 17 00:00:00 2001 From: Jennifer Wang Date: Fri, 23 Jul 2021 19:04:00 -0400 Subject: [PATCH 3/5] Rebase onto master --- documentation/en/cli-lotus-miner.md | 3 ++- documentation/en/cli-lotus-worker.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index 2ba693bff..7c45d5d43 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -42,7 +42,8 @@ COMMANDS: GLOBAL OPTIONS: --actor value, -a value specify other actor to check state for (read only) --color use color in display output (default: depends on output being a TTY) - --miner-repo value, --storagerepo value Specify miner repo path. flag(storagerepo) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON (default: "~/.lotusminer") [$LOTUS_MINER_PATH, $LOTUS_STORAGE_PATH] + --miner-repo value, --storagerepo value Specify miner repo path. flag(storagerepo) and env(LOTUS_STORAGE_PATH) + are DEPRECATION, will REMOVE SOON (default: "~/.lotusminer") [$LOTUS_MINER_PATH, $LOTUS_STORAGE_PATH] --help, -h show help (default: false) --version, -v print the version (default: false) ``` diff --git a/documentation/en/cli-lotus-worker.md b/documentation/en/cli-lotus-worker.md index dbfc8da29..f0f29afa0 100644 --- a/documentation/en/cli-lotus-worker.md +++ b/documentation/en/cli-lotus-worker.md @@ -20,7 +20,8 @@ COMMANDS: GLOBAL OPTIONS: --worker-repo value, --workerrepo value Specify worker repo path. flag workerrepo and env WORKER_PATH are DEPRECATION, will REMOVE SOON (default: "~/.lotusworker") [$LOTUS_WORKER_PATH, $WORKER_PATH] - --miner-repo value, --storagerepo value Specify miner repo path. flag storagerepo and env LOTUS_STORAGE_PATH are DEPRECATION, will REMOVE SOON (default: "~/.lotusminer") [$LOTUS_MINER_PATH, $LOTUS_STORAGE_PATH] + --miner-repo value, --storagerepo value Specify miner repo path. flag storagerepo and env LOTUS_STORAGE_PATH are + DEPRECATION, will REMOVE SOON (default: "~/.lotusminer") [$LOTUS_MINER_PATH, $LOTUS_STORAGE_PATH] --enable-gpu-proving enable use of GPU for mining operations (default: true) --help, -h show help (default: false) --version, -v print the version (default: false) From 5cb7411c2a005c6305b68be93ab24e75ac29889e Mon Sep 17 00:00:00 2001 From: Jennifer Wang Date: Fri, 23 Jul 2021 19:11:26 -0400 Subject: [PATCH 4/5] docs fix --- documentation/en/cli-lotus-miner.md | 3 +-- documentation/en/cli-lotus-worker.md | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index 7c45d5d43..2ba693bff 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -42,8 +42,7 @@ COMMANDS: GLOBAL OPTIONS: --actor value, -a value specify other actor to check state for (read only) --color use color in display output (default: depends on output being a TTY) - --miner-repo value, --storagerepo value Specify miner repo path. flag(storagerepo) and env(LOTUS_STORAGE_PATH) - are DEPRECATION, will REMOVE SOON (default: "~/.lotusminer") [$LOTUS_MINER_PATH, $LOTUS_STORAGE_PATH] + --miner-repo value, --storagerepo value Specify miner repo path. flag(storagerepo) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON (default: "~/.lotusminer") [$LOTUS_MINER_PATH, $LOTUS_STORAGE_PATH] --help, -h show help (default: false) --version, -v print the version (default: false) ``` diff --git a/documentation/en/cli-lotus-worker.md b/documentation/en/cli-lotus-worker.md index f0f29afa0..dbfc8da29 100644 --- a/documentation/en/cli-lotus-worker.md +++ b/documentation/en/cli-lotus-worker.md @@ -20,8 +20,7 @@ COMMANDS: GLOBAL OPTIONS: --worker-repo value, --workerrepo value Specify worker repo path. flag workerrepo and env WORKER_PATH are DEPRECATION, will REMOVE SOON (default: "~/.lotusworker") [$LOTUS_WORKER_PATH, $WORKER_PATH] - --miner-repo value, --storagerepo value Specify miner repo path. flag storagerepo and env LOTUS_STORAGE_PATH are - DEPRECATION, will REMOVE SOON (default: "~/.lotusminer") [$LOTUS_MINER_PATH, $LOTUS_STORAGE_PATH] + --miner-repo value, --storagerepo value Specify miner repo path. flag storagerepo and env LOTUS_STORAGE_PATH are DEPRECATION, will REMOVE SOON (default: "~/.lotusminer") [$LOTUS_MINER_PATH, $LOTUS_STORAGE_PATH] --enable-gpu-proving enable use of GPU for mining operations (default: true) --help, -h show help (default: false) --version, -v print the version (default: false) From 7de866d88182edf1a30bc9c837121176dd4a6d52 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 26 Jul 2021 16:39:12 +0200 Subject: [PATCH 5/5] lint = happy Signed-off-by: Jakub Sztandera --- cmd/lotus-shed/sectors.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/lotus-shed/sectors.go b/cmd/lotus-shed/sectors.go index bafbfddb7..726d992c4 100644 --- a/cmd/lotus-shed/sectors.go +++ b/cmd/lotus-shed/sectors.go @@ -295,8 +295,6 @@ var visAllocatedSectorsCmd = &cli.Command{ var lk sync.Mutex parmap.Par(32, miners, func(a address.Address) { pow, err := api.StateMinerPower(ctx, a, types.EmptyTSK) - if err != nil { - } lk.Lock() if err == nil {