Merge branch 'master' into jen/stale-pr

Makefile

@@ -92,7 +92,7 @@ BINS+=lotus
 
 lotus-miner: $(BUILD_DEPS)
 	rm -f lotus-miner
-	go build $(GOFLAGS) -o lotus-miner ./cmd/lotus-storage-miner
+	go build $(GOFLAGS) -o lotus-miner ./cmd/lotus-miner
 .PHONY: lotus-miner
 BINS+=lotus-miner
 

cmd/lotus-shed/sectors.go

@@ -1,8 +1,16 @@
 package main
 
 import (
+	"bytes"
+	"encoding/base64"
 	"fmt"
+	"image"
+	"image/color"
+	"image/png"
+	"os"
+	"sort"
 	"strconv"
+	"sync"
 
 	"golang.org/x/xerrors"
 
@@ -10,6 +18,7 @@ import (
 	"github.com/filecoin-project/go-bitfield"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
+	"github.com/ipfs/go-cid"
 	"github.com/urfave/cli/v2"
 
 	miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner"
@@ -18,6 +27,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/lib/parmap"
 )
 
 var sectorsCmd = &cli.Command{
@@ -27,6 +37,7 @@ var sectorsCmd = &cli.Command{
 	Subcommands: []*cli.Command{
 		terminateSectorCmd,
 		terminateSectorPenaltyEstimationCmd,
+		visAllocatedSectorsCmd,
 	},
 }
 
@@ -263,3 +274,188 @@ var terminateSectorPenaltyEstimationCmd = &cli.Command{
 		return nil
 	},
 }
+
+var visAllocatedSectorsCmd = &cli.Command{
+	Name:  "vis-allocated",
+	Usage: "Produces a html with visualisation of allocated sectors",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+		var miners []address.Address
+		if cctx.NArg() == 0 {
+			miners, err = api.StateListMiners(ctx, types.EmptyTSK)
+			if err != nil {
+				return err
+			}
+			powCache := make(map[address.Address]types.BigInt)
+			var lk sync.Mutex
+			parmap.Par(32, miners, func(a address.Address) {
+				pow, err := api.StateMinerPower(ctx, a, types.EmptyTSK)
+
+				lk.Lock()
+				if err == nil {
+					powCache[a] = pow.MinerPower.QualityAdjPower
+				} else {
+					powCache[a] = types.NewInt(0)
+				}
+				lk.Unlock()
+			})
+			sort.Slice(miners, func(i, j int) bool {
+				return powCache[miners[i]].GreaterThan(powCache[miners[j]])
+			})
+			n := sort.Search(len(miners), func(i int) bool {
+				pow := powCache[miners[i]]
+				log.Infof("pow @%d = %s", i, pow)
+				return pow.IsZero()
+			})
+			miners = miners[:n]
+		} else {
+			for _, mS := range cctx.Args().Slice() {
+				mA, err := address.NewFromString(mS)
+				if err != nil {
+					return xerrors.Errorf("parsing address '%s': %w", mS, err)
+				}
+				miners = append(miners, mA)
+			}
+		}
+
+		pngs := make([][]byte, len(miners))
+		for i := 0; i < len(miners); i++ {
+			func() {
+				state, err := api.StateReadState(ctx, miners[i], types.EmptyTSK)
+				if err != nil {
+					log.Errorf("getting state: %+v", err)
+					return
+				}
+				allocSString := state.State.(map[string]interface{})["AllocatedSectors"].(map[string]interface{})["/"].(string)
+
+				allocCid, err := cid.Decode(allocSString)
+				if err != nil {
+					log.Errorf("decoding cid: %+v", err)
+					return
+				}
+				rle, err := api.ChainReadObj(ctx, allocCid)
+				if err != nil {
+					log.Errorf("reading AllocatedSectors: %+v", err)
+					return
+				}
+				png, err := rleToPng(rle)
+				if err != nil {
+					log.Errorf("converting to png: %+v", err)
+					return
+				}
+				pngs[i] = png
+				encoded := base64.StdEncoding.EncodeToString(pngs[i])
+				fmt.Printf(`%s:</br><img src="data:image/png;base64,%s"></br>`+"\n", miners[i], encoded)
+				_ = os.Stdout.Sync()
+			}()
+		}
+
+		return nil
+	},
+}
+
+func rleToPng(rleBytes []byte) ([]byte, error) {
+	var bf bitfield.BitField
+	err := bf.UnmarshalCBOR(bytes.NewReader(rleBytes))
+	if err != nil {
+		return nil, xerrors.Errorf("decoding bitfield: %w", err)
+	}
+	{
+		last, err := bf.Last()
+		if err != nil {
+			return nil, xerrors.Errorf("getting last: %w", err)
+		}
+		if last == 0 {
+			return nil, nil
+		}
+	}
+	ri, err := bf.RunIterator()
+	if err != nil {
+		return nil, xerrors.Errorf("creating interator: %w", err)
+	}
+
+	const width = 1024
+	const skipTh = 64
+	const skipSize = 32
+
+	var size uint64
+	for ri.HasNext() {
+		run, err := ri.NextRun()
+		if err != nil {
+			return nil, xerrors.Errorf("getting next run: %w", err)
+		}
+		if run.Len > skipTh*width {
+			size += run.Len%(2*width) + skipSize*width
+		} else {
+			size += run.Len
+		}
+	}
+
+	img := image.NewRGBA(image.Rect(0, 0, width, int((size+width-1)/width)))
+	for i := range img.Pix {
+		img.Pix[i] = 255
+	}
+
+	ri, err = bf.RunIterator()
+	if err != nil {
+		return nil, xerrors.Errorf("creating interator: %w", err)
+	}
+
+	const shade = 15
+	idx := uint64(0)
+	realIdx := uint64(0)
+	for ri.HasNext() {
+		run, err := ri.NextRun()
+		if err != nil {
+			return nil, xerrors.Errorf("getting next run: %w", err)
+		}
+		var cut = false
+		var oldLen uint64
+		if run.Len > skipTh*width {
+			oldLen = run.Len
+			run.Len = run.Len%(2*width) + skipSize*width
+			cut = true
+		}
+		for i := uint64(0); i < run.Len; i++ {
+			col := color.Gray{0}
+			stripe := (realIdx+i)/width%256 >= 128
+			if cut && i > skipSize*width/2 {
+				stripe = (realIdx+i+(skipSize/2*width))/width%256 >= 128
+			}
+			if !run.Val {
+				col.Y = 255
+				if stripe {
+					col.Y -= shade
+				}
+			} else if stripe {
+				col.Y += shade
+			}
+			img.Set(int((idx+i)%width), int((idx+i)/width), col)
+		}
+		if cut {
+			i := (idx + run.Len/2 + width) &^ (width - 1)
+			iend := i + width
+			col := color.RGBA{255, 0, 0, 255}
+			for ; i < iend; i++ {
+				img.Set(int(i)%width, int(i)/width, col)
+			}
+			realIdx += oldLen
+			idx += run.Len
+		} else {
+			realIdx += run.Len
+			idx += run.Len
+		}
+	}
+	buf := &bytes.Buffer{}
+	err = png.Encode(buf, img)
+	if err != nil {
+		return nil, xerrors.Errorf("encoding png: %w", err)
+	}
+
+	return buf.Bytes(), nil
+}

go.mod

@@ -36,7 +36,7 @@ require (
 	github.com/filecoin-project/go-data-transfer v1.7.0
 	github.com/filecoin-project/go-fil-commcid v0.1.0
 	github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
-	github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210723225932-46b52248a0f2
+	github.com/filecoin-project/go-fil-markets v1.6.0
 	github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec
 	github.com/filecoin-project/go-multistore v0.0.3
 	github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1

go.sum

@@ -285,8 +285,8 @@ github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+
 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo=
 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8=
 github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c=
-github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210723225932-46b52248a0f2 h1:L5KvfMqZMxST/T4Zu8khwX0K1farbB0w2957ckkbO8A=
+github.com/filecoin-project/go-fil-markets v1.6.0 h1:+1usyX7rXz6Ey6hbHd/Fhx616ZvGCI94rW7wneMcptU=
-github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210723225932-46b52248a0f2/go.mod h1:ZuFDagROUV6GfvBU//KReTQDw+EZci4rH7jMYTD10vs=
+github.com/filecoin-project/go-fil-markets v1.6.0/go.mod h1:ZuFDagROUV6GfvBU//KReTQDw+EZci4rH7jMYTD10vs=
 github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM=
 github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24=
 github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM=

scripts/dev/sminer-init

@@ -7,4 +7,4 @@ export TRUST_PARAMS=1
 tag=${TAG:-debug}
 
 go run -tags=$tag ./cmd/lotus wallet import ~/.genesis-sectors/pre-seal-t01000.key
-go run -tags=$tag ./cmd/lotus-storage-miner init --actor=t01000 --genesis-miner --pre-sealed-sectors=~/.genesis-sectors --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t01000.json
+go run -tags=$tag ./cmd/lotus-miner init --actor=t01000 --genesis-miner --pre-sealed-sectors=~/.genesis-sectors --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t01000.json