Merge branch 'master' into feat/replace-multistore-carv2

.github/workflows/stale.yml

@@ -2,7 +2,7 @@ name: Close and mark stale issue
 
 on:
   schedule:
-  - cron: '0 0 * * *'
+  - cron: '0 12 * * *'
 
 jobs:
   stale:
@@ -18,10 +18,16 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 24 hours.'
         close-issue-message: 'This issue was closed because it is missing author input.'
+        stale-pr-message: 'Thank you for submitting the PR and contributing to lotus! Lotus maintainers need more of your input before merging it, please address the suggested changes or reply to the comments or this PR will be closed in 48 hours. You are always more than welcome to reopen the PR later as well!'
+        close-pr-message: 'This PR was closed because it is missing author input. Please feel free to reopen the PR when you get to it! Thank you for your interest in contributing to lotus!'
         stale-issue-label: 'kind/stale'
-        any-of-labels: 'hint/needs-author-input'
-        days-before-issue-stale: 5
+        stale-pr-label: 'kind/stale'
+        any-of-labels: 'need/author-input '
+        days-before-issue-stale: 3
         days-before-issue-close: 1
+        days-before-pr-stale: 5
+        days-before-pr-close: 2
+        remove-stale-when-updated: true
         enable-statistics: true
        
 

Makefile

@@ -92,7 +92,7 @@ BINS+=lotus
 
 lotus-miner: $(BUILD_DEPS)
 	rm -f lotus-miner
-	go build $(GOFLAGS) -o lotus-miner ./cmd/lotus-storage-miner
+	go build $(GOFLAGS) -o lotus-miner ./cmd/lotus-miner
 .PHONY: lotus-miner
 BINS+=lotus-miner
 

cmd/lotus-shed/sectors.go

@@ -1,8 +1,16 @@
 package main
 
 import (
+	"bytes"
+	"encoding/base64"
 	"fmt"
+	"image"
+	"image/color"
+	"image/png"
+	"os"
+	"sort"
 	"strconv"
+	"sync"
 
 	"golang.org/x/xerrors"
 
@@ -10,6 +18,7 @@ import (
 	"github.com/filecoin-project/go-bitfield"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
+	"github.com/ipfs/go-cid"
 	"github.com/urfave/cli/v2"
 
 	miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner"
@@ -18,6 +27,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/lib/parmap"
 )
 
 var sectorsCmd = &cli.Command{
@@ -27,6 +37,7 @@ var sectorsCmd = &cli.Command{
 	Subcommands: []*cli.Command{
 		terminateSectorCmd,
 		terminateSectorPenaltyEstimationCmd,
+		visAllocatedSectorsCmd,
 	},
 }
 
@@ -263,3 +274,188 @@ var terminateSectorPenaltyEstimationCmd = &cli.Command{
 		return nil
 	},
 }
+
+var visAllocatedSectorsCmd = &cli.Command{
+	Name:  "vis-allocated",
+	Usage: "Produces a html with visualisation of allocated sectors",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+		var miners []address.Address
+		if cctx.NArg() == 0 {
+			miners, err = api.StateListMiners(ctx, types.EmptyTSK)
+			if err != nil {
+				return err
+			}
+			powCache := make(map[address.Address]types.BigInt)
+			var lk sync.Mutex
+			parmap.Par(32, miners, func(a address.Address) {
+				pow, err := api.StateMinerPower(ctx, a, types.EmptyTSK)
+
+				lk.Lock()
+				if err == nil {
+					powCache[a] = pow.MinerPower.QualityAdjPower
+				} else {
+					powCache[a] = types.NewInt(0)
+				}
+				lk.Unlock()
+			})
+			sort.Slice(miners, func(i, j int) bool {
+				return powCache[miners[i]].GreaterThan(powCache[miners[j]])
+			})
+			n := sort.Search(len(miners), func(i int) bool {
+				pow := powCache[miners[i]]
+				log.Infof("pow @%d = %s", i, pow)
+				return pow.IsZero()
+			})
+			miners = miners[:n]
+		} else {
+			for _, mS := range cctx.Args().Slice() {
+				mA, err := address.NewFromString(mS)
+				if err != nil {
+					return xerrors.Errorf("parsing address '%s': %w", mS, err)
+				}
+				miners = append(miners, mA)
+			}
+		}
+
+		pngs := make([][]byte, len(miners))
+		for i := 0; i < len(miners); i++ {
+			func() {
+				state, err := api.StateReadState(ctx, miners[i], types.EmptyTSK)
+				if err != nil {
+					log.Errorf("getting state: %+v", err)
+					return
+				}
+				allocSString := state.State.(map[string]interface{})["AllocatedSectors"].(map[string]interface{})["/"].(string)
+
+				allocCid, err := cid.Decode(allocSString)
+				if err != nil {
+					log.Errorf("decoding cid: %+v", err)
+					return
+				}
+				rle, err := api.ChainReadObj(ctx, allocCid)
+				if err != nil {
+					log.Errorf("reading AllocatedSectors: %+v", err)
+					return
+				}
+				png, err := rleToPng(rle)
+				if err != nil {
+					log.Errorf("converting to png: %+v", err)
+					return
+				}
+				pngs[i] = png
+				encoded := base64.StdEncoding.EncodeToString(pngs[i])
+				fmt.Printf(`%s:</br><img src="data:image/png;base64,%s"></br>`+"\n", miners[i], encoded)
+				_ = os.Stdout.Sync()
+			}()
+		}
+
+		return nil
+	},
+}
+
+func rleToPng(rleBytes []byte) ([]byte, error) {
+	var bf bitfield.BitField
+	err := bf.UnmarshalCBOR(bytes.NewReader(rleBytes))
+	if err != nil {
+		return nil, xerrors.Errorf("decoding bitfield: %w", err)
+	}
+	{
+		last, err := bf.Last()
+		if err != nil {
+			return nil, xerrors.Errorf("getting last: %w", err)
+		}
+		if last == 0 {
+			return nil, nil
+		}
+	}
+	ri, err := bf.RunIterator()
+	if err != nil {
+		return nil, xerrors.Errorf("creating interator: %w", err)
+	}
+
+	const width = 1024
+	const skipTh = 64
+	const skipSize = 32
+
+	var size uint64
+	for ri.HasNext() {
+		run, err := ri.NextRun()
+		if err != nil {
+			return nil, xerrors.Errorf("getting next run: %w", err)
+		}
+		if run.Len > skipTh*width {
+			size += run.Len%(2*width) + skipSize*width
+		} else {
+			size += run.Len
+		}
+	}
+
+	img := image.NewRGBA(image.Rect(0, 0, width, int((size+width-1)/width)))
+	for i := range img.Pix {
+		img.Pix[i] = 255
+	}
+
+	ri, err = bf.RunIterator()
+	if err != nil {
+		return nil, xerrors.Errorf("creating interator: %w", err)
+	}
+
+	const shade = 15
+	idx := uint64(0)
+	realIdx := uint64(0)
+	for ri.HasNext() {
+		run, err := ri.NextRun()
+		if err != nil {
+			return nil, xerrors.Errorf("getting next run: %w", err)
+		}
+		var cut = false
+		var oldLen uint64
+		if run.Len > skipTh*width {
+			oldLen = run.Len
+			run.Len = run.Len%(2*width) + skipSize*width
+			cut = true
+		}
+		for i := uint64(0); i < run.Len; i++ {
+			col := color.Gray{0}
+			stripe := (realIdx+i)/width%256 >= 128
+			if cut && i > skipSize*width/2 {
+				stripe = (realIdx+i+(skipSize/2*width))/width%256 >= 128
+			}
+			if !run.Val {
+				col.Y = 255
+				if stripe {
+					col.Y -= shade
+				}
+			} else if stripe {
+				col.Y += shade
+			}
+			img.Set(int((idx+i)%width), int((idx+i)/width), col)
+		}
+		if cut {
+			i := (idx + run.Len/2 + width) &^ (width - 1)
+			iend := i + width
+			col := color.RGBA{255, 0, 0, 255}
+			for ; i < iend; i++ {
+				img.Set(int(i)%width, int(i)/width, col)
+			}
+			realIdx += oldLen
+			idx += run.Len
+		} else {
+			realIdx += run.Len
+			idx += run.Len
+		}
+	}
+	buf := &bytes.Buffer{}
+	err = png.Encode(buf, img)
+	if err != nil {
+		return nil, xerrors.Errorf("encoding png: %w", err)
+	}
+
+	return buf.Bytes(), nil
+}

go.mod

@@ -36,7 +36,7 @@ require (
 	github.com/filecoin-project/go-data-transfer v1.7.0
 	github.com/filecoin-project/go-fil-commcid v0.1.0
 	github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
-	github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210726135502-29002ec31b5d
+	github.com/filecoin-project/go-fil-markets v1.6.0
 	github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec
 	github.com/filecoin-project/go-multistore v0.0.3
 	github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1

scripts/dev/sminer-init

@@ -7,4 +7,4 @@ export TRUST_PARAMS=1
 tag=${TAG:-debug}
 
 go run -tags=$tag ./cmd/lotus wallet import ~/.genesis-sectors/pre-seal-t01000.key
-go run -tags=$tag ./cmd/lotus-storage-miner init --actor=t01000 --genesis-miner --pre-sealed-sectors=~/.genesis-sectors --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t01000.json
+go run -tags=$tag ./cmd/lotus-miner init --actor=t01000 --genesis-miner --pre-sealed-sectors=~/.genesis-sectors --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t01000.json