From 4579e9bb8438195ef1b4d7bec78f2ed202f4cb6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 28 Apr 2020 10:41:59 +0200
Subject: [PATCH 1/2] shed: add some bitfield tools

---
 cmd/lotus-shed/bitfield.go | 184 +++++++++++++++++++++++++++++++++++++
 cmd/lotus-shed/main.go     |   1 +
 2 files changed, 185 insertions(+)
 create mode 100644 cmd/lotus-shed/bitfield.go

diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go
new file mode 100644
index 000000000..4a6b1dca3
--- /dev/null
+++ b/cmd/lotus-shed/bitfield.go
@@ -0,0 +1,184 @@
+package main
+
+import (
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"golang.org/x/xerrors"
+	"gopkg.in/urfave/cli.v2"
+
+	rlepluslazy "github.com/filecoin-project/go-bitfield/rle"
+)
+
+var bitFieldCmd = &cli.Command{
+	Name:        "bitfield",
+	Description: "analyze bitfields",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "enc",
+			Value: "base64",
+			Usage: "specify input encoding to parse",
+		},
+	},
+	Subcommands: []*cli.Command{
+		bitFieldRunsCmd,
+		bitFieldStatCmd,
+	},
+}
+
+var bitFieldRunsCmd = &cli.Command{
+	Name:        "runs",
+	Description: "print bit runs in a bitfield",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "enc",
+			Value: "base64",
+			Usage: "specify input encoding to parse",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		var val string
+		if cctx.Args().Present() {
+			val = cctx.Args().Get(0)
+		} else {
+			b, err := ioutil.ReadAll(os.Stdin)
+			if err != nil {
+				return err
+			}
+			val = string(b)
+		}
+
+		var dec []byte
+		switch cctx.String("enc") {
+		case "base64":
+			d, err := base64.StdEncoding.DecodeString(val)
+			if err != nil {
+				return fmt.Errorf("decoding base64 value: %w", err)
+			}
+			dec = d
+		case "hex":
+			d, err := hex.DecodeString(val)
+			if err != nil {
+				return fmt.Errorf("decoding hex value: %w", err)
+			}
+			dec = d
+		default:
+			return fmt.Errorf("unrecognized encoding: %s", cctx.String("enc"))
+		}
+
+		rle, err := rlepluslazy.FromBuf(dec)
+		if err != nil {
+			return xerrors.Errorf("opening rle: %w", err)
+		}
+
+		rit, err := rle.RunIterator()
+		if err != nil {
+			return xerrors.Errorf("getting run iterator: %w", err)
+		}
+		var idx uint64
+		for rit.HasNext() {
+			r, err := rit.NextRun()
+			if err != nil {
+				return xerrors.Errorf("next run: %w", err)
+			}
+			if !r.Valid() {
+				fmt.Print("!INVALID ")
+			}
+			s := "TRUE "
+			if !r.Val {
+				s = "FALSE"
+			}
+
+			fmt.Printf("@%d %s * %d\n", idx, s, r.Len)
+
+			idx += r.Len
+		}
+
+		return nil
+	},
+}
+
+var bitFieldStatCmd = &cli.Command{
+	Name:        "stat",
+	Description: "print bitfield stats",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "enc",
+			Value: "base64",
+			Usage: "specify input encoding to parse",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		var val string
+		if cctx.Args().Present() {
+			val = cctx.Args().Get(0)
+		} else {
+			b, err := ioutil.ReadAll(os.Stdin)
+			if err != nil {
+				return err
+			}
+			val = string(b)
+		}
+
+		var dec []byte
+		switch cctx.String("enc") {
+		case "base64":
+			d, err := base64.StdEncoding.DecodeString(val)
+			if err != nil {
+				return fmt.Errorf("decoding base64 value: %w", err)
+			}
+			dec = d
+		case "hex":
+			d, err := hex.DecodeString(val)
+			if err != nil {
+				return fmt.Errorf("decoding hex value: %w", err)
+			}
+			dec = d
+		default:
+			return fmt.Errorf("unrecognized encoding: %s", cctx.String("enc"))
+		}
+
+		rle, err := rlepluslazy.FromBuf(dec)
+		if err != nil {
+			return xerrors.Errorf("opening rle: %w", err)
+		}
+
+		rit, err := rle.RunIterator()
+		if err != nil {
+			return xerrors.Errorf("getting run iterator: %w", err)
+		}
+
+		fmt.Printf("Raw length: %d bits (%d bytes)\n", len(dec) * 8, len(dec))
+
+		var ones, zeros, oneRuns, zeroRuns, invalid uint64
+
+		for rit.HasNext() {
+			r, err := rit.NextRun()
+			if err != nil {
+				return xerrors.Errorf("next run: %w", err)
+			}
+			if !r.Valid() {
+				invalid++
+			}
+			if r.Val {
+				ones += r.Len
+				oneRuns++
+			} else {
+				zeros += r.Len
+				zeroRuns++
+			}
+		}
+
+		fmt.Printf("Decoded length: %d bits\n", ones+zeros)
+		fmt.Printf("\tOnes:  %d\n", ones)
+		fmt.Printf("\tZeros: %d\n", zeros)
+		fmt.Printf("Runs: %d\n", oneRuns+zeroRuns)
+		fmt.Printf("\tOne Runs:  %d\n", oneRuns)
+		fmt.Printf("\tZero Runs: %d\n", zeroRuns)
+		fmt.Printf("Invalid runs: %d\n", invalid)
+		return nil
+	},
+}
diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go
index 7a950e14a..c56b2e396 100644
--- a/cmd/lotus-shed/main.go
+++ b/cmd/lotus-shed/main.go
@@ -17,6 +17,7 @@ func main() {
 	local := []*cli.Command{
 		base32Cmd,
 		base16Cmd,
+		bitFieldCmd,
 		keyinfoCmd,
 		peerkeyCmd,
 		noncefix,

From 60145301454e19d1cce251e6e87f694c836c0713 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 28 Apr 2020 21:36:58 +0200
Subject: [PATCH 2/2] shed bitfield: use Count to check for overflows

---
 cmd/lotus-shed/bitfield.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go
index 4a6b1dca3..44d79fcd6 100644
--- a/cmd/lotus-shed/bitfield.go
+++ b/cmd/lotus-shed/bitfield.go
@@ -172,6 +172,10 @@ var bitFieldStatCmd = &cli.Command{
 			}
 		}
 
+		if _, err := rle.Count(); err != nil { // check overflows
+			fmt.Println("Error: ", err)
+		}
+
 		fmt.Printf("Decoded length: %d bits\n", ones+zeros)
 		fmt.Printf("\tOnes:  %d\n", ones)
 		fmt.Printf("\tZeros: %d\n", zeros)