Merge remote-tracking branch 'origin/master' into feat/docs-reorg

CHANGELOG.md

@@ -1,5 +1,29 @@
 # Lotus changelog
 
+# 0.7.1 / 2020-09-17
+
+This optional release of Lotus introduces some critical fixes to the window PoSt process. It also upgrades some core dependencies, and introduces many improvements to the mining process, deal-making cycle, and overall User Experience.
+
+## Changes
+
+#### Some notable improvements: 
+
+- Correctly construct params for `SubmitWindowedPoSt` messages (https://github.com/filecoin-project/lotus/pull/3909)
+- Skip sectors correctly for Window PoSt (https://github.com/filecoin-project/lotus/pull/3839)
+- Split window PoST submission into multiple messages (https://github.com/filecoin-project/lotus/pull/3689)
+- Improve journal coverage (https://github.com/filecoin-project/lotus/pull/2455)
+- Allow retrievals while sealing (https://github.com/filecoin-project/lotus/pull/3778)
+- Don't prune locally published messages (https://github.com/filecoin-project/lotus/pull/3772)
+- Add get-ask, set-ask retrieval commands (https://github.com/filecoin-project/lotus/pull/3886)
+- Consistently name winning and window post in logs (https://github.com/filecoin-project/lotus/pull/3873))
+- Add auto flag to mpool replace (https://github.com/filecoin-project/lotus/pull/3752))
+
+#### Dependencies
+
+- Upgrade markets to `v0.6.1` (https://github.com/filecoin-project/lotus/pull/3906)
+- Upgrade specs-actors to `v0.9.10` (https://github.com/filecoin-project/lotus/pull/3846)
+- Upgrade badger (https://github.com/filecoin-project/lotus/pull/3739)
+
 # 0.7.0 / 2020-09-10
 
 This consensus-breaking release of Lotus is designed to test a network upgrade on the space race testnet. The changes that break consensus are:

build/version.go

@@ -29,7 +29,7 @@ func buildType() string {
 }
 
 // BuildVersion is the local build version, set by build system
-const BuildVersion = "0.7.0"
+const BuildVersion = "0.7.1"
 
 func UserVersion() string {
 	return BuildVersion + buildType() + CurrentCommit
@@ -83,7 +83,7 @@ func VersionForType(nodeType NodeType) (Version, error) {
 
 // semver versions of the rpc api exposed
 var (
-	FullAPIVersion   = newVer(0, 14, 0)
+	FullAPIVersion   = newVer(0, 15, 0)
 	MinerAPIVersion  = newVer(0, 14, 0)
 	WorkerAPIVersion = newVer(0, 14, 0)
 )

cmd/lotus-shed/bitfield.go

@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 
@@ -28,6 +29,9 @@ var bitFieldCmd = &cli.Command{
 		bitFieldRunsCmd,
 		bitFieldStatCmd,
 		bitFieldDecodeCmd,
+		bitFieldIntersectCmd,
+		bitFieldEncodeCmd,
+		bitFieldSubCmd,
 	},
 }
 
@@ -200,38 +204,9 @@ var bitFieldDecodeCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
-		var val string
-		if cctx.Args().Present() {
-			val = cctx.Args().Get(0)
-		} else {
-			b, err := ioutil.ReadAll(os.Stdin)
-			if err != nil {
-				return err
-			}
-			val = string(b)
-		}
-
-		var dec []byte
-		switch cctx.String("enc") {
-		case "base64":
-			d, err := base64.StdEncoding.DecodeString(val)
-			if err != nil {
-				return fmt.Errorf("decoding base64 value: %w", err)
-			}
-			dec = d
-		case "hex":
-			d, err := hex.DecodeString(val)
-			if err != nil {
-				return fmt.Errorf("decoding hex value: %w", err)
-			}
-			dec = d
-		default:
-			return fmt.Errorf("unrecognized encoding: %s", cctx.String("enc"))
-		}
-
-		rle, err := bitfield.NewFromBytes(dec)
+		rle, err := decode(cctx, 0)
 		if err != nil {
-			return xerrors.Errorf("failed to parse bitfield: %w", err)
+			return err
 		}
 
 		vals, err := rle.All(100000000000)
@@ -243,3 +218,170 @@ var bitFieldDecodeCmd = &cli.Command{
 		return nil
 	},
 }
+
+var bitFieldIntersectCmd = &cli.Command{
+	Name:        "intersect",
+	Description: "intersect 2 bitfields and print the resulting bitfield as base64",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "enc",
+			Value: "base64",
+			Usage: "specify input encoding to parse",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		b, err := decode(cctx, 1)
+		if err != nil {
+			return err
+		}
+
+		a, err := decode(cctx, 0)
+		if err != nil {
+			return err
+		}
+
+		o, err := bitfield.IntersectBitField(a, b)
+		if err != nil {
+			return xerrors.Errorf("intersect: %w", err)
+		}
+
+		s, err := o.RunIterator()
+		if err != nil {
+			return err
+		}
+
+		bytes, err := rlepluslazy.EncodeRuns(s, []byte{})
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(base64.StdEncoding.EncodeToString(bytes))
+
+		return nil
+	},
+}
+
+var bitFieldSubCmd = &cli.Command{
+	Name:        "sub",
+	Description: "subtract 2 bitfields and print the resulting bitfield as base64",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "enc",
+			Value: "base64",
+			Usage: "specify input encoding to parse",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		b, err := decode(cctx, 1)
+		if err != nil {
+			return err
+		}
+
+		a, err := decode(cctx, 0)
+		if err != nil {
+			return err
+		}
+
+		o, err := bitfield.SubtractBitField(a, b)
+		if err != nil {
+			return xerrors.Errorf("intersect: %w", err)
+		}
+
+		s, err := o.RunIterator()
+		if err != nil {
+			return err
+		}
+
+		bytes, err := rlepluslazy.EncodeRuns(s, []byte{})
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(base64.StdEncoding.EncodeToString(bytes))
+
+		return nil
+	},
+}
+
+var bitFieldEncodeCmd = &cli.Command{
+	Name:        "encode",
+	Description: "encode a series of decimal numbers into a bitfield",
+	ArgsUsage:   "[infile]",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "enc",
+			Value: "base64",
+			Usage: "specify input encoding to parse",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		f, err := os.Open(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+		defer f.Close() // nolint
+
+		out := bitfield.New()
+		for {
+			var i uint64
+			_, err := fmt.Fscan(f, &i)
+			if err == io.EOF {
+				break
+			}
+			out.Set(i)
+		}
+
+		s, err := out.RunIterator()
+		if err != nil {
+			return err
+		}
+
+		bytes, err := rlepluslazy.EncodeRuns(s, []byte{})
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(base64.StdEncoding.EncodeToString(bytes))
+
+		return nil
+	},
+}
+
+func decode(cctx *cli.Context, a int) (bitfield.BitField, error) {
+	var val string
+	if cctx.Args().Present() {
+		if a >= cctx.NArg() {
+			return bitfield.BitField{}, xerrors.Errorf("need more than %d args", a)
+		}
+		val = cctx.Args().Get(a)
+	} else {
+		if a > 0 {
+			return bitfield.BitField{}, xerrors.Errorf("need more than %d args", a)
+		}
+		b, err := ioutil.ReadAll(os.Stdin)
+		if err != nil {
+			return bitfield.BitField{}, err
+		}
+		val = string(b)
+	}
+
+	var dec []byte
+	switch cctx.String("enc") {
+	case "base64":
+		d, err := base64.StdEncoding.DecodeString(val)
+		if err != nil {
+			return bitfield.BitField{}, fmt.Errorf("decoding base64 value: %w", err)
+		}
+		dec = d
+	case "hex":
+		d, err := hex.DecodeString(val)
+		if err != nil {
+			return bitfield.BitField{}, fmt.Errorf("decoding hex value: %w", err)
+		}
+		dec = d
+	default:
+		return bitfield.BitField{}, fmt.Errorf("unrecognized encoding: %s", cctx.String("enc"))
+	}
+
+	return bitfield.NewFromBytes(dec)
+}

cmd/lotus-shed/jwt.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bufio"
 	"crypto/rand"
 	"encoding/hex"
 	"encoding/json"
@@ -8,10 +9,12 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"strings"
 
 	"github.com/gbrlsnchs/jwt/v3"
 	"github.com/urfave/cli/v2"
 
+	"github.com/filecoin-project/go-jsonrpc/auth"
 	"github.com/filecoin-project/lotus/api/apistruct"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/node/modules"
@@ -24,6 +27,102 @@ var jwtCmd = &cli.Command{
    having to run the lotus daemon.`,
 	Subcommands: []*cli.Command{
 		jwtNewCmd,
+		jwtTokenCmd,
+	},
+}
+
+var jwtTokenCmd = &cli.Command{
+	Name:      "token",
+	Usage:     "create a token for a given jwt secret",
+	ArgsUsage: "<name>",
+	Description: `The jwt tokens have four different levels of permissions that provide some ability
+   to control access to what methods can be invoked by the holder of the token.
+
+   This command only works on jwt secrets that are base16 encoded files, such as those produced by the
+   sibling 'new' command.
+	`,
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "output",
+			Value: "token",
+			Usage: "specify a name",
+		},
+		&cli.BoolFlag{
+			Name:  "read",
+			Value: false,
+			Usage: "add read permissions to the token",
+		},
+		&cli.BoolFlag{
+			Name:  "write",
+			Value: false,
+			Usage: "add write permissions to the token",
+		},
+		&cli.BoolFlag{
+			Name:  "sign",
+			Value: false,
+			Usage: "add sign permissions to the token",
+		},
+		&cli.BoolFlag{
+			Name:  "admin",
+			Value: false,
+			Usage: "add admin permissions to the token",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Args().Present() {
+			return fmt.Errorf("please specify a name")
+		}
+
+		inputFile, err := os.Open(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+		defer inputFile.Close() //nolint:errcheck
+		input := bufio.NewReader(inputFile)
+
+		encoded, err := ioutil.ReadAll(input)
+		if err != nil {
+			return err
+		}
+
+		decoded, err := hex.DecodeString(strings.TrimSpace(string(encoded)))
+		if err != nil {
+			return err
+		}
+
+		var keyInfo types.KeyInfo
+		if err := json.Unmarshal(decoded, &keyInfo); err != nil {
+			return err
+		}
+
+		perms := []auth.Permission{}
+
+		if cctx.Bool("read") {
+			perms = append(perms, apistruct.PermRead)
+		}
+
+		if cctx.Bool("write") {
+			perms = append(perms, apistruct.PermWrite)
+		}
+
+		if cctx.Bool("sign") {
+			perms = append(perms, apistruct.PermSign)
+		}
+
+		if cctx.Bool("admin") {
+			perms = append(perms, apistruct.PermAdmin)
+		}
+
+		p := modules.JwtPayload{
+			Allow: perms,
+		}
+
+		token, err := jwt.Sign(&p, jwt.NewHS256(keyInfo.PrivateKey))
+		if err != nil {
+			return err
+		}
+
+		return ioutil.WriteFile(cctx.String("output"), token, 0600)
 	},
 }
 

cmd/lotus-shed/keyinfo.go

@@ -9,16 +9,22 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"path"
 	"strings"
 	"text/template"
 
 	"github.com/urfave/cli/v2"
 
+	"golang.org/x/xerrors"
+
+	"github.com/multiformats/go-base32"
+
 	"github.com/libp2p/go-libp2p-core/crypto"
 	"github.com/libp2p/go-libp2p-core/peer"
 
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/wallet"
+	"github.com/filecoin-project/lotus/node/modules"
 	"github.com/filecoin-project/lotus/node/modules/lp2p"
 	"github.com/filecoin-project/lotus/node/repo"
 
@@ -43,6 +49,90 @@ var keyinfoCmd = &cli.Command{
 		keyinfoNewCmd,
 		keyinfoInfoCmd,
 		keyinfoImportCmd,
+		keyinfoVerifyCmd,
+	},
+}
+
+var keyinfoVerifyCmd = &cli.Command{
+	Name:  "verify",
+	Usage: "verify the filename of a keystore object on disk with it's contents",
+	Description: `Keystore objects are base32 enocded strings, with wallets being dynamically named via
+   the wallet address. This command can ensure that the naming of these keystore objects are correct`,
+	Action: func(cctx *cli.Context) error {
+		filePath := cctx.Args().First()
+		fileName := path.Base(filePath)
+
+		inputFile, err := os.Open(filePath)
+		if err != nil {
+			return err
+		}
+		defer inputFile.Close() //nolint:errcheck
+		input := bufio.NewReader(inputFile)
+
+		keyContent, err := ioutil.ReadAll(input)
+		if err != nil {
+			return err
+		}
+
+		var keyInfo types.KeyInfo
+		if err := json.Unmarshal(keyContent, &keyInfo); err != nil {
+			return err
+		}
+
+		switch keyInfo.Type {
+		case lp2p.KTLibp2pHost:
+			name, err := base32.RawStdEncoding.DecodeString(fileName)
+			if err != nil {
+				return xerrors.Errorf("decoding key: '%s': %w", fileName, err)
+			}
+
+			if string(name) != keyInfo.Type {
+				return fmt.Errorf("%s of type %s is incorrect", fileName, keyInfo.Type)
+			}
+		case modules.KTJwtHmacSecret:
+			name, err := base32.RawStdEncoding.DecodeString(fileName)
+			if err != nil {
+				return xerrors.Errorf("decoding key: '%s': %w", fileName, err)
+			}
+
+			if string(name) != modules.JWTSecretName {
+				return fmt.Errorf("%s of type %s is incorrect", fileName, keyInfo.Type)
+			}
+		case wallet.KTSecp256k1, wallet.KTBLS:
+			keystore := wallet.NewMemKeyStore()
+			w, err := wallet.NewWallet(keystore)
+			if err != nil {
+				return err
+			}
+
+			if _, err := w.Import(&keyInfo); err != nil {
+				return err
+			}
+
+			list, err := keystore.List()
+			if err != nil {
+				return err
+			}
+
+			if len(list) != 1 {
+				return fmt.Errorf("Unexpected number of keys, expected 1, found %d", len(list))
+			}
+
+			name, err := base32.RawStdEncoding.DecodeString(fileName)
+			if err != nil {
+				return xerrors.Errorf("decoding key: '%s': %w", fileName, err)
+			}
+
+			if string(name) != list[0] {
+				return fmt.Errorf("%s of type %s; file is named for %s, but key is actually %s", fileName, keyInfo.Type, string(name), list[0])
+			}
+
+			break
+		default:
+			return fmt.Errorf("Unknown keytype %s", keyInfo.Type)
+		}
+
+		return nil
 	},
 }
 

documentation/en/api-methods.md

@@ -212,7 +212,7 @@ Response:
 ```json
 {
   "Version": "string value",
-  "APIVersion": 3584,
+  "APIVersion": 3840,
   "BlockDelay": 42
 }
 ```

go.mod

@@ -25,9 +25,9 @@ require (
 	github.com/filecoin-project/go-bitfield v0.2.0
 	github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
 	github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
-	github.com/filecoin-project/go-data-transfer v0.6.3
+	github.com/filecoin-project/go-data-transfer v0.6.4
 	github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f
-	github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c
+	github.com/filecoin-project/go-fil-markets v0.6.1
 	github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52
 	github.com/filecoin-project/go-multistore v0.0.3
 	github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20

go.sum

@@ -222,12 +222,12 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:a
 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
-github.com/filecoin-project/go-data-transfer v0.6.3 h1:7TLwm8nuodHYD/uiwJjKc/PGRR+LwqM8jmlZqgWuUfY=
-github.com/filecoin-project/go-data-transfer v0.6.3/go.mod h1:PmBKVXkhh67/tnEdJXQwDHl5mT+7Tbcwe1NPninqhnM=
+github.com/filecoin-project/go-data-transfer v0.6.4 h1:Q08ABa+cOTOLoAyHeA94fPLcwu53p6eeAaxMxQb0m0A=
+github.com/filecoin-project/go-data-transfer v0.6.4/go.mod h1:PmBKVXkhh67/tnEdJXQwDHl5mT+7Tbcwe1NPninqhnM=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
-github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c h1:YGoyYmELQ0LHwDj/WcOvY3oYt+3iM0wdrAhqJQUAIy4=
-github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c/go.mod h1:PLr9svZxsnHkae1Ky7+66g7fP9AlneVxIVu+oSMq56A=
+github.com/filecoin-project/go-fil-markets v0.6.1 h1:qCFLcVkUCbxwEfH/6EcqTuQvibXt/TXZr+vh8tWv/BQ=
+github.com/filecoin-project/go-fil-markets v0.6.1/go.mod h1:dBJl59dAyl8+cGVb/ONPlEQW4+YzhjI3d6bxLfHVpX0=
 github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM=
 github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24=
 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 h1:FXtCp0ybqdQL9knb3OGDpkNTaBbPxgkqPeWKotUwkH0=

storage/wdpost_run.go

@@ -431,7 +431,9 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty
 		postSkipped := bitfield.New()
 		var postOut []proof.PoStProof
 		somethingToProve := true
+
 		for retries := 0; retries < 5; retries++ {
+			var partitions []miner.PoStPartition
 			var sinfos []proof.SectorInfo
 			for partIdx, partition := range batch {
 				// TODO: Can do this in parallel
@@ -477,7 +479,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty
 				}
 
 				sinfos = append(sinfos, ssi...)
-				params.Partitions = append(params.Partitions, miner.PoStPartition{
+				partitions = append(partitions, miner.PoStPartition{
 					Index:   uint64(batchPartitionStartIdx + partIdx),
 					Skipped: skipped,
 				})
@@ -511,6 +513,8 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty
 
 			if err == nil {
 				// Proof generation successful, stop retrying
+				params.Partitions = append(params.Partitions, partitions...)
+
 				break
 			}