From ef8ab20a311a8ebe5008390948cad1beec974049 Mon Sep 17 00:00:00 2001
From: Anton Evangelatov <anton.evangelatov@gmail.com>
Date: Tue, 14 Jul 2020 14:31:15 +0200
Subject: [PATCH 01/37] fix --actor flag for lotus-storage-miner info

---
 cmd/lotus-storage-miner/info.go    |  3 ---
 cmd/lotus-storage-miner/main.go    |  4 ++++
 cmd/lotus-storage-miner/proving.go | 13 +++++++++++++
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go
index 7dff5b36c..2c309d335 100644
--- a/cmd/lotus-storage-miner/info.go
+++ b/cmd/lotus-storage-miner/info.go
@@ -24,9 +24,6 @@ import (
 var infoCmd = &cli.Command{
 	Name:  "info",
 	Usage: "Print miner info",
-	Flags: []cli.Flag{
-		&cli.BoolFlag{Name: "color"},
-	},
 	Action: func(cctx *cli.Context) error {
 		color.NoColor = !cctx.Bool("color")
 
diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go
index 5fec6ad22..b372a2d67 100644
--- a/cmd/lotus-storage-miner/main.go
+++ b/cmd/lotus-storage-miner/main.go
@@ -76,6 +76,9 @@ func main() {
 				Usage:   "specify other actor to check state for (read only)",
 				Aliases: []string{"a"},
 			},
+			&cli.BoolFlag{
+				Name: "color",
+			},
 			&cli.StringFlag{
 				Name:    "repo",
 				EnvVars: []string{"LOTUS_PATH"},
@@ -108,6 +111,7 @@ func getActorAddress(ctx context.Context, nodeAPI api.StorageMiner, overrideMadd
 		if err != nil {
 			return maddr, err
 		}
+		return
 	}
 
 	maddr, err = nodeAPI.ActorAddress(ctx)
diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go
index cbd120e6e..c6cfcaf30 100644
--- a/cmd/lotus-storage-miner/proving.go
+++ b/cmd/lotus-storage-miner/proving.go
@@ -7,6 +7,7 @@ import (
 	"text/tabwriter"
 	"time"
 
+	"github.com/fatih/color"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
@@ -32,6 +33,8 @@ var provingFaultsCmd = &cli.Command{
 	Name:  "faults",
 	Usage: "View the currently known proving faulty sectors information",
 	Action: func(cctx *cli.Context) error {
+		color.NoColor = !cctx.Bool("color")
+
 		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
 		if err != nil {
 			return err
@@ -66,6 +69,8 @@ var provingFaultsCmd = &cli.Command{
 			}
 		}
 
+		fmt.Printf("Miner: %s\n", color.BlueString("%s", maddr))
+
 		head, err := api.ChainHead(ctx)
 		if err != nil {
 			return xerrors.Errorf("getting chain head: %w", err)
@@ -101,6 +106,8 @@ var provingInfoCmd = &cli.Command{
 	Name:  "info",
 	Usage: "View current state information",
 	Action: func(cctx *cli.Context) error {
+		color.NoColor = !cctx.Bool("color")
+
 		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
 		if err != nil {
 			return err
@@ -135,6 +142,8 @@ var provingInfoCmd = &cli.Command{
 			return xerrors.Errorf("getting miner deadlines: %w", err)
 		}
 
+		fmt.Printf("Miner: %s\n", color.BlueString("%s", maddr))
+
 		var mas miner.State
 		{
 			mact, err := api.StateGetActor(ctx, maddr, types.EmptyTSK)
@@ -240,6 +249,8 @@ var provingDeadlinesCmd = &cli.Command{
 	Name:  "deadlines",
 	Usage: "View the current proving period deadlines information",
 	Action: func(cctx *cli.Context) error {
+		color.NoColor = !cctx.Bool("color")
+
 		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
 		if err != nil {
 			return err
@@ -284,6 +295,8 @@ var provingDeadlinesCmd = &cli.Command{
 			}
 		}
 
+		fmt.Printf("Miner: %s\n", color.BlueString("%s", maddr))
+
 		tw := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0)
 		_, _ = fmt.Fprintln(tw, "deadline\tpartitions\tsectors\tproven")
 

From a62edd05de47ad741fffbf5d8b3a292705cb68e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sami=20M=C3=A4kel=C3=A4?= <mrsmkl@gmail.com>
Date: Mon, 20 Jul 2020 11:28:45 +0300
Subject: [PATCH 02/37] adding rootkey to genesis

---
 chain/gen/genesis/genesis.go | 61 +++++++++++++++++-------------------
 genesis/types.go             |  2 ++
 2 files changed, 30 insertions(+), 33 deletions(-)

diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go
index ebacf51cb..56b3f353d 100644
--- a/chain/gen/genesis/genesis.go
+++ b/chain/gen/genesis/genesis.go
@@ -207,7 +207,6 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
 		}
 
 		// var newAddress address.Address
-
 		if (info.Type == genesis.TAccount) {
 			var ainfo genesis.AccountMeta
 			if err := json.Unmarshal(info.Meta, &ainfo); err != nil {
@@ -230,31 +229,9 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
 			if err := json.Unmarshal(info.Meta, &ainfo); err != nil {
 				return nil, xerrors.Errorf("unmarshaling account meta: %w", err)
 			}
-
-			pending, err := adt.MakeEmptyMap(adt.WrapStore(ctx, cst)).Root()
-			if err != nil {
-				return nil, xerrors.Errorf("failed to create empty map: %v", err)
-			}
-
-			st, err := cst.Put(ctx, &multisig.State{
-				Signers: ainfo.Signers,
-				NumApprovalsThreshold: uint64(ainfo.Threshold),
-				StartEpoch: abi.ChainEpoch(ainfo.VestingStart),
-				UnlockDuration: abi.ChainEpoch(ainfo.VestingDuration),
-				PendingTxns: pending,
-				InitialBalance: info.Balance,
-			})
-			if err != nil {
+			if err = createMultisig(ctx, bs, cst, state, ida, info.Balance, ainfo); err != nil {
 				return nil, err
 			}
-			err = state.SetActor(ida, &types.Actor{
-				Code:    builtin.MultisigActorCodeID,
-				Balance: info.Balance,
-				Head:    st,
-			})
-			if err != nil {
-				return nil, xerrors.Errorf("setting account from actmap: %w", err)
-			}
 		}
 
 	}
@@ -264,22 +241,40 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
 		return nil, err
 	}
 
-	vrst, err := cst.Put(ctx, &account.State{Address: RootVerifierAddr})
-	if err != nil {
+	if err = createMultisig(ctx, bs, cst, state, vregroot, types.NewInt(0), template.VerifregRootKey); err != nil {
 		return nil, err
 	}
 
-	err = state.SetActor(vregroot, &types.Actor{
-		Code:    builtin.AccountActorCodeID,
-		Balance: types.NewInt(0),
-		Head:    vrst,
-	})
+	return state, nil
+}
 
+func createMultisig(ctx context.Context, bs bstore.Blockstore, cst cbor.IpldStore, state *state.StateTree, ida address.Address, balance abi.TokenAmount, ainfo genesis.MultisigMeta) error {
+
+	pending, err := adt.MakeEmptyMap(adt.WrapStore(ctx, cst)).Root()
 	if err != nil {
-		return nil, xerrors.Errorf("setting account from actmap: %w", err)
+		return xerrors.Errorf("failed to create empty map: %v", err)
 	}
 
-	return state, nil
+	st, err := cst.Put(ctx, &multisig.State{
+		Signers: ainfo.Signers,
+		NumApprovalsThreshold: uint64(ainfo.Threshold),
+		StartEpoch: abi.ChainEpoch(ainfo.VestingStart),
+		UnlockDuration: abi.ChainEpoch(ainfo.VestingDuration),
+		PendingTxns: pending,
+		InitialBalance: balance,
+	})
+	if err != nil {
+		return err
+	}
+	err = state.SetActor(ida, &types.Actor{
+		Code:    builtin.MultisigActorCodeID,
+		Balance: balance,
+		Head:    st,
+	})
+	if err != nil {
+		return xerrors.Errorf("setting account from actmap: %w", err)
+	}
+	return nil
 }
 
 func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot cid.Cid, template genesis.Template) (cid.Cid, error) {
diff --git a/genesis/types.go b/genesis/types.go
index acf69298f..69723e81b 100644
--- a/genesis/types.go
+++ b/genesis/types.go
@@ -78,4 +78,6 @@ type Template struct {
 
 	NetworkName string
 	Timestamp   uint64 `json:",omitempty"`
+
+	VerifregRootKey MultisigMeta
 }

From 3cc3c9a34e1a1deb378dbb3cea01d138256c9e66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sami=20M=C3=A4kel=C3=A4?= <mrsmkl@gmail.com>
Date: Mon, 20 Jul 2020 18:03:27 +0300
Subject: [PATCH 03/37] generate an account

---
 chain/gen/genesis/genesis.go  | 108 +++++++++++++++++-----------------
 chain/gen/genesis/t01_init.go |  12 +++-
 chain/gen/genesis/t06_vreg.go |   8 ---
 genesis/types.go              |   2 +-
 scripts/balances.sh           |  74 +++++++++++++++++++++++
 scripts/start-devnet.bash     | 108 ++++++++++++++++++++++++++++++++++
 6 files changed, 246 insertions(+), 66 deletions(-)
 create mode 100755 scripts/balances.sh
 create mode 100755 scripts/start-devnet.bash

diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go
index 56b3f353d..57a5f34bf 100644
--- a/chain/gen/genesis/genesis.go
+++ b/chain/gen/genesis/genesis.go
@@ -130,7 +130,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
 
 	// Create init actor
 
-	initact, err := SetupInitActor(bs, template.NetworkName, template.Accounts)
+	initact, err := SetupInitActor(bs, template.NetworkName, template.Accounts, template.VerifregRootKey)
 	if err != nil {
 		return nil, xerrors.Errorf("setup init actor: %w", err)
 	}
@@ -206,32 +206,8 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
 			return nil, err
 		}
 
-		// var newAddress address.Address
-		if (info.Type == genesis.TAccount) {
-			var ainfo genesis.AccountMeta
-			if err := json.Unmarshal(info.Meta, &ainfo); err != nil {
-				return nil, xerrors.Errorf("unmarshaling account meta: %w", err)
-			}
-			st, err := cst.Put(ctx, &account.State{Address: ainfo.Owner})
-			if err != nil {
-				return nil, err
-			}
-			err = state.SetActor(ida, &types.Actor{
-				Code:    builtin.AccountActorCodeID,
-				Balance: info.Balance,
-				Head:    st,
-			})
-			if err != nil {
-				return nil, xerrors.Errorf("setting account from actmap: %w", err)
-			}
-		} else if (info.Type == genesis.TMultisig) {
-			var ainfo genesis.MultisigMeta
-			if err := json.Unmarshal(info.Meta, &ainfo); err != nil {
-				return nil, xerrors.Errorf("unmarshaling account meta: %w", err)
-			}
-			if err = createMultisig(ctx, bs, cst, state, ida, info.Balance, ainfo); err != nil {
-				return nil, err
-			}
+		if err = createAccount(ctx, bs, cst, state, ida, info); err != nil {
+			return nil, err
 		}
 
 	}
@@ -241,39 +217,63 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
 		return nil, err
 	}
 
-	if err = createMultisig(ctx, bs, cst, state, vregroot, types.NewInt(0), template.VerifregRootKey); err != nil {
+	if err = createAccount(ctx, bs, cst, state, vregroot, template.VerifregRootKey); err != nil {
 		return nil, err
 	}
 
 	return state, nil
 }
 
-func createMultisig(ctx context.Context, bs bstore.Blockstore, cst cbor.IpldStore, state *state.StateTree, ida address.Address, balance abi.TokenAmount, ainfo genesis.MultisigMeta) error {
-
-	pending, err := adt.MakeEmptyMap(adt.WrapStore(ctx, cst)).Root()
-	if err != nil {
-		return xerrors.Errorf("failed to create empty map: %v", err)
+func createAccount(ctx context.Context, bs bstore.Blockstore, cst cbor.IpldStore, state *state.StateTree, ida address.Address, info genesis.Actor) error {
+	// var newAddress address.Address
+	if (info.Type == genesis.TAccount) {
+		var ainfo genesis.AccountMeta
+		if err := json.Unmarshal(info.Meta, &ainfo); err != nil {
+			return xerrors.Errorf("unmarshaling account meta: %w", err)
+		}
+		st, err := cst.Put(ctx, &account.State{Address: ainfo.Owner})
+		if err != nil {
+			return err
+		}
+		err = state.SetActor(ida, &types.Actor{
+			Code:    builtin.AccountActorCodeID,
+			Balance: info.Balance,
+			Head:    st,
+		})
+		if err != nil {
+			return xerrors.Errorf("setting account from actmap: %w", err)
+		}
+	} else if (info.Type == genesis.TMultisig) {
+		var ainfo genesis.MultisigMeta
+		if err := json.Unmarshal(info.Meta, &ainfo); err != nil {
+			return xerrors.Errorf("unmarshaling account meta: %w", err)
+		}
+		pending, err := adt.MakeEmptyMap(adt.WrapStore(ctx, cst)).Root()
+		if err != nil {
+			return xerrors.Errorf("failed to create empty map: %v", err)
+		}
+	
+		st, err := cst.Put(ctx, &multisig.State{
+			Signers: ainfo.Signers,
+			NumApprovalsThreshold: uint64(ainfo.Threshold),
+			StartEpoch: abi.ChainEpoch(ainfo.VestingStart),
+			UnlockDuration: abi.ChainEpoch(ainfo.VestingDuration),
+			PendingTxns: pending,
+			InitialBalance: info.Balance,
+		})
+		if err != nil {
+			return err
+		}
+		err = state.SetActor(ida, &types.Actor{
+			Code:    builtin.MultisigActorCodeID,
+			Balance: info.Balance,
+			Head:    st,
+		})
+		if err != nil {
+			return xerrors.Errorf("setting account from actmap: %w", err)
+		}
 	}
 
-	st, err := cst.Put(ctx, &multisig.State{
-		Signers: ainfo.Signers,
-		NumApprovalsThreshold: uint64(ainfo.Threshold),
-		StartEpoch: abi.ChainEpoch(ainfo.VestingStart),
-		UnlockDuration: abi.ChainEpoch(ainfo.VestingDuration),
-		PendingTxns: pending,
-		InitialBalance: balance,
-	})
-	if err != nil {
-		return err
-	}
-	err = state.SetActor(ida, &types.Actor{
-		Code:    builtin.MultisigActorCodeID,
-		Balance: balance,
-		Head:    st,
-	})
-	if err != nil {
-		return xerrors.Errorf("setting account from actmap: %w", err)
-	}
 	return nil
 }
 
@@ -298,13 +298,13 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci
 		return cid.Undef, xerrors.Errorf("failed to create NewVM: %w", err)
 	}
 
-	_, err = doExecValue(ctx, vm, builtin.VerifiedRegistryActorAddr, RootVerifierAddr, types.NewInt(0), builtin.MethodsVerifiedRegistry.AddVerifier, mustEnc(&verifreg.AddVerifierParams{
+	_, err = doExecValue(ctx, vm, builtin.VerifiedRegistryActorAddr, verifier, types.NewInt(0), builtin.MethodsVerifiedRegistry.AddVerifier, mustEnc(&verifreg.AddVerifierParams{
 		Address:   verifier,
 		Allowance: abi.NewStoragePower(int64(sum)), // eh, close enough
 
 	}))
 	if err != nil {
-		return cid.Undef, xerrors.Errorf("failed to failed to create verifier: %w", err)
+		return cid.Undef, xerrors.Errorf("failed to create verifier: %w", err)
 	}
 
 	for c, amt := range verifNeeds {
diff --git a/chain/gen/genesis/t01_init.go b/chain/gen/genesis/t01_init.go
index d37a9c306..62f892c47 100644
--- a/chain/gen/genesis/t01_init.go
+++ b/chain/gen/genesis/t01_init.go
@@ -17,7 +17,7 @@ import (
 	"github.com/filecoin-project/lotus/genesis"
 )
 
-func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesis.Actor) (*types.Actor, error) {
+func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesis.Actor, rootVerifier genesis.Actor) (*types.Actor, error) {
 	if len(initialActors) > MaxAccounts {
 		return nil, xerrors.New("too many initial actors")
 	}
@@ -50,8 +50,14 @@ func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesi
 		}
 	}
 
-	if err := amap.Set(context.TODO(), string(RootVerifierAddr.Bytes()), 80); err != nil {
-		return nil, err
+	if rootVerifier.Type == genesis.TAccount {
+		var ainfo genesis.AccountMeta
+		if err := json.Unmarshal(rootVerifier.Meta, &ainfo); err != nil {
+			return nil, xerrors.Errorf("unmarshaling account meta: %w", err)
+		}
+		if err := amap.Set(context.TODO(), string(ainfo.Owner.Bytes()), 80); err != nil {
+			return nil, err
+		}
 	}
 
 	if err := amap.Flush(context.TODO()); err != nil {
diff --git a/chain/gen/genesis/t06_vreg.go b/chain/gen/genesis/t06_vreg.go
index 093925d3e..4e77b924e 100644
--- a/chain/gen/genesis/t06_vreg.go
+++ b/chain/gen/genesis/t06_vreg.go
@@ -14,17 +14,9 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
-var RootVerifierAddr address.Address
-
 var RootVerifierID address.Address
 
 func init() {
-	k, err := address.NewFromString("t3qfoulel6fy6gn3hjmbhpdpf6fs5aqjb5fkurhtwvgssizq4jey5nw4ptq5up6h7jk7frdvvobv52qzmgjinq")
-	if err != nil {
-		panic(err)
-	}
-
-	RootVerifierAddr = k
 
 	idk, err := address.NewFromString("t080")
 	if err != nil {
diff --git a/genesis/types.go b/genesis/types.go
index 69723e81b..7d401fd0e 100644
--- a/genesis/types.go
+++ b/genesis/types.go
@@ -79,5 +79,5 @@ type Template struct {
 	NetworkName string
 	Timestamp   uint64 `json:",omitempty"`
 
-	VerifregRootKey MultisigMeta
+	VerifregRootKey Actor
 }
diff --git a/scripts/balances.sh b/scripts/balances.sh
new file mode 100755
index 000000000..949070f7f
--- /dev/null
+++ b/scripts/balances.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+
+sleep 20
+
+lotus wait-api
+
+lotus chain head
+
+export MAIN=$(cat ../localnet2.json | jq -r '.Accounts | .[0] | .Meta .Owner')
+
+# Send funds to root key
+lotus send --from $MAIN $ROOT 5000000
+
+export VERIFIER=$(lotus wallet new)
+export CLIENT=$(lotus wallet new)
+
+# Send funds to verifier
+lotus send --from $MAIN $VERIFIER 5000000
+
+# Send funds to client
+lotus send --from $MAIN $CLIENT 5000000
+
+while [ "5000000 FIL" != "$(lotus wallet balance $VERIFIER)" ]
+do
+ sleep 1
+ lotus wallet balance $ROOT
+done
+
+# export PARAM=$(lotus-shed verifreg add-verifier --dry t01001 100000000000000000000000000000000000000000)
+export PARAM=824300e907440076adf1
+
+# Add verifier
+# lotus-shed verifreg add-verifier --from $ROOT t01001 100000000000000000000000000000000000000000
+
+lotus msig propose --from $MAIN t080 t06 0 2 $PARAM
+lotus msig inspect t080
+
+lotus-shed verifreg list-verifiers
+
+# Add verified client
+lotus-shed verifreg verify-client --from $VERIFIER $CLIENT 10000
+lotus-shed verifreg list-clients
+
+# Remove verifier datacap
+lotus-shed verifreg add-verifier --from $ROOT t01001 0
+lotus-shed verifreg list-verifiers
+
+export DATA=$(lotus client import dddd | awk '{print $NF}')
+
+lotus client local
+
+# Client can make a verified deal
+lotus client deal --verified-deal --from $CLIENT $DATA t01000 0.005 100000
+
+while [ "3" != "$(lotus-miner sectors list | wc -l)" ]
+do
+ sleep 10
+ lotus-miner sectors list
+done
+
+curl -H "Content-Type: application/json" -H "Authorization: Bearer $(cat ~/.lotusminer/token)" -d '{"id": 1, "method": "Filecoin.SectorStartSealing", "params": [2]}' localhost:2345/rpc/v0
+
+lotus-miner info
+
+lotus-miner sectors list
+
+while [ "3" != "$(lotus-miner sectors list | grep Proving | wc -l)" ]
+do
+ sleep 5
+ lotus-miner sectors list | tail -n 1
+ lotus-miner info | grep "Actual Power"
+done
+
+sleep 300000
diff --git a/scripts/start-devnet.bash b/scripts/start-devnet.bash
new file mode 100755
index 000000000..451807ecc
--- /dev/null
+++ b/scripts/start-devnet.bash
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+
+
+__PWD=$PWD
+__DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+__PARENT_DIR=$(dirname $__DIR)
+LOTUS_PATH="${1:-$__PARENT_DIR}"
+LOTUS_PURGE_LOCALDIR=${LOTUS_PURGE_LOCALDIR:-false}
+LOTUS_COMPILE=${LOTUS_COMPILE:-false}
+
+
+LOTUS_BIN="$LOTUS_PATH/lotus"
+LOTUS_SEED="$LOTUS_PATH/lotus-seed"
+LOTUS_MINER="$LOTUS_PATH/lotus-miner"
+
+
+##### Functions
+
+configure_lotus() {
+
+  echo -e "\nDownloading the 2048 byte parameters:\n"
+  $LOTUS_BIN fetch-params 2048
+
+  echo -e "\nPre-seal some sectors:\n"
+  $LOTUS_SEED pre-seal --sector-size 2KiB --num-sectors 2
+#  $LOTUS_SEED pre-seal --sector-size 2KiB --num-sectors 2 --miner-addr t01001
+
+#  export ROOT=$(cat ~/.genesis-sectors/pre-seal-t01001.json | jq -r '.t01001 | .Owner')
+  export MINER=$(cat ~/.genesis-sectors/pre-seal-t01000.json | jq -r '.t01000 | .Owner')
+
+  echo -e "\nCreate the genesis block and start up the first node:\n"
+  $LOTUS_SEED genesis new $LOTUS_PATH/localnet.json
+  $LOTUS_SEED genesis add-miner $LOTUS_PATH/localnet.json ~/.genesis-sectors/pre-seal-t01000.json
+  # $LOTUS_SEED genesis add-msigs $LOTUS_PATH/localnet.json test.csv
+  jq --arg root $MINER '. + {VerifregRootKey: {Meta: {Signers: [$root], Threshold: 1 }, Type: "multisig", Balance: "50000000000000000000000000"}}' ../localnet.json > $LOTUS_PATH/localnet2.json
+  # cp $LOTUS_PATH/localnet.json $LOTUS_PATH/localnet2.json
+  # jq --arg root $ROOT '. + {RootKey: $root}' ../localnet.json > $LOTUS_PATH/localnet2.json
+  # jq --arg root $MINER '.Accounts |= . + [{Type: "multisig", Balance: "50000000000000000000000000", Meta: { Signers: [$root], Threshold: 1 }}]' ../localnet.json > $LOTUS_PATH/localnet2.json
+
+  tmux new-session -s lotus -n script -d bash balances.sh
+
+  sleep 5
+  tmux new-window -t lotus:1 -n daemon -d $LOTUS_BIN daemon --lotus-make-genesis=dev.gen --genesis-template=$LOTUS_PATH/localnet2.json --bootstrap=false
+  # $LOTUS_BIN daemon --lotus-make-genesis=dev.gen --genesis-template=$LOTUS_PATH/localnet2.json --bootstrap=false || exit
+
+  sleep 5
+  echo -e "\nImporting the genesis miner key:\n"
+  $LOTUS_BIN wallet import ~/.genesis-sectors/pre-seal-t01000.key
+#  $LOTUS_BIN wallet import ~/.genesis-sectors/pre-seal-t01001.key
+
+  sleep 3
+  echo -e "\nSetting up the genesis miner:\n"
+  $LOTUS_MINER init --genesis-miner --actor=t01000 --sector-size=2KiB \
+    --pre-sealed-sectors=~/.genesis-sectors \
+    --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t01000.json --nosync
+
+  sleep 3
+  echo -e "\nStarting up the miner:\n"
+  tmux new-window -t lotus:2 -n miner -d $LOTUS_MINER run --nosync
+}
+
+
+
+purge_local_dirs() {
+  echo -e "\nRemoving local state\n"
+  rm -rf ~/.lotus || true
+  rm -rf ~/.lotusstorage || true
+  rm -rf ~/.lotusminer || true
+  rm -rf ~/.genesis-sectors || true
+}
+
+
+compile_lotus() {
+  echo -e "\nCompiling lotus binary for devnet usage\n"
+  make clean && make 2k all
+  sudo make install
+
+}
+
+main() {
+
+  echo -e "\nUsing Lotus Path: $LOTUS_PATH \n"
+
+  if [[ "$LOTUS_PURGE_LOCALDIR" = true ]]; then
+    echo -e "Purging"
+    purge_local_dirs
+  fi
+
+
+  if [[ "$LOTUS_COMPILE" = true ]]; then
+    compile_lotus
+  fi
+
+
+  if [[ ! -f $LOTUS_BIN ]]; then
+    echo -e "Lotus binary doesn't exist"
+    echo -e "Please visit https://docs.lotu.sh/en+getting-started for compiling Lotus"
+    echo -e "Please remember to compile lotus with the 2k flag:"
+    echo -e "make 2k"
+    exit 1
+  fi
+
+  configure_lotus
+}
+
+#### Main
+
+main

From 510b9cfac62f47fc88f3ecc3c650981a6e0b952c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sami=20M=C3=A4kel=C3=A4?= <mrsmkl@gmail.com>
Date: Mon, 20 Jul 2020 18:05:17 +0300
Subject: [PATCH 04/37] cleanup

---
 scripts/balances.sh       |  74 --------------------------
 scripts/start-devnet.bash | 108 --------------------------------------
 2 files changed, 182 deletions(-)
 delete mode 100755 scripts/balances.sh
 delete mode 100755 scripts/start-devnet.bash

diff --git a/scripts/balances.sh b/scripts/balances.sh
deleted file mode 100755
index 949070f7f..000000000
--- a/scripts/balances.sh
+++ /dev/null
@@ -1,74 +0,0 @@
-#!/bin/bash
-
-sleep 20
-
-lotus wait-api
-
-lotus chain head
-
-export MAIN=$(cat ../localnet2.json | jq -r '.Accounts | .[0] | .Meta .Owner')
-
-# Send funds to root key
-lotus send --from $MAIN $ROOT 5000000
-
-export VERIFIER=$(lotus wallet new)
-export CLIENT=$(lotus wallet new)
-
-# Send funds to verifier
-lotus send --from $MAIN $VERIFIER 5000000
-
-# Send funds to client
-lotus send --from $MAIN $CLIENT 5000000
-
-while [ "5000000 FIL" != "$(lotus wallet balance $VERIFIER)" ]
-do
- sleep 1
- lotus wallet balance $ROOT
-done
-
-# export PARAM=$(lotus-shed verifreg add-verifier --dry t01001 100000000000000000000000000000000000000000)
-export PARAM=824300e907440076adf1
-
-# Add verifier
-# lotus-shed verifreg add-verifier --from $ROOT t01001 100000000000000000000000000000000000000000
-
-lotus msig propose --from $MAIN t080 t06 0 2 $PARAM
-lotus msig inspect t080
-
-lotus-shed verifreg list-verifiers
-
-# Add verified client
-lotus-shed verifreg verify-client --from $VERIFIER $CLIENT 10000
-lotus-shed verifreg list-clients
-
-# Remove verifier datacap
-lotus-shed verifreg add-verifier --from $ROOT t01001 0
-lotus-shed verifreg list-verifiers
-
-export DATA=$(lotus client import dddd | awk '{print $NF}')
-
-lotus client local
-
-# Client can make a verified deal
-lotus client deal --verified-deal --from $CLIENT $DATA t01000 0.005 100000
-
-while [ "3" != "$(lotus-miner sectors list | wc -l)" ]
-do
- sleep 10
- lotus-miner sectors list
-done
-
-curl -H "Content-Type: application/json" -H "Authorization: Bearer $(cat ~/.lotusminer/token)" -d '{"id": 1, "method": "Filecoin.SectorStartSealing", "params": [2]}' localhost:2345/rpc/v0
-
-lotus-miner info
-
-lotus-miner sectors list
-
-while [ "3" != "$(lotus-miner sectors list | grep Proving | wc -l)" ]
-do
- sleep 5
- lotus-miner sectors list | tail -n 1
- lotus-miner info | grep "Actual Power"
-done
-
-sleep 300000
diff --git a/scripts/start-devnet.bash b/scripts/start-devnet.bash
deleted file mode 100755
index 451807ecc..000000000
--- a/scripts/start-devnet.bash
+++ /dev/null
@@ -1,108 +0,0 @@
-#!/usr/bin/env bash
-
-
-__PWD=$PWD
-__DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-__PARENT_DIR=$(dirname $__DIR)
-LOTUS_PATH="${1:-$__PARENT_DIR}"
-LOTUS_PURGE_LOCALDIR=${LOTUS_PURGE_LOCALDIR:-false}
-LOTUS_COMPILE=${LOTUS_COMPILE:-false}
-
-
-LOTUS_BIN="$LOTUS_PATH/lotus"
-LOTUS_SEED="$LOTUS_PATH/lotus-seed"
-LOTUS_MINER="$LOTUS_PATH/lotus-miner"
-
-
-##### Functions
-
-configure_lotus() {
-
-  echo -e "\nDownloading the 2048 byte parameters:\n"
-  $LOTUS_BIN fetch-params 2048
-
-  echo -e "\nPre-seal some sectors:\n"
-  $LOTUS_SEED pre-seal --sector-size 2KiB --num-sectors 2
-#  $LOTUS_SEED pre-seal --sector-size 2KiB --num-sectors 2 --miner-addr t01001
-
-#  export ROOT=$(cat ~/.genesis-sectors/pre-seal-t01001.json | jq -r '.t01001 | .Owner')
-  export MINER=$(cat ~/.genesis-sectors/pre-seal-t01000.json | jq -r '.t01000 | .Owner')
-
-  echo -e "\nCreate the genesis block and start up the first node:\n"
-  $LOTUS_SEED genesis new $LOTUS_PATH/localnet.json
-  $LOTUS_SEED genesis add-miner $LOTUS_PATH/localnet.json ~/.genesis-sectors/pre-seal-t01000.json
-  # $LOTUS_SEED genesis add-msigs $LOTUS_PATH/localnet.json test.csv
-  jq --arg root $MINER '. + {VerifregRootKey: {Meta: {Signers: [$root], Threshold: 1 }, Type: "multisig", Balance: "50000000000000000000000000"}}' ../localnet.json > $LOTUS_PATH/localnet2.json
-  # cp $LOTUS_PATH/localnet.json $LOTUS_PATH/localnet2.json
-  # jq --arg root $ROOT '. + {RootKey: $root}' ../localnet.json > $LOTUS_PATH/localnet2.json
-  # jq --arg root $MINER '.Accounts |= . + [{Type: "multisig", Balance: "50000000000000000000000000", Meta: { Signers: [$root], Threshold: 1 }}]' ../localnet.json > $LOTUS_PATH/localnet2.json
-
-  tmux new-session -s lotus -n script -d bash balances.sh
-
-  sleep 5
-  tmux new-window -t lotus:1 -n daemon -d $LOTUS_BIN daemon --lotus-make-genesis=dev.gen --genesis-template=$LOTUS_PATH/localnet2.json --bootstrap=false
-  # $LOTUS_BIN daemon --lotus-make-genesis=dev.gen --genesis-template=$LOTUS_PATH/localnet2.json --bootstrap=false || exit
-
-  sleep 5
-  echo -e "\nImporting the genesis miner key:\n"
-  $LOTUS_BIN wallet import ~/.genesis-sectors/pre-seal-t01000.key
-#  $LOTUS_BIN wallet import ~/.genesis-sectors/pre-seal-t01001.key
-
-  sleep 3
-  echo -e "\nSetting up the genesis miner:\n"
-  $LOTUS_MINER init --genesis-miner --actor=t01000 --sector-size=2KiB \
-    --pre-sealed-sectors=~/.genesis-sectors \
-    --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t01000.json --nosync
-
-  sleep 3
-  echo -e "\nStarting up the miner:\n"
-  tmux new-window -t lotus:2 -n miner -d $LOTUS_MINER run --nosync
-}
-
-
-
-purge_local_dirs() {
-  echo -e "\nRemoving local state\n"
-  rm -rf ~/.lotus || true
-  rm -rf ~/.lotusstorage || true
-  rm -rf ~/.lotusminer || true
-  rm -rf ~/.genesis-sectors || true
-}
-
-
-compile_lotus() {
-  echo -e "\nCompiling lotus binary for devnet usage\n"
-  make clean && make 2k all
-  sudo make install
-
-}
-
-main() {
-
-  echo -e "\nUsing Lotus Path: $LOTUS_PATH \n"
-
-  if [[ "$LOTUS_PURGE_LOCALDIR" = true ]]; then
-    echo -e "Purging"
-    purge_local_dirs
-  fi
-
-
-  if [[ "$LOTUS_COMPILE" = true ]]; then
-    compile_lotus
-  fi
-
-
-  if [[ ! -f $LOTUS_BIN ]]; then
-    echo -e "Lotus binary doesn't exist"
-    echo -e "Please visit https://docs.lotu.sh/en+getting-started for compiling Lotus"
-    echo -e "Please remember to compile lotus with the 2k flag:"
-    echo -e "make 2k"
-    exit 1
-  fi
-
-  configure_lotus
-}
-
-#### Main
-
-main

From d254fb228bcf08619960c3b69f49d9572bb718b5 Mon Sep 17 00:00:00 2001
From: Aayush Rajasekaran <arajasek94@gmail.com>
Date: Tue, 21 Jul 2020 13:44:05 -0400
Subject: [PATCH 05/37] API documentation must not tell lies

---
 api/api_full.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api/api_full.go b/api/api_full.go
index 3b7188ec4..10f032c4a 100644
--- a/api/api_full.go
+++ b/api/api_full.go
@@ -57,7 +57,7 @@ type FullNode interface {
 	ChainGetParentMessages(ctx context.Context, blockCid cid.Cid) ([]Message, error)
 
 	// ChainGetTipSetByHeight looks back for a tipset at the specified epoch.
-	// If there are no blocks at the specified epoch, a tipset at higher epoch
+	// If there are no blocks at the specified epoch, a tipset at an earlier epoch
 	// will be returned.
 	ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)
 

From 556408a6f248e7c61aecc00715fbdb06d9399ca4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sami=20M=C3=A4kel=C3=A4?= <mrsmkl@gmail.com>
Date: Thu, 23 Jul 2020 18:07:57 +0300
Subject: [PATCH 06/37] add a default verifreg rootkey

---
 cmd/lotus-seed/genesis.go | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/cmd/lotus-seed/genesis.go b/cmd/lotus-seed/genesis.go
index 45397af35..13880e7be 100644
--- a/cmd/lotus-seed/genesis.go
+++ b/cmd/lotus-seed/genesis.go
@@ -46,10 +46,21 @@ var genesisNewCmd = &cli.Command{
 		if !cctx.Args().Present() {
 			return xerrors.New("seed genesis new [genesis.json]")
 		}
-
+		rootkey, _ := address.NewIDAddress(80)
+		defaultRootkey := genesis.MultisigMeta{
+			Signers:         []address.Address{rootkey},
+			Threshold:       1,
+			VestingDuration: 0,
+			VestingStart:    0,
+		}
 		out := genesis.Template{
-			Accounts:    []genesis.Actor{},
-			Miners:      []genesis.Miner{},
+			Accounts: []genesis.Actor{},
+			Miners:   []genesis.Miner{},
+			VerifregRootKey: genesis.Actor{
+				Type:    genesis.TMultisig,
+				Balance: big.NewInt(0),
+				Meta:    defaultRootkey.ActorMeta(),
+			},
 			NetworkName: cctx.String("network-name"),
 		}
 		if out.NetworkName == "" {

From 5a6e12dc9726134e9f54c3cf440a21a0214c3c7b Mon Sep 17 00:00:00 2001
From: Aayush Rajasekaran <arajasek94@gmail.com>
Date: Thu, 23 Jul 2020 20:34:27 -0400
Subject: [PATCH 07/37] Genesis miners need to be registered in the market
 actor's balance table

---
 chain/gen/genesis/genesis.go | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go
index b8ede8856..615b7b140 100644
--- a/chain/gen/genesis/genesis.go
+++ b/chain/gen/genesis/genesis.go
@@ -326,7 +326,16 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
 func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot cid.Cid, template genesis.Template) (cid.Cid, error) {
 	verifNeeds := make(map[address.Address]abi.PaddedPieceSize)
 	var sum abi.PaddedPieceSize
+
+	vm, err := vm.NewVM(stateroot, 0, &fakeRand{}, cs.Blockstore(), mkFakedSigSyscalls(cs.VMSys()))
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("failed to create NewVM: %w", err)
+	}
+
 	for _, m := range template.Miners {
+
+		// Add the miner to the market actor's balance table
+		_, err = doExec(ctx, vm, builtin.StorageMarketActorAddr, m.Owner, builtin.MethodsMarket.AddBalance, mustEnc(adt.Empty))
 		for _, s := range m.Sectors {
 			amt := s.Deal.PieceSize
 			verifNeeds[s.Deal.Client] += amt
@@ -339,11 +348,6 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci
 		return cid.Undef, err
 	}
 
-	vm, err := vm.NewVM(stateroot, 0, &fakeRand{}, cs.Blockstore(), mkFakedSigSyscalls(cs.VMSys()))
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("failed to create NewVM: %w", err)
-	}
-
 	_, err = doExecValue(ctx, vm, builtin.VerifiedRegistryActorAddr, RootVerifierAddr, types.NewInt(0), builtin.MethodsVerifiedRegistry.AddVerifier, mustEnc(&verifreg.AddVerifierParams{
 		Address:   verifier,
 		Allowance: abi.NewStoragePower(int64(sum)), // eh, close enough

From 2cf59a23e7113c7773491efb9766dfc30ea8cc1a Mon Sep 17 00:00:00 2001
From: whyrusleeping <why@ipfs.io>
Date: Thu, 23 Jul 2020 18:52:29 -0700
Subject: [PATCH 08/37] working on lotus client UX

---
 cli/client.go | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/cli/client.go b/cli/client.go
index 5cc16480c..693b2b83e 100644
--- a/cli/client.go
+++ b/cli/client.go
@@ -56,17 +56,17 @@ var clientCmd = &cli.Command{
 	Name:  "client",
 	Usage: "Make deals, store data, retrieve data",
 	Subcommands: []*cli.Command{
-		clientImportCmd,
-		clientDropCmd,
-		clientCommPCmd,
-		clientLocalCmd,
-		clientDealCmd,
-		clientFindCmd,
-		clientRetrieveCmd,
-		clientQueryAskCmd,
-		clientListDeals,
-		clientCarGenCmd,
-		clientGetDealCmd,
+		WithCategory("storage", clientDealCmd),
+		WithCategory("storage", clientQueryAskCmd),
+		WithCategory("storage", clientListDeals),
+		WithCategory("storage", clientGetDealCmd),
+		WithCategory("data", clientImportCmd),
+		WithCategory("data", clientDropCmd),
+		WithCategory("data", clientLocalCmd),
+		WithCategory("retrieval", clientFindCmd),
+		WithCategory("retrieval", clientRetrieveCmd),
+		WithCategory("util", clientCommPCmd),
+		WithCategory("util", clientCarGenCmd),
 	},
 }
 
@@ -164,7 +164,7 @@ var clientDropCmd = &cli.Command{
 
 var clientCommPCmd = &cli.Command{
 	Name:      "commP",
-	Usage:     "calculate the piece-cid (commP) of a CAR file",
+	Usage:     "Calculate the piece-cid (commP) of a CAR file",
 	ArgsUsage: "[inputFile minerAddress]",
 	Flags: []cli.Flag{
 		&CidBaseFlag,
@@ -204,7 +204,7 @@ var clientCommPCmd = &cli.Command{
 
 var clientCarGenCmd = &cli.Command{
 	Name:      "generate-car",
-	Usage:     "generate a car file from input",
+	Usage:     "Generate a car file from input",
 	ArgsUsage: "[inputPath outputPath]",
 	Action: func(cctx *cli.Context) error {
 		api, closer, err := GetFullNodeAPI(cctx)
@@ -429,7 +429,7 @@ var clientDealCmd = &cli.Command{
 
 var clientFindCmd = &cli.Command{
 	Name:      "find",
-	Usage:     "find data in the network",
+	Usage:     "Find data in the network",
 	ArgsUsage: "[dataCid]",
 	Flags: []cli.Flag{
 		&cli.StringFlag{
@@ -496,7 +496,7 @@ const DefaultMaxRetrievePrice = 1
 
 var clientRetrieveCmd = &cli.Command{
 	Name:      "retrieve",
-	Usage:     "retrieve data from network",
+	Usage:     "Retrieve data from network",
 	ArgsUsage: "[dataCid outputPath]",
 	Flags: []cli.Flag{
 		&cli.StringFlag{
@@ -642,7 +642,7 @@ var clientRetrieveCmd = &cli.Command{
 
 var clientQueryAskCmd = &cli.Command{
 	Name:      "query-ask",
-	Usage:     "find a miners ask",
+	Usage:     "Find a miners ask",
 	ArgsUsage: "[minerAddress]",
 	Flags: []cli.Flag{
 		&cli.StringFlag{

From 9207315b25d6b3b34f2629c72d2851796e94d5c8 Mon Sep 17 00:00:00 2001
From: lanzafame <adrianlanzafame92@gmail.com>
Date: Fri, 24 Jul 2020 15:47:41 +1000
Subject: [PATCH 09/37] Change block validation aggregation type from sum to
 distribution

---
 chain/sync.go      |  6 ++----
 metrics/metrics.go | 16 ++++++++++++++--
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/chain/sync.go b/chain/sync.go
index 20475171e..22edd4060 100644
--- a/chain/sync.go
+++ b/chain/sync.go
@@ -634,10 +634,8 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
 
 	validationStart := build.Clock.Now()
 	defer func() {
-		dur := time.Since(validationStart)
-		durMilli := dur.Seconds() * float64(1000)
-		stats.Record(ctx, metrics.BlockValidationDurationMilliseconds.M(durMilli))
-		log.Infow("block validation", "took", dur, "height", b.Header.Height)
+		stats.Record(ctx, metrics.BlockValidationDurationMilliseconds.M(metrics.SinceInMilliseconds(validationStart)))
+		log.Infow("block validation", "took", time.Since(validationStart), "height", b.Header.Height)
 	}()
 
 	ctx, span := trace.StartSpan(ctx, "validateBlock")
diff --git a/metrics/metrics.go b/metrics/metrics.go
index 0ef63de4f..5ed34a620 100644
--- a/metrics/metrics.go
+++ b/metrics/metrics.go
@@ -1,6 +1,8 @@
 package metrics
 
 import (
+	"time"
+
 	"go.opencensus.io/stats"
 	"go.opencensus.io/stats/view"
 	"go.opencensus.io/tag"
@@ -8,6 +10,9 @@ import (
 	rpcmetrics "github.com/filecoin-project/go-jsonrpc/metrics"
 )
 
+// Distribution
+var defaultMillisecondsDistribution = view.Distribution(0.01, 0.05, 0.1, 0.3, 0.6, 0.8, 1, 2, 3, 4, 5, 6, 8, 10, 13, 16, 20, 25, 30, 40, 50, 65, 80, 100, 130, 160, 200, 250, 300, 400, 500, 650, 800, 1000, 2000, 5000, 10000, 20000, 50000, 100000)
+
 // Global Tags
 var (
 	Version, _      = tag.NewKey("version")
@@ -66,7 +71,7 @@ var (
 	}
 	BlockValidationDurationView = &view.View{
 		Measure:     BlockValidationDurationMilliseconds,
-		Aggregation: view.Sum(),
+		Aggregation: defaultMillisecondsDistribution,
 	}
 	MessageReceivedView = &view.View{
 		Measure:     MessageReceived,
@@ -99,4 +104,11 @@ var DefaultViews = append([]*view.View{
 	MessageReceivedView,
 	MessageValidationFailureView,
 	MessageValidationSuccessView,
-	PeerCountView}, rpcmetrics.DefaultViews...)
+	PeerCountView,
+	},
+	rpcmetrics.DefaultViews...)
+
+// SinceInMilliseconds returns the duration of time since the provide time as a float64.
+func SinceInMilliseconds(startTime time.Time) float64 {
+	return float64(time.Since(startTime).Nanoseconds()) / 1e6
+}
\ No newline at end of file

From d90fac6e09712b1145dc6bd818435f9c93631652 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sami=20M=C3=A4kel=C3=A4?= <mrsmkl@gmail.com>
Date: Fri, 24 Jul 2020 12:22:50 +0300
Subject: [PATCH 10/37] tests are working now

---
 chain/gen/gen.go          | 21 +++++++++++++++++++--
 cmd/lotus-seed/genesis.go | 20 +++++---------------
 node/node_test.go         | 16 +++++++++-------
 3 files changed, 33 insertions(+), 24 deletions(-)

diff --git a/chain/gen/gen.go b/chain/gen/gen.go
index 206b6c8cf..4d535b5f4 100644
--- a/chain/gen/gen.go
+++ b/chain/gen/gen.go
@@ -10,6 +10,7 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	saminer "github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-actors/actors/crypto"
 	block "github.com/ipfs/go-block-format"
@@ -91,6 +92,21 @@ func (m mybs) Get(c cid.Cid) (block.Block, error) {
 	return b, nil
 }
 
+var rootkey, _ = address.NewIDAddress(80)
+
+var rootkeyMultisig = genesis.MultisigMeta{
+	Signers:         []address.Address{rootkey},
+	Threshold:       1,
+	VestingDuration: 0,
+	VestingStart:    0,
+}
+
+var DefaultVerifregRootkeyActor = genesis.Actor{
+	Type:    genesis.TMultisig,
+	Balance: big.NewInt(0),
+	Meta:    rootkeyMultisig.ActorMeta(),
+}
+
 func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
 	saminer.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{
 		abi.RegisteredSealProof_StackedDrg2KiBV1: {},
@@ -194,8 +210,9 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
 			*genm1,
 			*genm2,
 		},
-		NetworkName: "",
-		Timestamp:   uint64(build.Clock.Now().Add(-500 * time.Duration(build.BlockDelaySecs) * time.Second).Unix()),
+		VerifregRootKey: DefaultVerifregRootkeyActor,
+		NetworkName:     "",
+		Timestamp:       uint64(build.Clock.Now().Add(-500 * time.Duration(build.BlockDelaySecs) * time.Second).Unix()),
 	}
 
 	genb, err := genesis2.MakeGenesisBlock(context.TODO(), bs, sys, tpl)
diff --git a/cmd/lotus-seed/genesis.go b/cmd/lotus-seed/genesis.go
index 13880e7be..478e30b97 100644
--- a/cmd/lotus-seed/genesis.go
+++ b/cmd/lotus-seed/genesis.go
@@ -19,6 +19,7 @@ import (
 	"github.com/filecoin-project/specs-actors/actors/abi/big"
 
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/gen"
 	genesis2 "github.com/filecoin-project/lotus/chain/gen/genesis"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/genesis"
@@ -46,22 +47,11 @@ var genesisNewCmd = &cli.Command{
 		if !cctx.Args().Present() {
 			return xerrors.New("seed genesis new [genesis.json]")
 		}
-		rootkey, _ := address.NewIDAddress(80)
-		defaultRootkey := genesis.MultisigMeta{
-			Signers:         []address.Address{rootkey},
-			Threshold:       1,
-			VestingDuration: 0,
-			VestingStart:    0,
-		}
 		out := genesis.Template{
-			Accounts: []genesis.Actor{},
-			Miners:   []genesis.Miner{},
-			VerifregRootKey: genesis.Actor{
-				Type:    genesis.TMultisig,
-				Balance: big.NewInt(0),
-				Meta:    defaultRootkey.ActorMeta(),
-			},
-			NetworkName: cctx.String("network-name"),
+			Accounts:        []genesis.Actor{},
+			Miners:          []genesis.Miner{},
+			VerifregRootKey: gen.DefaultVerifregRootkeyActor,
+			NetworkName:     cctx.String("network-name"),
 		}
 		if out.NetworkName == "" {
 			out.NetworkName = "localnet-" + uuid.New().String()
diff --git a/node/node_test.go b/node/node_test.go
index 714449dac..b729217b2 100644
--- a/node/node_test.go
+++ b/node/node_test.go
@@ -36,6 +36,7 @@ import (
 	"github.com/filecoin-project/lotus/api/test"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/gen"
 	genesis2 "github.com/filecoin-project/lotus/chain/gen/genesis"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/wallet"
@@ -211,11 +212,11 @@ func builder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestN
 		maddrs = append(maddrs, maddr)
 		genms = append(genms, *genm)
 	}
-
 	templ := &genesis.Template{
-		Accounts:  genaccs,
-		Miners:    genms,
-		Timestamp: uint64(time.Now().Unix() - 10000), // some time sufficiently far in the past
+		Accounts:        genaccs,
+		Miners:          genms,
+		Timestamp:       uint64(time.Now().Unix() - 10000), // some time sufficiently far in the past
+		VerifregRootKey: gen.DefaultVerifregRootkeyActor,
 	}
 
 	// END PRESEAL SECTION
@@ -347,9 +348,10 @@ func mockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test
 		genms = append(genms, *genm)
 	}
 	templ := &genesis.Template{
-		Accounts:  genaccs,
-		Miners:    genms,
-		Timestamp: uint64(time.Now().Unix()) - (build.BlockDelaySecs * 20000),
+		Accounts:        genaccs,
+		Miners:          genms,
+		Timestamp:       uint64(time.Now().Unix()) - (build.BlockDelaySecs * 20000),
+		VerifregRootKey: gen.DefaultVerifregRootkeyActor,
 	}
 
 	// END PRESEAL SECTION

From d764d86e89826757c1f2779f9d0a6dabd8dbfda0 Mon Sep 17 00:00:00 2001
From: whyrusleeping <why@ipfs.io>
Date: Fri, 24 Jul 2020 11:36:19 -0700
Subject: [PATCH 11/37] pull in specs actors fix for testnet

---
 go.mod | 4 ++--
 go.sum | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/go.mod b/go.mod
index 1e2e5e869..ed7dd6923 100644
--- a/go.mod
+++ b/go.mod
@@ -31,7 +31,7 @@ require (
 	github.com/filecoin-project/go-statestore v0.1.0
 	github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
 	github.com/filecoin-project/sector-storage v0.0.0-20200630180318-4c1968f62a8f
-	github.com/filecoin-project/specs-actors v0.6.2-0.20200702170846-2cd72643a5cf
+	github.com/filecoin-project/specs-actors v0.6.2-0.20200724182911-a473017855a4
 	github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea
 	github.com/filecoin-project/storage-fsm v0.0.0-20200625160832-379a4655b044
 	github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
@@ -102,7 +102,7 @@ require (
 	github.com/multiformats/go-multiaddr-dns v0.2.0
 	github.com/multiformats/go-multiaddr-net v0.1.5
 	github.com/multiformats/go-multibase v0.0.3
-	github.com/multiformats/go-multihash v0.0.13
+	github.com/multiformats/go-multihash v0.0.14
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/stretchr/objx v0.2.0 // indirect
 	github.com/stretchr/testify v1.6.1
diff --git a/go.sum b/go.sum
index 35885247f..d6778be79 100644
--- a/go.sum
+++ b/go.sum
@@ -264,6 +264,8 @@ github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8o
 github.com/filecoin-project/specs-actors v0.6.1/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=
 github.com/filecoin-project/specs-actors v0.6.2-0.20200702170846-2cd72643a5cf h1:2ERozAZteHYef3tVLVJRepzYieLtJdxvfXNUel19CeU=
 github.com/filecoin-project/specs-actors v0.6.2-0.20200702170846-2cd72643a5cf/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=
+github.com/filecoin-project/specs-actors v0.6.2-0.20200724182911-a473017855a4 h1:fo4430nUAUJMx2fVnb8IdWtIxMUJ0tqIOQC71Sk6yWI=
+github.com/filecoin-project/specs-actors v0.6.2-0.20200724182911-a473017855a4/go.mod h1:ppIYDlWQvcfzDW0zxar53Z1Ag69er4BmFvJAtV1uDMI=
 github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sgOVdJbvFjOnD5w94=
 github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k=
 github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea h1:iixjULRQFPn7Q9KlIqfwLJnlAXO10bbkI+xy5GKGdLY=
@@ -1100,6 +1102,7 @@ github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa
 github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
 github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc=
 github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
+github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
 github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
 github.com/multiformats/go-multistream v0.0.4/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
 github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=

From 1163d36b0248c3e369726b62da0ebd5f2832383c Mon Sep 17 00:00:00 2001
From: whyrusleeping <why@ipfs.io>
Date: Fri, 24 Jul 2020 12:33:04 -0700
Subject: [PATCH 12/37] hardcode legacy multihash types for backward compat

---
 go.mod | 4 ++--
 go.sum | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/go.mod b/go.mod
index ed7dd6923..0cbb581ab 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,7 @@ require (
 	github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
 	github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
 	github.com/filecoin-project/go-data-transfer v0.3.0
-	github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5
+	github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f
 	github.com/filecoin-project/go-fil-markets v0.3.2-0.20200702145639-4034a18364e4
 	github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24
 	github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6
@@ -31,7 +31,7 @@ require (
 	github.com/filecoin-project/go-statestore v0.1.0
 	github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
 	github.com/filecoin-project/sector-storage v0.0.0-20200630180318-4c1968f62a8f
-	github.com/filecoin-project/specs-actors v0.6.2-0.20200724182911-a473017855a4
+	github.com/filecoin-project/specs-actors v0.6.2-0.20200724193152-534b25bdca30
 	github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea
 	github.com/filecoin-project/storage-fsm v0.0.0-20200625160832-379a4655b044
 	github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
diff --git a/go.sum b/go.sum
index d6778be79..c5150f7a8 100644
--- a/go.sum
+++ b/go.sum
@@ -235,6 +235,7 @@ github.com/filecoin-project/go-data-transfer v0.3.0 h1:BwBrrXu9Unh9JjjX4GAc5FfzU
 github.com/filecoin-project/go-data-transfer v0.3.0/go.mod h1:cONglGP4s/d+IUQw5mWZrQK+FQATQxr3AXzi4dRh0l4=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1:yvQJCW9mmi9zy+51xA01Ea2X7/dL7r8eKDPuGUjRmbo=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA=
+github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
 github.com/filecoin-project/go-fil-markets v0.3.2-0.20200702145639-4034a18364e4 h1:VqNmKGy4/ryzo/TqevSa1kancc3hSdws7sl/NCTZzT0=
 github.com/filecoin-project/go-fil-markets v0.3.2-0.20200702145639-4034a18364e4/go.mod h1:UY+/zwNXHN73HcrN6HxNDpv6KKM6ehqfCuE9vK9khF8=
 github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms=
@@ -266,6 +267,8 @@ github.com/filecoin-project/specs-actors v0.6.2-0.20200702170846-2cd72643a5cf h1
 github.com/filecoin-project/specs-actors v0.6.2-0.20200702170846-2cd72643a5cf/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=
 github.com/filecoin-project/specs-actors v0.6.2-0.20200724182911-a473017855a4 h1:fo4430nUAUJMx2fVnb8IdWtIxMUJ0tqIOQC71Sk6yWI=
 github.com/filecoin-project/specs-actors v0.6.2-0.20200724182911-a473017855a4/go.mod h1:ppIYDlWQvcfzDW0zxar53Z1Ag69er4BmFvJAtV1uDMI=
+github.com/filecoin-project/specs-actors v0.6.2-0.20200724193152-534b25bdca30 h1:7wH0V1OhS5gkjlJF/PvwS6MuS1oeVqCJlNECgf9eabw=
+github.com/filecoin-project/specs-actors v0.6.2-0.20200724193152-534b25bdca30/go.mod h1:ppIYDlWQvcfzDW0zxar53Z1Ag69er4BmFvJAtV1uDMI=
 github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sgOVdJbvFjOnD5w94=
 github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k=
 github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea h1:iixjULRQFPn7Q9KlIqfwLJnlAXO10bbkI+xy5GKGdLY=

From 3bdcae9a593b6ff0a6e7fa7800d59a4a4638177e Mon Sep 17 00:00:00 2001
From: whyrusleeping <why@ipfs.io>
Date: Fri, 24 Jul 2020 12:41:23 -0700
Subject: [PATCH 13/37] dont update go-fil-commcid

---
 go.mod | 4 +++-
 go.sum | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 0cbb581ab..d3a97ad30 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,9 @@ require (
 	github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
 	github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
 	github.com/filecoin-project/go-data-transfer v0.3.0
-	github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f
+
+	github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5
+
 	github.com/filecoin-project/go-fil-markets v0.3.2-0.20200702145639-4034a18364e4
 	github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24
 	github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6
diff --git a/go.sum b/go.sum
index c5150f7a8..d8e48fb19 100644
--- a/go.sum
+++ b/go.sum
@@ -1105,6 +1105,7 @@ github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa
 github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
 github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc=
 github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
+github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I=
 github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
 github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
 github.com/multiformats/go-multistream v0.0.4/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=

From cda0326cc3beab5457e9746f0c95e6e232a20715 Mon Sep 17 00:00:00 2001
From: whyrusleeping <why@ipfs.io>
Date: Fri, 24 Jul 2020 13:09:19 -0700
Subject: [PATCH 14/37] go mod tidy

---
 go.mod | 2 --
 go.sum | 5 -----
 2 files changed, 7 deletions(-)

diff --git a/go.mod b/go.mod
index d3a97ad30..2c388c4d5 100644
--- a/go.mod
+++ b/go.mod
@@ -23,9 +23,7 @@ require (
 	github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
 	github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
 	github.com/filecoin-project/go-data-transfer v0.3.0
-
 	github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5
-
 	github.com/filecoin-project/go-fil-markets v0.3.2-0.20200702145639-4034a18364e4
 	github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24
 	github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6
diff --git a/go.sum b/go.sum
index d8e48fb19..96cb17a1f 100644
--- a/go.sum
+++ b/go.sum
@@ -235,7 +235,6 @@ github.com/filecoin-project/go-data-transfer v0.3.0 h1:BwBrrXu9Unh9JjjX4GAc5FfzU
 github.com/filecoin-project/go-data-transfer v0.3.0/go.mod h1:cONglGP4s/d+IUQw5mWZrQK+FQATQxr3AXzi4dRh0l4=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1:yvQJCW9mmi9zy+51xA01Ea2X7/dL7r8eKDPuGUjRmbo=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA=
-github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
 github.com/filecoin-project/go-fil-markets v0.3.2-0.20200702145639-4034a18364e4 h1:VqNmKGy4/ryzo/TqevSa1kancc3hSdws7sl/NCTZzT0=
 github.com/filecoin-project/go-fil-markets v0.3.2-0.20200702145639-4034a18364e4/go.mod h1:UY+/zwNXHN73HcrN6HxNDpv6KKM6ehqfCuE9vK9khF8=
 github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms=
@@ -263,10 +262,6 @@ github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.m
 github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y=
 github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=
 github.com/filecoin-project/specs-actors v0.6.1/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=
-github.com/filecoin-project/specs-actors v0.6.2-0.20200702170846-2cd72643a5cf h1:2ERozAZteHYef3tVLVJRepzYieLtJdxvfXNUel19CeU=
-github.com/filecoin-project/specs-actors v0.6.2-0.20200702170846-2cd72643a5cf/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=
-github.com/filecoin-project/specs-actors v0.6.2-0.20200724182911-a473017855a4 h1:fo4430nUAUJMx2fVnb8IdWtIxMUJ0tqIOQC71Sk6yWI=
-github.com/filecoin-project/specs-actors v0.6.2-0.20200724182911-a473017855a4/go.mod h1:ppIYDlWQvcfzDW0zxar53Z1Ag69er4BmFvJAtV1uDMI=
 github.com/filecoin-project/specs-actors v0.6.2-0.20200724193152-534b25bdca30 h1:7wH0V1OhS5gkjlJF/PvwS6MuS1oeVqCJlNECgf9eabw=
 github.com/filecoin-project/specs-actors v0.6.2-0.20200724193152-534b25bdca30/go.mod h1:ppIYDlWQvcfzDW0zxar53Z1Ag69er4BmFvJAtV1uDMI=
 github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sgOVdJbvFjOnD5w94=

From 958cc8b94757105de9150b696f415e11c3db5dfe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Fri, 24 Jul 2020 17:07:31 +0200
Subject: [PATCH 15/37] Update sector-storage, parallel fetch limit

---
 cmd/lotus-seal-worker/main.go   | 9 +++++++--
 cmd/lotus-storage-miner/init.go | 2 +-
 go.mod                          | 2 +-
 go.sum                          | 4 ++--
 node/config/def.go              | 4 ++++
 5 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go
index 82154d328..5cf40dd7e 100644
--- a/cmd/lotus-seal-worker/main.go
+++ b/cmd/lotus-seal-worker/main.go
@@ -95,7 +95,7 @@ var runCmd = &cli.Command{
 		&cli.StringFlag{
 			Name:  "address",
 			Usage: "locally reachable address",
-			Value: "0.0.0.0",
+			Value: "0.0.0.0:3456",
 		},
 		&cli.BoolFlag{
 			Name:  "no-local-storage",
@@ -116,6 +116,11 @@ var runCmd = &cli.Command{
 			Usage: "enable commit (32G sectors: all cores or GPUs, 128GiB Memory + 64GiB swap)",
 			Value: true,
 		},
+		&cli.IntFlag{
+			Name:  "parallel-fetch-limit",
+			Usage: "maximum fetch operations to run in parallel",
+			Value: 5,
+		},
 		&cli.StringFlag{
 			Name:  "timeout",
 			Usage: "used when address is unspecified. must be a valid duration recognized by golang's time.ParseDuration function",
@@ -295,7 +300,7 @@ var runCmd = &cli.Command{
 			return xerrors.Errorf("could not get api info: %w", err)
 		}
 
-		remote := stores.NewRemote(localStore, nodeApi, sminfo.AuthHeader())
+		remote := stores.NewRemote(localStore, nodeApi, sminfo.AuthHeader(), cctx.Int("parallel-fetch-limit"))
 
 		// Create / expose the worker
 
diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go
index 01a14093e..adc537c8b 100644
--- a/cmd/lotus-storage-miner/init.go
+++ b/cmd/lotus-storage-miner/init.go
@@ -436,7 +436,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode,
 
 			smgr, err := sectorstorage.New(ctx, lr, stores.NewIndex(), &ffiwrapper.Config{
 				SealProofType: spt,
-			}, sectorstorage.SealerConfig{true, true, true, true}, nil, sa)
+			}, sectorstorage.SealerConfig{10, true, true, true, true}, nil, sa)
 			if err != nil {
 				return err
 			}
diff --git a/go.mod b/go.mod
index 89a63ec2c..187c6d6b4 100644
--- a/go.mod
+++ b/go.mod
@@ -28,7 +28,7 @@ require (
 	github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261
 	github.com/filecoin-project/go-statestore v0.1.0
 	github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
-	github.com/filecoin-project/sector-storage v0.0.0-20200723200950-ed2e57dde6df
+	github.com/filecoin-project/sector-storage v0.0.0-20200724175351-295300ff4671
 	github.com/filecoin-project/specs-actors v0.8.1-0.20200724015154-3c690d9b7e1d
 	github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea
 	github.com/filecoin-project/storage-fsm v0.0.0-20200720190000-2cfe2fe3c334
diff --git a/go.sum b/go.sum
index 5bafb5f91..c065177ef 100644
--- a/go.sum
+++ b/go.sum
@@ -264,8 +264,8 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/
 github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM=
 github.com/filecoin-project/sector-storage v0.0.0-20200712023225-1d67dcfa3c15 h1:miw6hiusb/MkV1ryoqUKKWnvHhPW00AYtyeCj0L8pqo=
 github.com/filecoin-project/sector-storage v0.0.0-20200712023225-1d67dcfa3c15/go.mod h1:salgVdX7qeXFo/xaiEQE29J4pPkjn71T0kt0n+VDBzo=
-github.com/filecoin-project/sector-storage v0.0.0-20200723200950-ed2e57dde6df h1:VDdWrCNUNx6qeHnGU9oAy+izuGM02it9V/5+MJyhZQw=
-github.com/filecoin-project/sector-storage v0.0.0-20200723200950-ed2e57dde6df/go.mod h1:7EE+f7jM4kCy2MKHoiiwNDQGJSb+QQzZ+y+/17ugq4w=
+github.com/filecoin-project/sector-storage v0.0.0-20200724175351-295300ff4671 h1:9TqtFwy+6/Ywh3j81vFy3jQerD5DkxFY2Ul8nJPUHwg=
+github.com/filecoin-project/sector-storage v0.0.0-20200724175351-295300ff4671/go.mod h1:f9W29dKqNFm8Su4OddGwkAQOYMKYUR5Fk2oC/JZDjCI=
 github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA=
 github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y=
 github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=
diff --git a/node/config/def.go b/node/config/def.go
index a9caf1377..81ffdcd9d 100644
--- a/node/config/def.go
+++ b/node/config/def.go
@@ -125,6 +125,10 @@ func DefaultStorageMiner() *StorageMiner {
 			AllowPreCommit2: true,
 			AllowCommit:     true,
 			AllowUnseal:     true,
+
+			// Default to 10 - tcp should still be able to figure this out, and
+			// it's the ratio between 10gbit / 1gbit
+			ParallelFetchLimit: 10,
 		},
 
 		Dealmaking: DealmakingConfig{

From dd885b7302351cefeb52ca781725155a4dd8d51a Mon Sep 17 00:00:00 2001
From: hannahhoward <hannah@hannahhoward.net>
Date: Fri, 24 Jul 2020 14:47:22 -0700
Subject: [PATCH 16/37] feat(multistore): extract multistore code to repo

Extract multistore + multiread blockstore to a seperate repo
---
 go.mod                            |   1 +
 go.sum                            |   2 +
 node/impl/client/client.go        |   3 +-
 node/modules/client.go            |   3 +-
 node/modules/dtypes/storage.go    |   3 +-
 node/repo/importmgr/mbstore.go    |  95 ---------------
 node/repo/importmgr/mgr.go        |  14 +--
 node/repo/importmgr/multistore.go | 188 ------------------------------
 node/repo/importmgr/store.go      |  54 ---------
 9 files changed, 16 insertions(+), 347 deletions(-)
 delete mode 100644 node/repo/importmgr/mbstore.go
 delete mode 100644 node/repo/importmgr/multistore.go
 delete mode 100644 node/repo/importmgr/store.go

diff --git a/go.mod b/go.mod
index 89a63ec2c..b15ddb81e 100644
--- a/go.mod
+++ b/go.mod
@@ -25,6 +25,7 @@ require (
 	github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f
 	github.com/filecoin-project/go-fil-markets v0.5.1
 	github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24
+	github.com/filecoin-project/go-multistore v0.0.1
 	github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261
 	github.com/filecoin-project/go-statestore v0.1.0
 	github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
diff --git a/go.sum b/go.sum
index 5bafb5f91..52de2ca8d 100644
--- a/go.sum
+++ b/go.sum
@@ -248,6 +248,8 @@ github.com/filecoin-project/go-fil-markets v0.5.1 h1:Y69glslNCuXnygfesCmyilTVhEE
 github.com/filecoin-project/go-fil-markets v0.5.1/go.mod h1:GKGigsFNMvKmx/+Mcn7093TdZTiCDLc7YGxQ7d6fq2s=
 github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms=
 github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM=
+github.com/filecoin-project/go-multistore v0.0.1 h1:wXCd02azCxEcMNlDE9lksraQO+iIjFGNw01IZyf8GPA=
+github.com/filecoin-project/go-multistore v0.0.1/go.mod h1:z8NeSPWubEvrzi0XolhZ1NjTeW9ZDR779M+EDhf4QIQ=
 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs=
 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6/go.mod h1:0HgYnrkeSU4lu1p+LEOeDpFsNBssa0OGGriWdA4hvaE=
 github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
diff --git a/node/impl/client/client.go b/node/impl/client/client.go
index 53dacb8de..1e7fa8146 100644
--- a/node/impl/client/client.go
+++ b/node/impl/client/client.go
@@ -33,6 +33,7 @@ import (
 	rm "github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/shared"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/filecoin-project/go-multistore"
 	"github.com/filecoin-project/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-actors/actors/abi/big"
@@ -556,7 +557,7 @@ func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath stri
 	return f.Close()
 }
 
-func (a *API) clientImport(ctx context.Context, ref api.FileRef, store *importmgr.Store) (cid.Cid, error) {
+func (a *API) clientImport(ctx context.Context, ref api.FileRef, store *multistore.Store) (cid.Cid, error) {
 	f, err := os.Open(ref.Path)
 	if err != nil {
 		return cid.Undef, err
diff --git a/node/modules/client.go b/node/modules/client.go
index a24709beb..77e66a69a 100644
--- a/node/modules/client.go
+++ b/node/modules/client.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"time"
 
+	"github.com/filecoin-project/go-multistore"
 	"github.com/filecoin-project/lotus/lib/bufbstore"
 	"golang.org/x/xerrors"
 
@@ -42,7 +43,7 @@ func ClientMultiDatastore(lc fx.Lifecycle, r repo.LockedRepo) (dtypes.ClientMult
 		return nil, xerrors.Errorf("getting datastore out of reop: %w", err)
 	}
 
-	mds, err := importmgr.NewMultiDstore(ds)
+	mds, err := multistore.NewMultiDstore(ds)
 	if err != nil {
 		return nil, err
 	}
diff --git a/node/modules/dtypes/storage.go b/node/modules/dtypes/storage.go
index 7ede8aaab..b4712a37e 100644
--- a/node/modules/dtypes/storage.go
+++ b/node/modules/dtypes/storage.go
@@ -9,6 +9,7 @@ import (
 	format "github.com/ipfs/go-ipld-format"
 
 	"github.com/filecoin-project/go-fil-markets/storagemarket/impl/requestvalidation"
+	"github.com/filecoin-project/go-multistore"
 
 	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/filecoin-project/go-fil-markets/piecestore"
@@ -27,7 +28,7 @@ type ChainGCBlockstore blockstore.GCBlockstore
 type ChainExchange exchange.Interface
 type ChainBlockService bserv.BlockService
 
-type ClientMultiDstore *importmgr.MultiStore
+type ClientMultiDstore *multistore.MultiStore
 type ClientImportMgr *importmgr.Mgr
 type ClientBlockstore blockstore.Blockstore
 type ClientDealStore *statestore.StateStore
diff --git a/node/repo/importmgr/mbstore.go b/node/repo/importmgr/mbstore.go
deleted file mode 100644
index 3b6058bee..000000000
--- a/node/repo/importmgr/mbstore.go
+++ /dev/null
@@ -1,95 +0,0 @@
-package importmgr
-
-import (
-	"context"
-
-	"github.com/hashicorp/go-multierror"
-	"golang.org/x/xerrors"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
-)
-
-type multiReadBs struct {
-	// TODO: some caching
-	mds *MultiStore
-}
-
-func (m *multiReadBs) Has(cid cid.Cid) (bool, error) {
-	m.mds.lk.RLock()
-	defer m.mds.lk.RUnlock()
-
-	var merr error
-	for i, store := range m.mds.open {
-		has, err := store.Bstore.Has(cid)
-		if err != nil {
-			merr = multierror.Append(merr, xerrors.Errorf("has (ds %d): %w", i, err))
-			continue
-		}
-		if !has {
-			continue
-		}
-
-		return true, nil
-	}
-
-	return false, merr
-}
-
-func (m *multiReadBs) Get(cid cid.Cid) (blocks.Block, error) {
-	m.mds.lk.RLock()
-	defer m.mds.lk.RUnlock()
-
-	var merr error
-	for i, store := range m.mds.open {
-		has, err := store.Bstore.Has(cid)
-		if err != nil {
-			merr = multierror.Append(merr, xerrors.Errorf("has (ds %d): %w", i, err))
-			continue
-		}
-		if !has {
-			continue
-		}
-
-		val, err := store.Bstore.Get(cid)
-		if err != nil {
-			merr = multierror.Append(merr, xerrors.Errorf("get (ds %d): %w", i, err))
-			continue
-		}
-
-		return val, nil
-	}
-
-	if merr == nil {
-		return nil, blockstore.ErrNotFound
-	}
-
-	return nil, merr
-}
-
-func (m *multiReadBs) DeleteBlock(cid cid.Cid) error {
-	return xerrors.Errorf("operation not supported")
-}
-
-func (m *multiReadBs) GetSize(cid cid.Cid) (int, error) {
-	return 0, xerrors.Errorf("operation not supported")
-}
-
-func (m *multiReadBs) Put(block blocks.Block) error {
-	return xerrors.Errorf("operation not supported")
-}
-
-func (m *multiReadBs) PutMany(blocks []blocks.Block) error {
-	return xerrors.Errorf("operation not supported")
-}
-
-func (m *multiReadBs) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return nil, xerrors.Errorf("operation not supported")
-}
-
-func (m *multiReadBs) HashOnRead(enabled bool) {
-	return
-}
-
-var _ blockstore.Blockstore = &multiReadBs{}
diff --git a/node/repo/importmgr/mgr.go b/node/repo/importmgr/mgr.go
index d3139918e..754f41df2 100644
--- a/node/repo/importmgr/mgr.go
+++ b/node/repo/importmgr/mgr.go
@@ -9,10 +9,12 @@ import (
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
+
+	"github.com/filecoin-project/go-multistore"
 )
 
 type Mgr struct {
-	mds *MultiStore
+	mds *multistore.MultiStore
 	ds  datastore.Batching
 
 	Blockstore blockstore.Blockstore
@@ -27,12 +29,10 @@ const (
 	LMTime    = "mtime"    // File modification timestamp
 )
 
-func New(mds *MultiStore, ds datastore.Batching) *Mgr {
+func New(mds *multistore.MultiStore, ds datastore.Batching) *Mgr {
 	return &Mgr{
-		mds: mds,
-		Blockstore: &multiReadBs{
-			mds: mds,
-		},
+		mds:        mds,
+		Blockstore: mds.MultiReadBlockstore(),
 
 		ds: datastore.NewLogDatastore(namespace.Wrap(ds, datastore.NewKey("/stores")), "storess"),
 	}
@@ -42,7 +42,7 @@ type StoreMeta struct {
 	Labels map[string]string
 }
 
-func (m *Mgr) NewStore() (int, *Store, error) {
+func (m *Mgr) NewStore() (int, *multistore.Store, error) {
 	id := m.mds.Next()
 	st, err := m.mds.Get(id)
 	if err != nil {
diff --git a/node/repo/importmgr/multistore.go b/node/repo/importmgr/multistore.go
deleted file mode 100644
index e7a814d4c..000000000
--- a/node/repo/importmgr/multistore.go
+++ /dev/null
@@ -1,188 +0,0 @@
-package importmgr
-
-import (
-	"encoding/json"
-	"fmt"
-	"sort"
-	"sync"
-
-	"go.uber.org/multierr"
-	"golang.org/x/xerrors"
-
-	"github.com/ipfs/go-datastore"
-	ktds "github.com/ipfs/go-datastore/keytransform"
-	"github.com/ipfs/go-datastore/query"
-)
-
-type MultiStore struct {
-	ds datastore.Batching
-
-	open map[int]*Store
-	next int
-
-	lk sync.RWMutex
-}
-
-var dsListKey = datastore.NewKey("/list")
-var dsMultiKey = datastore.NewKey("/multi")
-
-func NewMultiDstore(ds datastore.Batching) (*MultiStore, error) {
-	listBytes, err := ds.Get(dsListKey)
-	if xerrors.Is(err, datastore.ErrNotFound) {
-		listBytes, _ = json.Marshal([]int{})
-	} else if err != nil {
-		return nil, xerrors.Errorf("could not read multistore list: %w", err)
-	}
-
-	var ids []int
-	if err := json.Unmarshal(listBytes, &ids); err != nil {
-		return nil, xerrors.Errorf("could not unmarshal multistore list: %w", err)
-	}
-
-	mds := &MultiStore{
-		ds:   ds,
-		open: map[int]*Store{},
-	}
-
-	for _, i := range ids {
-		if i > mds.next {
-			mds.next = i
-		}
-
-		_, err := mds.Get(i)
-		if err != nil {
-			return nil, xerrors.Errorf("open store %d: %w", i, err)
-		}
-	}
-
-	return mds, nil
-}
-
-func (mds *MultiStore) Next() int {
-	mds.lk.Lock()
-	defer mds.lk.Unlock()
-
-	mds.next++
-	return mds.next
-}
-
-func (mds *MultiStore) updateStores() error {
-	stores := make([]int, 0, len(mds.open))
-	for k := range mds.open {
-		stores = append(stores, k)
-	}
-	sort.Ints(stores)
-
-	listBytes, err := json.Marshal(stores)
-	if err != nil {
-		return xerrors.Errorf("could not marshal list: %w", err)
-	}
-	err = mds.ds.Put(dsListKey, listBytes)
-	if err != nil {
-		return xerrors.Errorf("could not save stores list: %w", err)
-	}
-	return nil
-}
-
-func (mds *MultiStore) Get(i int) (*Store, error) {
-	mds.lk.Lock()
-	defer mds.lk.Unlock()
-
-	store, ok := mds.open[i]
-	if ok {
-		return store, nil
-	}
-
-	wds := ktds.Wrap(mds.ds, ktds.PrefixTransform{
-		Prefix: dsMultiKey.ChildString(fmt.Sprintf("%d", i)),
-	})
-
-	var err error
-	mds.open[i], err = openStore(wds)
-	if err != nil {
-		return nil, xerrors.Errorf("could not open new store: %w", err)
-	}
-
-	err = mds.updateStores()
-	if err != nil {
-		return nil, xerrors.Errorf("updating stores: %w", err)
-	}
-
-	return mds.open[i], nil
-}
-
-func (mds *MultiStore) List() []int {
-	mds.lk.RLock()
-	defer mds.lk.RUnlock()
-
-	out := make([]int, 0, len(mds.open))
-	for i := range mds.open {
-		out = append(out, i)
-	}
-	sort.Ints(out)
-
-	return out
-}
-
-func (mds *MultiStore) Delete(i int) error {
-	mds.lk.Lock()
-	defer mds.lk.Unlock()
-
-	store, ok := mds.open[i]
-	if !ok {
-		return nil
-	}
-	delete(mds.open, i)
-	err := store.Close()
-	if err != nil {
-		return xerrors.Errorf("closing store: %w", err)
-	}
-
-	err = mds.updateStores()
-	if err != nil {
-		return xerrors.Errorf("updating stores: %w", err)
-	}
-
-	qres, err := store.ds.Query(query.Query{KeysOnly: true})
-	if err != nil {
-		return xerrors.Errorf("query error: %w", err)
-	}
-	defer qres.Close() //nolint:errcheck
-
-	b, err := store.ds.Batch()
-	if err != nil {
-		return xerrors.Errorf("batch error: %w", err)
-	}
-
-	for r := range qres.Next() {
-		if r.Error != nil {
-			_ = b.Commit()
-			return xerrors.Errorf("iterator error: %w", err)
-		}
-		err := b.Delete(datastore.NewKey(r.Key))
-		if err != nil {
-			_ = b.Commit()
-			return xerrors.Errorf("adding to batch: %w", err)
-		}
-	}
-
-	err = b.Commit()
-	if err != nil {
-		return xerrors.Errorf("committing: %w", err)
-	}
-
-	return nil
-}
-
-func (mds *MultiStore) Close() error {
-	mds.lk.Lock()
-	defer mds.lk.Unlock()
-
-	var err error
-	for _, s := range mds.open {
-		err = multierr.Append(err, s.Close())
-	}
-	mds.open = make(map[int]*Store)
-
-	return err
-}
diff --git a/node/repo/importmgr/store.go b/node/repo/importmgr/store.go
deleted file mode 100644
index 78bb7462b..000000000
--- a/node/repo/importmgr/store.go
+++ /dev/null
@@ -1,54 +0,0 @@
-package importmgr
-
-import (
-	"github.com/ipfs/go-blockservice"
-	"github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-datastore/namespace"
-	"github.com/ipfs/go-filestore"
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
-	offline "github.com/ipfs/go-ipfs-exchange-offline"
-	ipld "github.com/ipfs/go-ipld-format"
-	"github.com/ipfs/go-merkledag"
-)
-
-type Store struct {
-	ds datastore.Batching
-
-	fm     *filestore.FileManager
-	Fstore *filestore.Filestore
-
-	Bstore blockstore.Blockstore
-
-	bsvc blockservice.BlockService
-	DAG  ipld.DAGService
-}
-
-func openStore(ds datastore.Batching) (*Store, error) {
-	blocks := namespace.Wrap(ds, datastore.NewKey("blocks"))
-	bs := blockstore.NewBlockstore(blocks)
-
-	fm := filestore.NewFileManager(ds, "/")
-	fm.AllowFiles = true
-
-	fstore := filestore.NewFilestore(bs, fm)
-	ibs := blockstore.NewIdStore(fstore)
-
-	bsvc := blockservice.New(ibs, offline.Exchange(ibs))
-	dag := merkledag.NewDAGService(bsvc)
-
-	return &Store{
-		ds: ds,
-
-		fm:     fm,
-		Fstore: fstore,
-
-		Bstore: ibs,
-
-		bsvc: bsvc,
-		DAG:  dag,
-	}, nil
-}
-
-func (s *Store) Close() error {
-	return s.bsvc.Close()
-}

From 718dd26043eb3a4d4ebef20157d3b0ec5c8d41c9 Mon Sep 17 00:00:00 2001
From: Mike Greenberg <mg+github@protocol.ai>
Date: Fri, 24 Jul 2020 16:11:16 -0400
Subject: [PATCH 17/37] fix(chainwatch): Move scheduler-specific schema; Add
 max_height view

---
 cmd/lotus-chainwatch/processor/reward.go      | 19 -------
 .../refresh_top_miners_by_base_reward.go      | 57 ++++++++++++++++++-
 cmd/lotus-chainwatch/scheduler/scheduler.go   | 13 +++++
 3 files changed, 69 insertions(+), 20 deletions(-)

diff --git a/cmd/lotus-chainwatch/processor/reward.go b/cmd/lotus-chainwatch/processor/reward.go
index fb628bd6f..af339b74d 100644
--- a/cmd/lotus-chainwatch/processor/reward.go
+++ b/cmd/lotus-chainwatch/processor/reward.go
@@ -51,30 +51,11 @@ create table if not exists chain_power
 			primary key,
 	baseline_power text not null
 );
-
-create materialized view if not exists top_miners_by_base_reward as
-	with total_rewards_by_miner as (
-		select
-			b.miner,
-			sum(bbr.base_block_reward) as total_reward
-		from blocks b
-		inner join base_block_rewards bbr on b.parentstateroot = bbr.state_root
-		group by 1
-	) select
-		rank() over (order by total_reward desc),
-		miner,
-		total_reward
-	from total_rewards_by_miner
-	group by 2, 3;
-
-create index if not exists top_miners_by_base_reward_miner_index
-	on top_miners_by_base_reward (miner);
 `); err != nil {
 		return err
 	}
 
 	return tx.Commit()
-
 }
 
 func (p *Processor) HandleRewardChanges(ctx context.Context, rewardTips ActorTips) error {
diff --git a/cmd/lotus-chainwatch/scheduler/refresh_top_miners_by_base_reward.go b/cmd/lotus-chainwatch/scheduler/refresh_top_miners_by_base_reward.go
index 50bb561a1..28e164f34 100644
--- a/cmd/lotus-chainwatch/scheduler/refresh_top_miners_by_base_reward.go
+++ b/cmd/lotus-chainwatch/scheduler/refresh_top_miners_by_base_reward.go
@@ -8,6 +8,56 @@ import (
 	"golang.org/x/xerrors"
 )
 
+func setupTopMinerByBaseRewardSchema(ctx context.Context, db *sql.DB) error {
+	select {
+	case <-ctx.Done():
+		return nil
+	default:
+	}
+
+	tx, err := db.Begin()
+	if err != nil {
+		return err
+	}
+	if _, err := tx.Exec(`
+		create materialized view if not exists top_miners_by_base_reward as
+			with total_rewards_by_miner as (
+				select
+					b.miner,
+					sum(bbr.base_block_reward) as total_reward
+				from blocks b
+				inner join base_block_rewards bbr on b.parentstateroot = bbr.state_root
+				group by 1
+			) select
+				rank() over (order by total_reward desc),
+				miner,
+				total_reward
+			from total_rewards_by_miner
+			group by 2, 3;
+
+		create index if not exists top_miners_by_base_reward_miner_index
+			on top_miners_by_base_reward (miner);
+
+		create materialized view if not exists top_miners_by_base_reward_max_height as
+			select
+				b."timestamp"as current_timestamp,
+				max(b.height) as current_height
+			from blocks b
+			join base_block_rewards bbr on b.parentstateroot = bbr.state_root
+			where bbr.base_block_reward is not null
+			group by 1
+			order by 1 desc
+			limit 1;
+	`); err != nil {
+		return xerrors.Errorf("create top_miner_by_base_reward views", err)
+	}
+
+	if err := tx.Commit(); err != nil {
+		return xerrors.Errorf("commiting top_miner_by_base_reward views", err)
+	}
+	return nil
+}
+
 func refreshTopMinerByBaseReward(ctx context.Context, db *sql.DB) error {
 	select {
 	case <-ctx.Done():
@@ -20,10 +70,15 @@ func refreshTopMinerByBaseReward(ctx context.Context, db *sql.DB) error {
 		log.Debugw("refresh top_miners_by_base_reward", "duration", time.Since(t).String())
 	}()
 
-	_, err := db.Exec("REFRESH MATERIALIZED VIEW top_miners_by_base_reward;")
+	_, err := db.Exec("refresh materialized view top_miners_by_base_reward;")
 	if err != nil {
 		return xerrors.Errorf("refresh top_miners_by_base_reward: %w", err)
 	}
 
+	_, err = db.Exec("refresh materialized view top_miners_by_base_reward_max_height;")
+	if err != nil {
+		return xerrors.Errorf("refresh top_miners_by_base_reward_max_height: %w", err)
+	}
+
 	return nil
 }
diff --git a/cmd/lotus-chainwatch/scheduler/scheduler.go b/cmd/lotus-chainwatch/scheduler/scheduler.go
index c5c93c310..54cc30929 100644
--- a/cmd/lotus-chainwatch/scheduler/scheduler.go
+++ b/cmd/lotus-chainwatch/scheduler/scheduler.go
@@ -6,6 +6,8 @@ import (
 	"time"
 
 	logging "github.com/ipfs/go-log/v2"
+
+	"golang.org/x/xerrors"
 )
 
 var log = logging.Logger("scheduler")
@@ -21,10 +23,21 @@ func PrepareScheduler(db *sql.DB) *Scheduler {
 	return &Scheduler{db}
 }
 
+func (s *Scheduler) setupSchema(ctx context.Context) error {
+	if err := setupTopMinerByBaseRewardSchema(ctx, s.db); err != nil {
+		return xerrors.Errorf("setup top miners by reward schema", err)
+	}
+	return nil
+}
+
 // Start the scheduler jobs at the defined intervals
 func (s *Scheduler) Start(ctx context.Context) {
 	log.Debug("Starting Scheduler")
 
+	if err := s.setupSchema(ctx); err != nil {
+		log.Fatalw("applying scheduling schema", err)
+	}
+
 	go func() {
 		// run once on start after schema has initialized
 		time.Sleep(5 * time.Second)

From c6e0a69bbc92c19d6825444ba1f8ab5c1040acf0 Mon Sep 17 00:00:00 2001
From: Mike Greenberg <mg+github@protocol.ai>
Date: Fri, 24 Jul 2020 16:19:22 -0400
Subject: [PATCH 18/37] fix(chainwatch): Consider win_count in
 top_miner_by_base_reward view

---
 .../scheduler/refresh_top_miners_by_base_reward.go              | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/lotus-chainwatch/scheduler/refresh_top_miners_by_base_reward.go b/cmd/lotus-chainwatch/scheduler/refresh_top_miners_by_base_reward.go
index 28e164f34..550f7a2bd 100644
--- a/cmd/lotus-chainwatch/scheduler/refresh_top_miners_by_base_reward.go
+++ b/cmd/lotus-chainwatch/scheduler/refresh_top_miners_by_base_reward.go
@@ -24,7 +24,7 @@ func setupTopMinerByBaseRewardSchema(ctx context.Context, db *sql.DB) error {
 			with total_rewards_by_miner as (
 				select
 					b.miner,
-					sum(bbr.base_block_reward) as total_reward
+					sum(bbr.base_block_reward * b.win_count) as total_reward
 				from blocks b
 				inner join base_block_rewards bbr on b.parentstateroot = bbr.state_root
 				group by 1

From 6cf2fe50dc0cae18cb686da1c38c48647a232cf6 Mon Sep 17 00:00:00 2001
From: Aayush Rajasekaran <arajasek94@gmail.com>
Date: Fri, 24 Jul 2020 18:21:34 -0400
Subject: [PATCH 19/37] Add CLI command to calculate pledge collateral of a CC
 sector

---
 cmd/lotus-storage-miner/sectors.go | 49 ++++++++++++++++++++++++++++++
 node/impl/full/state.go            | 12 ++++++--
 2 files changed, 58 insertions(+), 3 deletions(-)

diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go
index 5166d1c3d..c1a5a663a 100644
--- a/cmd/lotus-storage-miner/sectors.go
+++ b/cmd/lotus-storage-miner/sectors.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"os"
 	"sort"
 	"strconv"
@@ -31,6 +32,7 @@ var sectorsCmd = &cli.Command{
 		sectorsMarkForUpgradeCmd,
 		sectorsStartSealCmd,
 		sectorsSealDelayCmd,
+		sectorsCapacityCollateralCmd,
 	},
 }
 
@@ -321,6 +323,53 @@ var sectorsSealDelayCmd = &cli.Command{
 	},
 }
 
+var sectorsCapacityCollateralCmd = &cli.Command{
+	Name:  "get-cc-collateral",
+	Usage: "Get the collateral required to pledge a committed capacity sector",
+	Flags: []cli.Flag{
+		&cli.Uint64Flag{
+			Name:  "expiration",
+			Usage: "the epoch when the sector will expire",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+
+		mApi, mCloser, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer mCloser()
+
+		nApi, nCloser, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer nCloser()
+
+		ctx := lcli.ReqContext(cctx)
+
+		maddr, err := mApi.ActorAddress(ctx)
+		if err != nil {
+			return err
+		}
+
+		pci := miner.SectorPreCommitInfo{
+			Expiration: abi.ChainEpoch(cctx.Uint64("expiration")),
+		}
+		if pci.Expiration == 0 {
+			pci.Expiration = miner.MaxSectorExpirationExtension
+		}
+		pc, err := nApi.StateMinerInitialPledgeCollateral(ctx, maddr, pci, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Estimated collateral: %s\n", types.FIL(pc))
+
+		return nil
+	},
+}
+
 var sectorsUpdateCmd = &cli.Command{
 	Name:  "update-state",
 	Usage: "ADVANCED: manually update the state of a sector, this may aid in error recovery",
diff --git a/node/impl/full/state.go b/node/impl/full/state.go
index 917bfd841..0333082b8 100644
--- a/node/impl/full/state.go
+++ b/node/impl/full/state.go
@@ -952,8 +952,12 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr
 		return types.EmptyInt, err
 	}
 
-	var dealWeights market.VerifyDealsForActivationReturn
-	{
+	dealWeights := market.VerifyDealsForActivationReturn{
+		DealWeight:         big.Zero(),
+		VerifiedDealWeight: big.Zero(),
+	}
+
+	if len(pci.DealIDs) != 0 {
 		var err error
 		params, err := actors.SerializeParams(&market.VerifyDealsForActivationParams{
 			DealIDs:      pci.DealIDs,
@@ -980,11 +984,13 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr
 		}
 	}
 
-	ssize, err := pci.SealProof.SectorSize()
+	mi, err := a.StateMinerInfo(ctx, maddr, tsk)
 	if err != nil {
 		return types.EmptyInt, err
 	}
 
+	ssize := mi.SectorSize
+
 	duration := pci.Expiration - ts.Height() // NB: not exactly accurate, but should always lead us to *over* estimate, not under
 
 	circSupply, err := a.StateManager.CirculatingSupply(ctx, ts)

From b409c8f2beca7b19e29c8854a0f7286e8d730c2a Mon Sep 17 00:00:00 2001
From: Mike Goelzer <mike@protocol.ai>
Date: Sat, 25 Jul 2020 01:54:21 +0000
Subject: [PATCH 20/37] Parse amount as FIL in paych get

---
 cli/paych.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/cli/paych.go b/cli/paych.go
index 070645152..7a4462e11 100644
--- a/cli/paych.go
+++ b/cli/paych.go
@@ -29,7 +29,7 @@ var paychGetCmd = &cli.Command{
 	ArgsUsage: "[fromAddress toAddress amount]",
 	Action: func(cctx *cli.Context) error {
 		if cctx.Args().Len() != 3 {
-			return fmt.Errorf("must pass three arguments: <from> <to> <available funds>")
+			return fmt.Errorf("must pass three arguments: <from> <to> <available funds in FIL>")
 		}
 
 		from, err := address.NewFromString(cctx.Args().Get(0))
@@ -42,9 +42,9 @@ var paychGetCmd = &cli.Command{
 			return fmt.Errorf("failed to parse to address: %s", err)
 		}
 
-		amt, err := types.BigFromString(cctx.Args().Get(2))
+		amt, err := types.ParseFIL(cctx.Args().Get(2))
 		if err != nil {
-			return fmt.Errorf("parsing amount failed: %s", err)
+			return fmt.Errorf("parsing amount as whole FIL failed: %s", err)
 		}
 
 		api, closer, err := GetFullNodeAPI(cctx)
@@ -55,7 +55,7 @@ var paychGetCmd = &cli.Command{
 
 		ctx := ReqContext(cctx)
 
-		info, err := api.PaychGet(ctx, from, to, amt)
+		info, err := api.PaychGet(ctx, from, to, types.BigInt(amt))
 		if err != nil {
 			return err
 		}

From de1574d21f2e1fdec427d524d4663bd81ec3cb6f Mon Sep 17 00:00:00 2001
From: ognots <ognots@protonmail.com>
Date: Fri, 24 Jul 2020 22:43:24 -0400
Subject: [PATCH 21/37] add port to default address flag

port is required and things will break otherwise
---
 cmd/lotus-seal-worker/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go
index 82154d328..c2b1bd6f5 100644
--- a/cmd/lotus-seal-worker/main.go
+++ b/cmd/lotus-seal-worker/main.go
@@ -95,7 +95,7 @@ var runCmd = &cli.Command{
 		&cli.StringFlag{
 			Name:  "address",
 			Usage: "locally reachable address",
-			Value: "0.0.0.0",
+			Value: "0.0.0.0:3456",
 		},
 		&cli.BoolFlag{
 			Name:  "no-local-storage",

From df33a16f78589f45a29d144582370b0d1d0560cd Mon Sep 17 00:00:00 2001
From: Aayush Rajasekaran <arajasek94@gmail.com>
Date: Sat, 25 Jul 2020 12:59:12 -0400
Subject: [PATCH 22/37] CLI: Optionally include vesting details in msig inspect

---
 cli/multisig.go | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/cli/multisig.go b/cli/multisig.go
index 983bf55c1..e2ff4e5a7 100644
--- a/cli/multisig.go
+++ b/cli/multisig.go
@@ -156,7 +156,12 @@ var msigInspectCmd = &cli.Command{
 	Name:      "inspect",
 	Usage:     "Inspect a multisig wallet",
 	ArgsUsage: "[address]",
-	Flags:     []cli.Flag{},
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "vesting",
+			Usage: "Include vesting details",
+		},
+	},
 	Action: func(cctx *cli.Context) error {
 		if !cctx.Args().Present() {
 			return ShowHelp(cctx, fmt.Errorf("must specify address of multisig to inspect"))
@@ -197,6 +202,13 @@ var msigInspectCmd = &cli.Command{
 		locked := mstate.AmountLocked(head.Height() - mstate.StartEpoch)
 		fmt.Printf("Balance: %s\n", types.FIL(act.Balance))
 		fmt.Printf("Spendable: %s\n", types.FIL(types.BigSub(act.Balance, locked)))
+
+		if cctx.Bool("vesting") {
+			fmt.Printf("InitialBalance: %s\n", types.FIL(mstate.InitialBalance))
+			fmt.Printf("StartEpoch: %d\n", mstate.StartEpoch)
+			fmt.Printf("UnlockDuration: %d\n", mstate.UnlockDuration)
+		}
+
 		fmt.Printf("Threshold: %d / %d\n", mstate.NumApprovalsThreshold, len(mstate.Signers))
 		fmt.Println("Signers:")
 		for _, s := range mstate.Signers {

From 135b77dab5ffa4f2fe8cce90092faf6819859db5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 27 Jul 2020 13:23:43 +0200
Subject: [PATCH 23/37] Update sector-storage, sealing sched-diag

---
 api/api_storage.go                 |  3 +++
 api/apistruct/struct.go            |  6 ++++++
 cmd/lotus-storage-miner/sealing.go | 30 ++++++++++++++++++++++++++++++
 go.mod                             |  2 +-
 go.sum                             |  4 ++--
 node/impl/storminer.go             |  4 ++++
 6 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/api/api_storage.go b/api/api_storage.go
index 7e6d81b21..3853a8e0f 100644
--- a/api/api_storage.go
+++ b/api/api_storage.go
@@ -63,6 +63,9 @@ type StorageMiner interface {
 	WorkerStats(context.Context) (map[uint64]storiface.WorkerStats, error)
 	WorkerJobs(context.Context) (map[uint64][]storiface.WorkerJob, error)
 
+	// SealingSchedDiag dumps internal sealing scheduler state
+	SealingSchedDiag(context.Context) (interface{}, error)
+
 	stores.SectorIndex
 
 	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error
diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go
index 6b41fa9dd..a7a392610 100644
--- a/api/apistruct/struct.go
+++ b/api/apistruct/struct.go
@@ -231,6 +231,8 @@ type StorageMinerStruct struct {
 		WorkerStats   func(context.Context) (map[uint64]storiface.WorkerStats, error) `perm:"admin"`
 		WorkerJobs    func(context.Context) (map[uint64][]storiface.WorkerJob, error) `perm:"admin"`
 
+		SealingSchedDiag func(context.Context) (interface{}, error) `perm:"admin"`
+
 		StorageList          func(context.Context) (map[stores.ID][]stores.Decl, error)                                                                                    `perm:"admin"`
 		StorageLocal         func(context.Context) (map[stores.ID]string, error)                                                                                           `perm:"admin"`
 		StorageStat          func(context.Context, stores.ID) (fsutil.FsStat, error)                                                                                       `perm:"admin"`
@@ -902,6 +904,10 @@ func (c *StorageMinerStruct) WorkerJobs(ctx context.Context) (map[uint64][]stori
 	return c.Internal.WorkerJobs(ctx)
 }
 
+func (c *StorageMinerStruct) SealingSchedDiag(ctx context.Context) (interface{}, error) {
+	return c.Internal.SealingSchedDiag(ctx)
+}
+
 func (c *StorageMinerStruct) StorageAttach(ctx context.Context, si stores.StorageInfo, st fsutil.FsStat) error {
 	return c.Internal.StorageAttach(ctx, si, st)
 }
diff --git a/cmd/lotus-storage-miner/sealing.go b/cmd/lotus-storage-miner/sealing.go
index 68b0a246f..25a1ae890 100644
--- a/cmd/lotus-storage-miner/sealing.go
+++ b/cmd/lotus-storage-miner/sealing.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"golang.org/x/xerrors"
 	"os"
@@ -24,6 +25,7 @@ var sealingCmd = &cli.Command{
 	Subcommands: []*cli.Command{
 		sealingJobsCmd,
 		sealingWorkersCmd,
+		sealingSchedDiagCmd,
 	},
 }
 
@@ -176,3 +178,31 @@ var sealingJobsCmd = &cli.Command{
 		return tw.Flush()
 	},
 }
+
+var sealingSchedDiagCmd = &cli.Command{
+	Name:  "sched-diag",
+	Usage: "Dump internal scheduler state",
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		st, err := nodeApi.SealingSchedDiag(ctx)
+		if err != nil {
+			return err
+		}
+
+		j, err := json.MarshalIndent(&st, "", "  ")
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(string(j))
+
+		return nil
+	},
+}
diff --git a/go.mod b/go.mod
index 187c6d6b4..c1a65f9c9 100644
--- a/go.mod
+++ b/go.mod
@@ -28,7 +28,7 @@ require (
 	github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261
 	github.com/filecoin-project/go-statestore v0.1.0
 	github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
-	github.com/filecoin-project/sector-storage v0.0.0-20200724175351-295300ff4671
+	github.com/filecoin-project/sector-storage v0.0.0-20200727112136-9377cb376d25
 	github.com/filecoin-project/specs-actors v0.8.1-0.20200724015154-3c690d9b7e1d
 	github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea
 	github.com/filecoin-project/storage-fsm v0.0.0-20200720190000-2cfe2fe3c334
diff --git a/go.sum b/go.sum
index c065177ef..4bd8bb88b 100644
--- a/go.sum
+++ b/go.sum
@@ -264,8 +264,8 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/
 github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM=
 github.com/filecoin-project/sector-storage v0.0.0-20200712023225-1d67dcfa3c15 h1:miw6hiusb/MkV1ryoqUKKWnvHhPW00AYtyeCj0L8pqo=
 github.com/filecoin-project/sector-storage v0.0.0-20200712023225-1d67dcfa3c15/go.mod h1:salgVdX7qeXFo/xaiEQE29J4pPkjn71T0kt0n+VDBzo=
-github.com/filecoin-project/sector-storage v0.0.0-20200724175351-295300ff4671 h1:9TqtFwy+6/Ywh3j81vFy3jQerD5DkxFY2Ul8nJPUHwg=
-github.com/filecoin-project/sector-storage v0.0.0-20200724175351-295300ff4671/go.mod h1:f9W29dKqNFm8Su4OddGwkAQOYMKYUR5Fk2oC/JZDjCI=
+github.com/filecoin-project/sector-storage v0.0.0-20200727112136-9377cb376d25 h1:sTonFkDw3KrIFIJTfIevYXyk+Mu9LbjbOHn/fWoMOMc=
+github.com/filecoin-project/sector-storage v0.0.0-20200727112136-9377cb376d25/go.mod h1:f9W29dKqNFm8Su4OddGwkAQOYMKYUR5Fk2oC/JZDjCI=
 github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA=
 github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y=
 github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=
diff --git a/node/impl/storminer.go b/node/impl/storminer.go
index a5a405735..40b1d8d8c 100644
--- a/node/impl/storminer.go
+++ b/node/impl/storminer.go
@@ -229,6 +229,10 @@ func (sm *StorageMinerAPI) WorkerConnect(ctx context.Context, url string) error
 	return sm.StorageMgr.AddWorker(ctx, w)
 }
 
+func (sm *StorageMinerAPI) SealingSchedDiag(ctx context.Context) (interface{}, error) {
+	return sm.StorageMgr.SchedDiag(ctx)
+}
+
 func (sm *StorageMinerAPI) MarketImportDealData(ctx context.Context, propCid cid.Cid, path string) error {
 	fi, err := os.Open(path)
 	if err != nil {

From 786fe647231274b080bf740a1d98435237a3455a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 27 Jul 2020 13:23:55 +0200
Subject: [PATCH 24/37] gofmt

---
 api/test/window_post.go | 2 +-
 node/node_test.go       | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/api/test/window_post.go b/api/test/window_post.go
index 074ba2fbb..b03fe36cc 100644
--- a/api/test/window_post.go
+++ b/api/test/window_post.go
@@ -233,7 +233,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 	require.Equal(t, p.MinerPower, p.TotalPower)
 
 	sectors := p.MinerPower.RawBytePower.Uint64() / uint64(ssz)
-	require.Equal(t, nSectors+GenesisPreseals - 3, int(sectors)) // -3 just removed sectors
+	require.Equal(t, nSectors+GenesisPreseals-3, int(sectors)) // -3 just removed sectors
 
 	mine = false
 	<-done
diff --git a/node/node_test.go b/node/node_test.go
index fcef0c444..6e2437437 100644
--- a/node/node_test.go
+++ b/node/node_test.go
@@ -380,7 +380,6 @@ func mockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test
 		}
 	}
 
-
 	for i, def := range storage {
 		// TODO: support non-bootstrap miners
 

From 26998fca32658faf6a5fef21f256f91faa402f09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=88=98=E6=9E=97=E6=AC=A3?= <llx@ipfsmain.cn>
Date: Fri, 24 Jul 2020 16:04:04 +0800
Subject: [PATCH 25/37] add on-chain-info to lotus-miner sectors status command

---
 api/api_storage.go                 | 15 ++++++++++-
 api/apistruct/struct.go            |  6 ++---
 api/test/deals.go                  |  2 +-
 api/test/window_post.go            |  2 +-
 cmd/lotus-storage-miner/info.go    |  2 +-
 cmd/lotus-storage-miner/sectors.go | 22 ++++++++++++++--
 node/impl/storminer.go             | 40 +++++++++++++++++++++++++++---
 7 files changed, 77 insertions(+), 12 deletions(-)

diff --git a/api/api_storage.go b/api/api_storage.go
index 7e6d81b21..3e4f54867 100644
--- a/api/api_storage.go
+++ b/api/api_storage.go
@@ -30,7 +30,7 @@ type StorageMiner interface {
 	PledgeSector(context.Context) error
 
 	// Get the status of a given sector by ID
-	SectorsStatus(context.Context, abi.SectorNumber) (SectorInfo, error)
+	SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (SectorInfo, error)
 
 	// List all staged sectors
 	SectorsList(context.Context) ([]abi.SectorNumber, error)
@@ -117,6 +117,19 @@ type SectorInfo struct {
 	LastErr string
 
 	Log []SectorLog
+
+	// On Chain Info
+	SealProof          abi.RegisteredSealProof // The seal proof type implies the PoSt proof/s
+	Activation         abi.ChainEpoch  // Epoch during which the sector proof was accepted
+	Expiration         abi.ChainEpoch  // Epoch during which the sector expires
+	DealWeight         abi.DealWeight  // Integral of active deals over sector lifetime
+	VerifiedDealWeight abi.DealWeight  // Integral of active verified deals over sector lifetime
+	InitialPledge      abi.TokenAmount // Pledge collected to commit this sector
+	// Expiration Info
+	OnTime abi.ChainEpoch
+	// non-zero if sector is faulty, epoch at which it will be permanently
+	// removed if it doesn't recover
+	Early abi.ChainEpoch
 }
 
 type SealedRef struct {
diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go
index 6b41fa9dd..c0320fdcd 100644
--- a/api/apistruct/struct.go
+++ b/api/apistruct/struct.go
@@ -215,7 +215,7 @@ type StorageMinerStruct struct {
 
 		PledgeSector func(context.Context) error `perm:"write"`
 
-		SectorsStatus                 func(context.Context, abi.SectorNumber) (api.SectorInfo, error) `perm:"read"`
+		SectorsStatus                 func(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) `perm:"read"`
 		SectorsList                   func(context.Context) ([]abi.SectorNumber, error)               `perm:"read"`
 		SectorsRefs                   func(context.Context) (map[string][]api.SealedRef, error)       `perm:"read"`
 		SectorStartSealing            func(context.Context, abi.SectorNumber) error                   `perm:"write"`
@@ -845,8 +845,8 @@ func (c *StorageMinerStruct) PledgeSector(ctx context.Context) error {
 }
 
 // Get the status of a given sector by ID
-func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid abi.SectorNumber) (api.SectorInfo, error) {
-	return c.Internal.SectorsStatus(ctx, sid)
+func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) {
+	return c.Internal.SectorsStatus(ctx, sid, showOnChainInfo)
 }
 
 // List all staged sectors
diff --git a/api/test/deals.go b/api/test/deals.go
index f8a81409f..31224ecd2 100644
--- a/api/test/deals.go
+++ b/api/test/deals.go
@@ -192,7 +192,7 @@ func startSealingWaiting(t *testing.T, ctx context.Context, miner TestStorageNod
 	require.NoError(t, err)
 
 	for _, snum := range snums {
-		si, err := miner.SectorsStatus(ctx, snum)
+		si, err := miner.SectorsStatus(ctx, snum, false)
 		require.NoError(t, err)
 
 		t.Logf("Sector state: %s", si.State)
diff --git a/api/test/window_post.go b/api/test/window_post.go
index 074ba2fbb..c94e7bc95 100644
--- a/api/test/window_post.go
+++ b/api/test/window_post.go
@@ -97,7 +97,7 @@ func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n i
 
 	for len(toCheck) > 0 {
 		for n := range toCheck {
-			st, err := miner.SectorsStatus(ctx, n)
+			st, err := miner.SectorsStatus(ctx, n, false)
 			require.NoError(t, err)
 			if st.State == api.SectorState(sealing.Proving) {
 				delete(toCheck, n)
diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go
index 347116f2b..bbcab62f2 100644
--- a/cmd/lotus-storage-miner/info.go
+++ b/cmd/lotus-storage-miner/info.go
@@ -222,7 +222,7 @@ func sectorsInfo(ctx context.Context, napi api.StorageMiner) error {
 		"Total": len(sectors),
 	}
 	for _, s := range sectors {
-		st, err := napi.SectorsStatus(ctx, s)
+		st, err := napi.SectorsStatus(ctx, s, false)
 		if err != nil {
 			return err
 		}
diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go
index c1a5a663a..59bb97c99 100644
--- a/cmd/lotus-storage-miner/sectors.go
+++ b/cmd/lotus-storage-miner/sectors.go
@@ -60,6 +60,10 @@ var sectorsStatusCmd = &cli.Command{
 			Name:  "log",
 			Usage: "display event log",
 		},
+		&cli.BoolFlag{
+			Name:  "on-chain-info",
+			Usage: "show sector on chain info",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
@@ -78,7 +82,8 @@ var sectorsStatusCmd = &cli.Command{
 			return err
 		}
 
-		status, err := nodeApi.SectorsStatus(ctx, abi.SectorNumber(id))
+		onChainInfo := cctx.Bool("on-chain-info")
+		status, err := nodeApi.SectorsStatus(ctx, abi.SectorNumber(id), onChainInfo)
 		if err != nil {
 			return err
 		}
@@ -98,6 +103,19 @@ var sectorsStatusCmd = &cli.Command{
 			fmt.Printf("Last Error:\t\t%s\n", status.LastErr)
 		}
 
+		if onChainInfo {
+			fmt.Printf("\nSector On Chain Info\n")
+			fmt.Printf("SealProof:\t\t%x\n", status.SealProof)
+			fmt.Printf("Activation:\t\t%v\n", status.Activation)
+			fmt.Printf("Expiration:\t\t%v\n", status.Expiration)
+			fmt.Printf("DealWeight:\t\t%v\n", status.DealWeight)
+			fmt.Printf("VerifiedDealWeight:\t\t%v\n", status.VerifiedDealWeight)
+			fmt.Printf("InitialPledge:\t\t%v\n", status.InitialPledge)
+			fmt.Printf("\nExpiration Info\n")
+			fmt.Printf("OnTime:\t\t%v\n", status.OnTime)
+			fmt.Printf("Early:\t\t%v\n", status.Early)
+		}
+
 		if cctx.Bool("log") {
 			fmt.Printf("--------\nEvent Log:\n")
 
@@ -165,7 +183,7 @@ var sectorsListCmd = &cli.Command{
 		w := tabwriter.NewWriter(os.Stdout, 8, 4, 1, ' ', 0)
 
 		for _, s := range list {
-			st, err := nodeApi.SectorsStatus(ctx, s)
+			st, err := nodeApi.SectorsStatus(ctx, s, false)
 			if err != nil {
 				fmt.Fprintf(w, "%d:\tError: %s\n", s, err)
 				continue
diff --git a/node/impl/storminer.go b/node/impl/storminer.go
index a5a405735..f6619bd2f 100644
--- a/node/impl/storminer.go
+++ b/node/impl/storminer.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"github.com/filecoin-project/sector-storage/fsutil"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
 	"net/http"
@@ -102,7 +103,7 @@ func (sm *StorageMinerAPI) PledgeSector(ctx context.Context) error {
 	return sm.Miner.PledgeSector()
 }
 
-func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumber) (api.SectorInfo, error) {
+func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) {
 	info, err := sm.Miner.GetSectorInfo(sid)
 	if err != nil {
 		return api.SectorInfo{}, err
@@ -126,7 +127,7 @@ func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumb
 		}
 	}
 
-	return api.SectorInfo{
+	sInfo := api.SectorInfo{
 		SectorID: sid,
 		State:    api.SectorState(info.State),
 		CommD:    info.CommD,
@@ -145,7 +146,40 @@ func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumb
 
 		LastErr: info.LastErr,
 		Log:     log,
-	}, nil
+		// on chain info
+		SealProof: 0,
+		Activation: 0,
+		Expiration: 0,
+		DealWeight: big.Zero(),
+		VerifiedDealWeight: big.Zero(),
+		InitialPledge: big.Zero(),
+		OnTime: 0,
+		Early: 0,
+	}
+
+	if !showOnChainInfo {
+		return sInfo, nil
+	}
+
+	onChainInfo, err := sm.Full.StateSectorGetInfo(ctx, sm.Miner.Address(), sid, types.EmptyTSK)
+	if err != nil {
+		return sInfo, nil
+	}
+	sInfo.SealProof = onChainInfo.SealProof
+	sInfo.Activation = onChainInfo.Activation
+	sInfo.Expiration = onChainInfo.Expiration
+	sInfo.DealWeight = onChainInfo.DealWeight
+	sInfo.VerifiedDealWeight = onChainInfo.VerifiedDealWeight
+	sInfo.InitialPledge = onChainInfo.InitialPledge
+
+	ex, err := sm.Full.StateSectorExpiration(ctx, sm.Miner.Address(), sid, types.EmptyTSK)
+	if err != nil {
+		return sInfo, nil
+	}
+	sInfo.OnTime = ex.OnTime
+	sInfo.Early = ex.Early
+
+	return sInfo, nil
 }
 
 // List all staged sectors

From 0adfd68faeec87f4c74d8506cba2840c1fed53d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 27 Jul 2020 18:15:47 +0200
Subject: [PATCH 26/37] Better miner proving deadlines

---
 cmd/lotus-storage-miner/proving.go | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go
index c6cfcaf30..85379a748 100644
--- a/cmd/lotus-storage-miner/proving.go
+++ b/cmd/lotus-storage-miner/proving.go
@@ -298,7 +298,7 @@ var provingDeadlinesCmd = &cli.Command{
 		fmt.Printf("Miner: %s\n", color.BlueString("%s", maddr))
 
 		tw := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0)
-		_, _ = fmt.Fprintln(tw, "deadline\tpartitions\tsectors\tproven")
+		_, _ = fmt.Fprintln(tw, "deadline\tpartitions\tsectors (faults)\tproven partitions")
 
 		for dlIdx, deadline := range deadlines {
 			partitions, err := api.StateMinerPartitions(ctx, maddr, uint64(dlIdx), types.EmptyTSK)
@@ -311,11 +311,30 @@ var provingDeadlinesCmd = &cli.Command{
 				return err
 			}
 
+			sectors := uint64(0)
+			faults := uint64(0)
+
+			for _, partition := range partitions {
+				sc, err := partition.Sectors.Count()
+				if err != nil {
+					return err
+				}
+
+				sectors += sc
+
+				fc, err := partition.Faults.Count()
+				if err != nil {
+					return err
+				}
+
+				faults += fc
+			}
+
 			var cur string
 			if di.Index == uint64(dlIdx) {
 				cur += "\t(current)"
 			}
-			_, _ = fmt.Fprintf(tw, "%d\t%d\t%d%s\n", dlIdx, len(partitions), provenPartitions, cur)
+			_, _ = fmt.Fprintf(tw, "%d\t%d\t%d (%d)\t%d%s\n", dlIdx, len(partitions), sectors, faults, provenPartitions, cur)
 		}
 
 		return tw.Flush()

From d0f4ede99bb3372f8ff33242e67580349ef9ae1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Mon, 27 Jul 2020 18:23:01 +0200
Subject: [PATCH 27/37] Print totals in mpool stat

---
 cli/mpool.go | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/cli/mpool.go b/cli/mpool.go
index d5f3eaa10..169becf45 100644
--- a/cli/mpool.go
+++ b/cli/mpool.go
@@ -225,10 +225,19 @@ var mpoolStat = &cli.Command{
 			return out[i].addr < out[j].addr
 		})
 
+		var total mpStat
+
 		for _, stat := range out {
-			fmt.Printf("%s, past: %d, cur: %d, future: %d\n", stat.addr, stat.past, stat.cur, stat.future)
+			total.past += stat.past
+			total.cur += stat.cur
+			total.future += stat.future
+
+			fmt.Printf("%s: past: %d, cur: %d, future: %d\n", stat.addr, stat.past, stat.cur, stat.future)
 		}
 
+		fmt.Println("-----")
+		fmt.Printf("total: past: %d, cur: %d, future: %d\n", total.past, total.cur, total.future)
+
 		return nil
 	},
 }

From 10362ab9b43fb79e0e3a6482996aa944523eabc1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 28 Jul 2020 01:22:20 +0200
Subject: [PATCH 28/37] markets: Fix offset in LocatePieceForDealWithinSector

---
 api/api_storage.go                 |  2 +-
 api/cbor_gen.go                    |  2 +-
 api/test/deals.go                  | 84 +++++++++++++++++++++++++++++-
 api/test/mining.go                 |  2 +-
 markets/storageadapter/provider.go |  2 +-
 node/node_test.go                  |  6 +++
 storage/sectorblocks/blocks.go     |  4 +-
 7 files changed, 94 insertions(+), 8 deletions(-)

diff --git a/api/api_storage.go b/api/api_storage.go
index b1c106c00..502ebeab2 100644
--- a/api/api_storage.go
+++ b/api/api_storage.go
@@ -137,7 +137,7 @@ type SectorInfo struct {
 
 type SealedRef struct {
 	SectorID abi.SectorNumber
-	Offset   uint64
+	Offset   abi.PaddedPieceSize
 	Size     abi.UnpaddedPieceSize
 }
 
diff --git a/api/cbor_gen.go b/api/cbor_gen.go
index f4d06d917..3414aaf8c 100644
--- a/api/cbor_gen.go
+++ b/api/cbor_gen.go
@@ -317,7 +317,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 				if maj != cbg.MajUnsignedInt {
 					return fmt.Errorf("wrong type for uint64 field")
 				}
-				t.Offset = uint64(extra)
+				t.Offset = abi.PaddedPieceSize(extra)
 
 			}
 			// t.Size (abi.UnpaddedPieceSize) (uint64)
diff --git a/api/test/deals.go b/api/test/deals.go
index 31224ecd2..3b274977d 100644
--- a/api/test/deals.go
+++ b/api/test/deals.go
@@ -129,7 +129,7 @@ func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNod
 
 	// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
 	time.Sleep(time.Second)
-	waitDealSealed(t, ctx, miner, client, deal)
+	waitDealSealed(t, ctx, miner, client, deal, false)
 
 	// Retrieval
 	info, err := client.ClientGetDealInfo(ctx, *deal)
@@ -138,6 +138,83 @@ func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNod
 	testRetrieval(t, ctx, err, client, fcid, &info.PieceCID, carExport, data)
 }
 
+func TestSenondDealRetrieval(t *testing.T, b APIBuilder, blocktime time.Duration) {
+	_ = os.Setenv("BELLMAN_NO_GPU", "1")
+
+	ctx := context.Background()
+	n, sn := b(t, 1, oneMiner)
+	client := n[0].FullNode.(*impl.FullNodeAPI)
+	miner := sn[0]
+
+	addrinfo, err := client.NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := miner.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Second)
+
+	mine := int64(1)
+	done := make(chan struct{})
+
+	go func() {
+		defer close(done)
+		for atomic.LoadInt64(&mine) == 1 {
+			time.Sleep(blocktime)
+			if err := sn[0].MineOne(ctx, func(bool, error) {}); err != nil {
+				t.Error(err)
+			}
+		}
+	}()
+
+	{
+		data1 := make([]byte, 800)
+		rand.New(rand.NewSource(int64(3))).Read(data1)
+		r := bytes.NewReader(data1)
+
+
+		fcid1, err := client.ClientImportLocal(ctx, r)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		data2 := make([]byte, 800)
+		rand.New(rand.NewSource(int64(9))).Read(data2)
+		r2 := bytes.NewReader(data2)
+
+		fcid2, err := client.ClientImportLocal(ctx, r2)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		deal1 := startDeal(t, ctx, miner, client, fcid1, true)
+
+		// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
+		time.Sleep(time.Second)
+		waitDealSealed(t, ctx, miner, client, deal1, true)
+
+		deal2 := startDeal(t, ctx, miner, client, fcid2, true)
+
+		time.Sleep(time.Second)
+		waitDealSealed(t, ctx, miner, client, deal2, false)
+
+		// Retrieval
+		info, err := client.ClientGetDealInfo(ctx, *deal2)
+		require.NoError(t, err)
+
+		rf, _ := miner.SectorsRefs(ctx)
+		fmt.Printf("refs: %+v\n", rf)
+
+		testRetrieval(t, ctx, err, client, fcid2, &info.PieceCID, false, data2)
+	}
+
+	atomic.AddInt64(&mine, -1)
+	fmt.Println("shutting down mining")
+	<-done
+}
+
 func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, fcid cid.Cid, fastRet bool) *cid.Cid {
 	maddr, err := miner.ActorAddress(ctx)
 	if err != nil {
@@ -162,7 +239,7 @@ func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client
 	return deal
 }
 
-func waitDealSealed(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, deal *cid.Cid) {
+func waitDealSealed(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, deal *cid.Cid, noseal bool) {
 loop:
 	for {
 		di, err := client.ClientGetDealInfo(ctx, *deal)
@@ -171,6 +248,9 @@ loop:
 		}
 		switch di.State {
 		case storagemarket.StorageDealSealing:
+			if noseal {
+				return
+			}
 			startSealingWaiting(t, ctx, miner)
 		case storagemarket.StorageDealProposalRejected:
 			t.Fatal("deal rejected")
diff --git a/api/test/mining.go b/api/test/mining.go
index dcbd59dd1..d5c9ab6e7 100644
--- a/api/test/mining.go
+++ b/api/test/mining.go
@@ -194,7 +194,7 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo
 	// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
 	time.Sleep(time.Second)
 
-	waitDealSealed(t, ctx, provider, client, deal)
+	waitDealSealed(t, ctx, provider, client, deal, false)
 
 	<-minedTwo
 
diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go
index 968ced4cf..86419df42 100644
--- a/markets/storageadapter/provider.go
+++ b/markets/storageadapter/provider.go
@@ -225,7 +225,7 @@ func (n *ProviderNodeAdapter) LocatePieceForDealWithinSector(ctx context.Context
 	if bestSi.State == sealing.UndefinedSectorState {
 		return 0, 0, 0, xerrors.New("no sealed sector found")
 	}
-	return uint64(best.SectorID), best.Offset, uint64(best.Size), nil
+	return uint64(best.SectorID), uint64(best.Offset.Unpadded()), uint64(best.Size), nil
 }
 
 func (n *ProviderNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider address.Address, dealID abi.DealID, cb storagemarket.DealSectorCommittedCallback) error {
diff --git a/node/node_test.go b/node/node_test.go
index 6e2437437..bfa154512 100644
--- a/node/node_test.go
+++ b/node/node_test.go
@@ -491,6 +491,8 @@ func TestAPIDealFlowReal(t *testing.T) {
 	logging.SetLogLevel("sub", "ERROR")
 	logging.SetLogLevel("storageminer", "ERROR")
 
+	saminer.PreCommitChallengeDelay = 5
+
 	t.Run("basic", func(t *testing.T) {
 		test.TestDealFlow(t, builder, time.Second, false, false)
 	})
@@ -498,6 +500,10 @@ func TestAPIDealFlowReal(t *testing.T) {
 	t.Run("fast-retrieval", func(t *testing.T) {
 		test.TestDealFlow(t, builder, time.Second, false, true)
 	})
+
+	t.Run("retrieval-second", func(t *testing.T) {
+		test.TestSenondDealRetrieval(t, builder, time.Second)
+	})
 }
 
 func TestDealMining(t *testing.T) {
diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go
index a09305ed2..af4fa52d2 100644
--- a/storage/sectorblocks/blocks.go
+++ b/storage/sectorblocks/blocks.go
@@ -64,7 +64,7 @@ func NewSectorBlocks(miner *storage.Miner, ds dtypes.MetadataDS) *SectorBlocks {
 	return sbc
 }
 
-func (st *SectorBlocks) writeRef(dealID abi.DealID, sectorID abi.SectorNumber, offset uint64, size abi.UnpaddedPieceSize) error {
+func (st *SectorBlocks) writeRef(dealID abi.DealID, sectorID abi.SectorNumber, offset abi.PaddedPieceSize, size abi.UnpaddedPieceSize) error {
 	st.keyLk.Lock() // TODO: make this multithreaded
 	defer st.keyLk.Unlock()
 
@@ -102,7 +102,7 @@ func (st *SectorBlocks) AddPiece(ctx context.Context, size abi.UnpaddedPieceSize
 		return 0, err
 	}
 
-	err = st.writeRef(d.DealID, sn, offset, size)
+	err = st.writeRef(d.DealID, sn, abi.PaddedPieceSize(offset), size)
 	if err != nil {
 		return 0, xerrors.Errorf("writeRef: %w", err)
 	}

From c133a37b7c2c73633b9ed4d3716ea9845dcc8511 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 28 Jul 2020 01:22:41 +0200
Subject: [PATCH 29/37] gofmt

---
 api/api_storage.go      | 10 +++++-----
 api/apistruct/struct.go | 20 ++++++++++----------
 api/test/deals.go       |  1 -
 node/impl/storminer.go  | 14 +++++++-------
 4 files changed, 22 insertions(+), 23 deletions(-)

diff --git a/api/api_storage.go b/api/api_storage.go
index 502ebeab2..1a94f8f0d 100644
--- a/api/api_storage.go
+++ b/api/api_storage.go
@@ -123,11 +123,11 @@ type SectorInfo struct {
 
 	// On Chain Info
 	SealProof          abi.RegisteredSealProof // The seal proof type implies the PoSt proof/s
-	Activation         abi.ChainEpoch  // Epoch during which the sector proof was accepted
-	Expiration         abi.ChainEpoch  // Epoch during which the sector expires
-	DealWeight         abi.DealWeight  // Integral of active deals over sector lifetime
-	VerifiedDealWeight abi.DealWeight  // Integral of active verified deals over sector lifetime
-	InitialPledge      abi.TokenAmount // Pledge collected to commit this sector
+	Activation         abi.ChainEpoch          // Epoch during which the sector proof was accepted
+	Expiration         abi.ChainEpoch          // Epoch during which the sector expires
+	DealWeight         abi.DealWeight          // Integral of active deals over sector lifetime
+	VerifiedDealWeight abi.DealWeight          // Integral of active verified deals over sector lifetime
+	InitialPledge      abi.TokenAmount         // Pledge collected to commit this sector
 	// Expiration Info
 	OnTime abi.ChainEpoch
 	// non-zero if sector is faulty, epoch at which it will be permanently
diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go
index 0ef159448..77b500cc9 100644
--- a/api/apistruct/struct.go
+++ b/api/apistruct/struct.go
@@ -216,16 +216,16 @@ type StorageMinerStruct struct {
 		PledgeSector func(context.Context) error `perm:"write"`
 
 		SectorsStatus                 func(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) `perm:"read"`
-		SectorsList                   func(context.Context) ([]abi.SectorNumber, error)               `perm:"read"`
-		SectorsRefs                   func(context.Context) (map[string][]api.SealedRef, error)       `perm:"read"`
-		SectorStartSealing            func(context.Context, abi.SectorNumber) error                   `perm:"write"`
-		SectorSetSealDelay            func(context.Context, time.Duration) error                      `perm:"write"`
-		SectorGetSealDelay            func(context.Context) (time.Duration, error)                    `perm:"read"`
-		SectorSetExpectedSealDuration func(context.Context, time.Duration) error                      `perm:"write"`
-		SectorGetExpectedSealDuration func(context.Context) (time.Duration, error)                    `perm:"read"`
-		SectorsUpdate                 func(context.Context, abi.SectorNumber, api.SectorState) error  `perm:"admin"`
-		SectorRemove                  func(context.Context, abi.SectorNumber) error                   `perm:"admin"`
-		SectorMarkForUpgrade          func(ctx context.Context, id abi.SectorNumber) error            `perm:"admin"`
+		SectorsList                   func(context.Context) ([]abi.SectorNumber, error)                                             `perm:"read"`
+		SectorsRefs                   func(context.Context) (map[string][]api.SealedRef, error)                                     `perm:"read"`
+		SectorStartSealing            func(context.Context, abi.SectorNumber) error                                                 `perm:"write"`
+		SectorSetSealDelay            func(context.Context, time.Duration) error                                                    `perm:"write"`
+		SectorGetSealDelay            func(context.Context) (time.Duration, error)                                                  `perm:"read"`
+		SectorSetExpectedSealDuration func(context.Context, time.Duration) error                                                    `perm:"write"`
+		SectorGetExpectedSealDuration func(context.Context) (time.Duration, error)                                                  `perm:"read"`
+		SectorsUpdate                 func(context.Context, abi.SectorNumber, api.SectorState) error                                `perm:"admin"`
+		SectorRemove                  func(context.Context, abi.SectorNumber) error                                                 `perm:"admin"`
+		SectorMarkForUpgrade          func(ctx context.Context, id abi.SectorNumber) error                                          `perm:"admin"`
 
 		WorkerConnect func(context.Context, string) error                             `perm:"admin"` // TODO: worker perm
 		WorkerStats   func(context.Context) (map[uint64]storiface.WorkerStats, error) `perm:"admin"`
diff --git a/api/test/deals.go b/api/test/deals.go
index 3b274977d..7fbb1a2b8 100644
--- a/api/test/deals.go
+++ b/api/test/deals.go
@@ -174,7 +174,6 @@ func TestSenondDealRetrieval(t *testing.T, b APIBuilder, blocktime time.Duration
 		rand.New(rand.NewSource(int64(3))).Read(data1)
 		r := bytes.NewReader(data1)
 
-
 		fcid1, err := client.ClientImportLocal(ctx, r)
 		if err != nil {
 			t.Fatal(err)
diff --git a/node/impl/storminer.go b/node/impl/storminer.go
index 68bbefb8d..51e3dcb9e 100644
--- a/node/impl/storminer.go
+++ b/node/impl/storminer.go
@@ -147,14 +147,14 @@ func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumb
 		LastErr: info.LastErr,
 		Log:     log,
 		// on chain info
-		SealProof: 0,
-		Activation: 0,
-		Expiration: 0,
-		DealWeight: big.Zero(),
+		SealProof:          0,
+		Activation:         0,
+		Expiration:         0,
+		DealWeight:         big.Zero(),
 		VerifiedDealWeight: big.Zero(),
-		InitialPledge: big.Zero(),
-		OnTime: 0,
-		Early: 0,
+		InitialPledge:      big.Zero(),
+		OnTime:             0,
+		Early:              0,
 	}
 
 	if !showOnChainInfo {

From ba7eaf3cc790baaab06ffdd0f22ac9e190bffce5 Mon Sep 17 00:00:00 2001
From: frrist <forrest@protocol.ai>
Date: Mon, 27 Jul 2020 16:55:21 -0700
Subject: [PATCH 30/37] fix: reward actor processes null round states

---
 cmd/lotus-chainwatch/processor/processor.go | 21 ++++++++---
 cmd/lotus-chainwatch/processor/reward.go    | 41 +++++++++++++++++++--
 2 files changed, 52 insertions(+), 10 deletions(-)

diff --git a/cmd/lotus-chainwatch/processor/processor.go b/cmd/lotus-chainwatch/processor/processor.go
index f91709b56..ef43601ec 100644
--- a/cmd/lotus-chainwatch/processor/processor.go
+++ b/cmd/lotus-chainwatch/processor/processor.go
@@ -119,7 +119,7 @@ func (p *Processor) Start(ctx context.Context) {
 				// TODO special case genesis state handling here to avoid all the special cases that will be needed for it else where
 				// before doing "normal" processing.
 
-				actorChanges, err := p.collectActorChanges(ctx, toProcess)
+				actorChanges, nullRounds, err := p.collectActorChanges(ctx, toProcess)
 				if err != nil {
 					log.Fatalw("Failed to collect actor changes", "error", err)
 				}
@@ -141,7 +141,7 @@ func (p *Processor) Start(ctx context.Context) {
 				})
 
 				grp.Go(func() error {
-					if err := p.HandleRewardChanges(ctx, actorChanges[builtin.RewardActorCodeID]); err != nil {
+					if err := p.HandleRewardChanges(ctx, actorChanges[builtin.RewardActorCodeID], nullRounds); err != nil {
 						return xerrors.Errorf("Failed to handle reward changes: %w", err)
 					}
 					return nil
@@ -191,7 +191,7 @@ func (p *Processor) refreshViews() error {
 	return nil
 }
 
-func (p *Processor) collectActorChanges(ctx context.Context, toProcess map[cid.Cid]*types.BlockHeader) (map[cid.Cid]ActorTips, error) {
+func (p *Processor) collectActorChanges(ctx context.Context, toProcess map[cid.Cid]*types.BlockHeader) (map[cid.Cid]ActorTips, []types.TipSetKey, error) {
 	start := time.Now()
 	defer func() {
 		log.Debugw("Collected Actor Changes", "duration", time.Since(start).String())
@@ -204,6 +204,9 @@ func (p *Processor) collectActorChanges(ctx context.Context, toProcess map[cid.C
 	var changes map[string]types.Actor
 	actorsSeen := map[cid.Cid]struct{}{}
 
+	var nullRounds []types.TipSetKey
+	var nullBlkMu sync.Mutex
+
 	// collect all actor state that has changes between block headers
 	paDone := 0
 	parmap.Par(50, parmap.MapArr(toProcess), func(bh *types.BlockHeader) {
@@ -217,6 +220,12 @@ func (p *Processor) collectActorChanges(ctx context.Context, toProcess map[cid.C
 			panic(err)
 		}
 
+		if pts.ParentState().Equals(bh.ParentStateRoot) {
+			nullBlkMu.Lock()
+			nullRounds = append(nullRounds, pts.Key())
+			nullBlkMu.Unlock()
+		}
+
 		// collect all actors that had state changes between the blockheader parent-state and its grandparent-state.
 		// TODO: changes will contain deleted actors, this causes needless processing further down the pipeline, consider
 		// a separate strategy for deleted actors
@@ -232,12 +241,12 @@ func (p *Processor) collectActorChanges(ctx context.Context, toProcess map[cid.C
 
 			addr, err := address.NewFromString(a)
 			if err != nil {
-				panic(err)
+				log.Fatal(err.Error())
 			}
 
 			ast, err := p.node.StateReadState(ctx, addr, pts.Key())
 			if err != nil {
-				panic(err)
+				log.Fatal(err.Error())
 			}
 
 			// TODO look here for an empty state, maybe thats a sign the actor was deleted?
@@ -267,7 +276,7 @@ func (p *Processor) collectActorChanges(ctx context.Context, toProcess map[cid.C
 			outMu.Unlock()
 		}
 	})
-	return out, nil
+	return out, nullRounds, nil
 }
 
 func (p *Processor) unprocessedBlocks(ctx context.Context, batch int) (map[cid.Cid]*types.BlockHeader, error) {
diff --git a/cmd/lotus-chainwatch/processor/reward.go b/cmd/lotus-chainwatch/processor/reward.go
index af339b74d..19c34fa61 100644
--- a/cmd/lotus-chainwatch/processor/reward.go
+++ b/cmd/lotus-chainwatch/processor/reward.go
@@ -8,10 +8,12 @@ import (
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/reward"
+
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/types"
 )
 
 type rewardActorInfo struct {
@@ -58,8 +60,8 @@ create table if not exists chain_power
 	return tx.Commit()
 }
 
-func (p *Processor) HandleRewardChanges(ctx context.Context, rewardTips ActorTips) error {
-	rewardChanges, err := p.processRewardActors(ctx, rewardTips)
+func (p *Processor) HandleRewardChanges(ctx context.Context, rewardTips ActorTips, nullRounds []types.TipSetKey) error {
+	rewardChanges, err := p.processRewardActors(ctx, rewardTips, nullRounds)
 	if err != nil {
 		log.Fatalw("Failed to process reward actors", "error", err)
 	}
@@ -71,7 +73,7 @@ func (p *Processor) HandleRewardChanges(ctx context.Context, rewardTips ActorTip
 	return nil
 }
 
-func (p *Processor) processRewardActors(ctx context.Context, rewardTips ActorTips) ([]rewardActorInfo, error) {
+func (p *Processor) processRewardActors(ctx context.Context, rewardTips ActorTips, nullRounds []types.TipSetKey) ([]rewardActorInfo, error) {
 	start := time.Now()
 	defer func() {
 		log.Debugw("Processed Reward Actors", "duration", time.Since(start).String())
@@ -104,6 +106,37 @@ func (p *Processor) processRewardActors(ctx context.Context, rewardTips ActorTip
 			out = append(out, rw)
 		}
 	}
+	for _, tsKey := range nullRounds {
+		var rw rewardActorInfo
+		tipset , err := p.node.ChainGetTipSet(ctx, tsKey)
+		if err != nil {
+			return nil, err
+		}
+		rw.common.tsKey = tipset.Key()
+		rw.common.height = tipset.Height()
+		rw.common.stateroot = tipset.ParentState()
+		rw.common.parentTsKey = tipset.Parents()
+		// get reward actor states at each tipset once for all updates
+		rewardActor, err := p.node.StateGetActor(ctx, builtin.RewardActorAddr, tsKey)
+		if err != nil {
+			return nil, err
+		}
+
+		rewardStateRaw, err := p.node.ChainReadObj(ctx, rewardActor.Head)
+		if err != nil {
+			return nil, err
+		}
+
+		var rewardActorState reward.State
+		if err := rewardActorState.UnmarshalCBOR(bytes.NewReader(rewardStateRaw)); err != nil {
+			return nil, err
+		}
+
+		rw.baseBlockReward = rewardActorState.ThisEpochReward
+		rw.baselinePower = rewardActorState.ThisEpochBaselinePower
+		out = append(out, rw)
+	}
+
 	return out, nil
 }
 

From 5b6006f8fe51a623e5729b008ed51aeb74a7fd7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 28 Jul 2020 02:25:04 +0200
Subject: [PATCH 31/37] stmgr: Execute cron tick on null rounds

---
 chain/stmgr/stmgr.go        | 97 ++++++++++++++++++++++++++-----------
 chain/validation/applier.go |  2 +-
 2 files changed, 69 insertions(+), 30 deletions(-)

diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go
index a888e869a..73540388a 100644
--- a/chain/stmgr/stmgr.go
+++ b/chain/stmgr/stmgr.go
@@ -146,12 +146,46 @@ type BlockMessages struct {
 
 type ExecCallback func(cid.Cid, *types.Message, *vm.ApplyRet) error
 
-func (sm *StateManager) ApplyBlocks(ctx context.Context, pstate cid.Cid, bms []BlockMessages, epoch abi.ChainEpoch, r vm.Rand, cb ExecCallback) (cid.Cid, cid.Cid, error) {
-	vmi, err := sm.newVM(pstate, epoch, r, sm.cs.Blockstore(), sm.cs.VMSys())
+func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEpoch, pstate cid.Cid, bms []BlockMessages, epoch abi.ChainEpoch, r vm.Rand, cb ExecCallback) (cid.Cid, cid.Cid, error) {
+	vmi, err := sm.newVM(pstate, parentEpoch+1, r, sm.cs.Blockstore(), sm.cs.VMSys())
 	if err != nil {
 		return cid.Undef, cid.Undef, xerrors.Errorf("instantiating VM failed: %w", err)
 	}
 
+	// run cron for null rounds if any
+	for i := parentEpoch + 1; i < epoch; i++ {
+		// TODO: this nonce-getting is a tiny bit ugly
+		ca, err := vmi.StateTree().GetActor(builtin.SystemActorAddr)
+		if err != nil {
+			return cid.Undef, cid.Undef, err
+		}
+
+		cronMsg := &types.Message{
+			To:       builtin.CronActorAddr,
+			From:     builtin.SystemActorAddr,
+			Nonce:    ca.Nonce,
+			Value:    types.NewInt(0),
+			GasPrice: types.NewInt(0),
+			GasLimit: build.BlockGasLimit * 10, // Make super sure this is never too little
+			Method:   builtin.MethodsCron.EpochTick,
+			Params:   nil,
+		}
+		ret, err := vmi.ApplyImplicitMessage(ctx, cronMsg)
+		if err != nil {
+			return cid.Undef, cid.Undef, err
+		}
+		if cb != nil {
+			if err := cb(cronMsg.Cid(), cronMsg, ret); err != nil {
+				return cid.Undef, cid.Undef, xerrors.Errorf("callback failed on cron message: %w", err)
+			}
+		}
+		if ret.ExitCode != 0 {
+			return cid.Undef, cid.Undef, xerrors.Errorf("CheckProofSubmissions exit was non-zero: %d", ret.ExitCode)
+		}
+
+		vmi.SetBlockHeight(i + 1)
+	}
+
 	var receipts []cbg.CBORMarshaler
 	processedMsgs := map[cid.Cid]bool{}
 	for _, b := range bms {
@@ -222,33 +256,35 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, pstate cid.Cid, bms []B
 
 	}
 
-	// TODO: this nonce-getting is a tiny bit ugly
-	ca, err := vmi.StateTree().GetActor(builtin.SystemActorAddr)
-	if err != nil {
-		return cid.Undef, cid.Undef, err
-	}
-
-	cronMsg := &types.Message{
-		To:       builtin.CronActorAddr,
-		From:     builtin.SystemActorAddr,
-		Nonce:    ca.Nonce,
-		Value:    types.NewInt(0),
-		GasPrice: types.NewInt(0),
-		GasLimit: build.BlockGasLimit * 10, // Make super sure this is never too little
-		Method:   builtin.MethodsCron.EpochTick,
-		Params:   nil,
-	}
-	ret, err := vmi.ApplyImplicitMessage(ctx, cronMsg)
-	if err != nil {
-		return cid.Undef, cid.Undef, err
-	}
-	if cb != nil {
-		if err := cb(cronMsg.Cid(), cronMsg, ret); err != nil {
-			return cid.Undef, cid.Undef, xerrors.Errorf("callback failed on cron message: %w", err)
+	{
+		// TODO: this nonce-getting is a tiny bit ugly
+		ca, err := vmi.StateTree().GetActor(builtin.SystemActorAddr)
+		if err != nil {
+			return cid.Undef, cid.Undef, err
+		}
+
+		cronMsg := &types.Message{
+			To:       builtin.CronActorAddr,
+			From:     builtin.SystemActorAddr,
+			Nonce:    ca.Nonce,
+			Value:    types.NewInt(0),
+			GasPrice: types.NewInt(0),
+			GasLimit: build.BlockGasLimit * 10, // Make super sure this is never too little
+			Method:   builtin.MethodsCron.EpochTick,
+			Params:   nil,
+		}
+		ret, err := vmi.ApplyImplicitMessage(ctx, cronMsg)
+		if err != nil {
+			return cid.Undef, cid.Undef, err
+		}
+		if cb != nil {
+			if err := cb(cronMsg.Cid(), cronMsg, ret); err != nil {
+				return cid.Undef, cid.Undef, xerrors.Errorf("callback failed on cron message: %w", err)
+			}
+		}
+		if ret.ExitCode != 0 {
+			return cid.Undef, cid.Undef, xerrors.Errorf("CheckProofSubmissions exit was non-zero: %d", ret.ExitCode)
 		}
-	}
-	if ret.ExitCode != 0 {
-		return cid.Undef, cid.Undef, xerrors.Errorf("CheckProofSubmissions exit was non-zero: %d", ret.ExitCode)
 	}
 
 	rectarr := adt.MakeEmptyArray(sm.cs.Store(ctx))
@@ -284,6 +320,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl
 		}
 	}
 
+	var parentEpoch abi.ChainEpoch
 	pstate := blks[0].ParentStateRoot
 	if len(blks[0].Parents) > 0 { // don't support forks on genesis
 		parent, err := sm.cs.GetBlock(blks[0].Parents[0])
@@ -291,6 +328,8 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl
 			return cid.Undef, cid.Undef, xerrors.Errorf("getting parent block: %w", err)
 		}
 
+		parentEpoch = parent.Height
+
 		pstate, err = sm.handleStateForks(ctx, blks[0].ParentStateRoot, blks[0].Height, parent.Height)
 		if err != nil {
 			return cid.Undef, cid.Undef, xerrors.Errorf("error handling state forks: %w", err)
@@ -329,7 +368,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl
 		blkmsgs = append(blkmsgs, bm)
 	}
 
-	return sm.ApplyBlocks(ctx, pstate, blkmsgs, blks[0].Height, r, cb)
+	return sm.ApplyBlocks(ctx, parentEpoch, pstate, blkmsgs, blks[0].Height, r, cb)
 }
 
 func (sm *StateManager) parentState(ts *types.TipSet) cid.Cid {
diff --git a/chain/validation/applier.go b/chain/validation/applier.go
index 62ea09894..3044c77a1 100644
--- a/chain/validation/applier.go
+++ b/chain/validation/applier.go
@@ -88,7 +88,7 @@ func (a *Applier) ApplyTipSetMessages(epoch abi.ChainEpoch, blocks []vtypes.Bloc
 	}
 
 	var receipts []vtypes.MessageReceipt
-	sroot, _, err := sm.ApplyBlocks(context.TODO(), a.stateWrapper.Root(), bms, epoch, &randWrapper{rnd}, func(c cid.Cid, msg *types.Message, ret *vm.ApplyRet) error {
+	sroot, _, err := sm.ApplyBlocks(context.TODO(), epoch-1, a.stateWrapper.Root(), bms, epoch, &randWrapper{rnd}, func(c cid.Cid, msg *types.Message, ret *vm.ApplyRet) error {
 		if msg.From == builtin.SystemActorAddr {
 			return nil // ignore reward and cron calls
 		}

From 8889c5cfd380b71e0b5eaf542837d439b6b92908 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 28 Jul 2020 02:28:22 +0200
Subject: [PATCH 32/37] stmgr: Dedupe cron invocation

---
 chain/stmgr/stmgr.go | 52 ++++++++++++++------------------------------
 1 file changed, 16 insertions(+), 36 deletions(-)

diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go
index 73540388a..bc9c6b846 100644
--- a/chain/stmgr/stmgr.go
+++ b/chain/stmgr/stmgr.go
@@ -152,12 +152,11 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
 		return cid.Undef, cid.Undef, xerrors.Errorf("instantiating VM failed: %w", err)
 	}
 
-	// run cron for null rounds if any
-	for i := parentEpoch + 1; i < epoch; i++ {
+	runCron := func() error {
 		// TODO: this nonce-getting is a tiny bit ugly
 		ca, err := vmi.StateTree().GetActor(builtin.SystemActorAddr)
 		if err != nil {
-			return cid.Undef, cid.Undef, err
+			return err
 		}
 
 		cronMsg := &types.Message{
@@ -172,15 +171,24 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
 		}
 		ret, err := vmi.ApplyImplicitMessage(ctx, cronMsg)
 		if err != nil {
-			return cid.Undef, cid.Undef, err
+			return err
 		}
 		if cb != nil {
 			if err := cb(cronMsg.Cid(), cronMsg, ret); err != nil {
-				return cid.Undef, cid.Undef, xerrors.Errorf("callback failed on cron message: %w", err)
+				return xerrors.Errorf("callback failed on cron message: %w", err)
 			}
 		}
 		if ret.ExitCode != 0 {
-			return cid.Undef, cid.Undef, xerrors.Errorf("CheckProofSubmissions exit was non-zero: %d", ret.ExitCode)
+			return xerrors.Errorf("CheckProofSubmissions exit was non-zero: %d", ret.ExitCode)
+		}
+
+		return nil
+	}
+
+	// run cron for null rounds if any
+	for i := parentEpoch + 1; i < epoch; i++ {
+		if err := runCron(); err != nil {
+			return cid.Cid{}, cid.Cid{}, err
 		}
 
 		vmi.SetBlockHeight(i + 1)
@@ -253,38 +261,10 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
 		if ret.ExitCode != 0 {
 			return cid.Undef, cid.Undef, xerrors.Errorf("reward application message failed (exit %d): %s", ret.ExitCode, ret.ActorErr)
 		}
-
 	}
 
-	{
-		// TODO: this nonce-getting is a tiny bit ugly
-		ca, err := vmi.StateTree().GetActor(builtin.SystemActorAddr)
-		if err != nil {
-			return cid.Undef, cid.Undef, err
-		}
-
-		cronMsg := &types.Message{
-			To:       builtin.CronActorAddr,
-			From:     builtin.SystemActorAddr,
-			Nonce:    ca.Nonce,
-			Value:    types.NewInt(0),
-			GasPrice: types.NewInt(0),
-			GasLimit: build.BlockGasLimit * 10, // Make super sure this is never too little
-			Method:   builtin.MethodsCron.EpochTick,
-			Params:   nil,
-		}
-		ret, err := vmi.ApplyImplicitMessage(ctx, cronMsg)
-		if err != nil {
-			return cid.Undef, cid.Undef, err
-		}
-		if cb != nil {
-			if err := cb(cronMsg.Cid(), cronMsg, ret); err != nil {
-				return cid.Undef, cid.Undef, xerrors.Errorf("callback failed on cron message: %w", err)
-			}
-		}
-		if ret.ExitCode != 0 {
-			return cid.Undef, cid.Undef, xerrors.Errorf("CheckProofSubmissions exit was non-zero: %d", ret.ExitCode)
-		}
+	if err := runCron(); err != nil {
+		return cid.Cid{}, cid.Cid{}, err
 	}
 
 	rectarr := adt.MakeEmptyArray(sm.cs.Store(ctx))

From 5a623cccb5085ad1bce1431cc169580c741d9065 Mon Sep 17 00:00:00 2001
From: hannahhoward <hannah@hannahhoward.net>
Date: Mon, 27 Jul 2020 23:13:10 -0700
Subject: [PATCH 33/37] feat(markets): update markets mulitple deal stores

---
 api/api_full.go                |  7 ++--
 api/apistruct/struct.go        |  5 ++-
 api/test/deals.go              |  5 ++-
 cli/client.go                  |  5 ++-
 go.mod                         |  8 ++--
 go.sum                         | 17 ++++----
 node/builder.go                |  1 +
 node/impl/client/client.go     | 73 ++++++++++++++++++++++++----------
 node/modules/client.go         | 17 +++-----
 node/modules/dtypes/storage.go |  1 +
 node/modules/storageminer.go   | 29 ++++++++++++--
 node/repo/importmgr/mgr.go     | 10 ++---
 12 files changed, 117 insertions(+), 61 deletions(-)

diff --git a/api/api_full.go b/api/api_full.go
index 134427b67..2b14c62d8 100644
--- a/api/api_full.go
+++ b/api/api_full.go
@@ -9,6 +9,7 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/filecoin-project/go-multistore"
 	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin/market"
@@ -201,7 +202,7 @@ type FullNode interface {
 	// ClientImport imports file under the specified path into filestore.
 	ClientImport(ctx context.Context, ref FileRef) (*ImportRes, error)
 	// ClientRemoveImport removes file import
-	ClientRemoveImport(ctx context.Context, importID int) error
+	ClientRemoveImport(ctx context.Context, importID multistore.StoreID) error
 	// ClientStartDeal proposes a deal with a miner.
 	ClientStartDeal(ctx context.Context, params *StartDealParams) (*cid.Cid, error)
 	// ClientGetDealInfo returns the latest information about a given deal.
@@ -404,11 +405,11 @@ type SectorLocation struct {
 
 type ImportRes struct {
 	Root     cid.Cid
-	ImportID int
+	ImportID multistore.StoreID
 }
 
 type Import struct {
-	Key int
+	Key multistore.StoreID
 	Err string
 
 	Root     *cid.Cid
diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go
index 77b500cc9..ad5982a52 100644
--- a/api/apistruct/struct.go
+++ b/api/apistruct/struct.go
@@ -12,6 +12,7 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-jsonrpc/auth"
+	"github.com/filecoin-project/go-multistore"
 	"github.com/filecoin-project/sector-storage/fsutil"
 	"github.com/filecoin-project/sector-storage/sealtasks"
 	"github.com/filecoin-project/sector-storage/stores"
@@ -115,7 +116,7 @@ type FullNodeStruct struct {
 
 		ClientImport          func(ctx context.Context, ref api.FileRef) (*api.ImportRes, error)                                     `perm:"admin"`
 		ClientListImports     func(ctx context.Context) ([]api.Import, error)                                                        `perm:"write"`
-		ClientRemoveImport    func(ctx context.Context, importID int) error                                                          `perm:"admin"`
+		ClientRemoveImport    func(ctx context.Context, importID multistore.StoreID) error                                           `perm:"admin"`
 		ClientHasLocal        func(ctx context.Context, root cid.Cid) (bool, error)                                                  `perm:"write"`
 		ClientFindData        func(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]api.QueryOffer, error)                      `perm:"read"`
 		ClientMinerQueryOffer func(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error) `perm:"read"`
@@ -360,7 +361,7 @@ func (c *FullNodeStruct) ClientListImports(ctx context.Context) ([]api.Import, e
 	return c.Internal.ClientListImports(ctx)
 }
 
-func (c *FullNodeStruct) ClientRemoveImport(ctx context.Context, importID int) error {
+func (c *FullNodeStruct) ClientRemoveImport(ctx context.Context, importID multistore.StoreID) error {
 	return c.Internal.ClientRemoveImport(ctx, importID)
 }
 
diff --git a/api/test/deals.go b/api/test/deals.go
index 7fbb1a2b8..fb102a143 100644
--- a/api/test/deals.go
+++ b/api/test/deals.go
@@ -225,7 +225,10 @@ func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client
 		t.Fatal(err)
 	}
 	deal, err := client.ClientStartDeal(ctx, &api.StartDealParams{
-		Data:              &storagemarket.DataRef{Root: fcid},
+		Data: &storagemarket.DataRef{
+			TransferType: storagemarket.TTGraphsync,
+			Root:         fcid,
+		},
 		Wallet:            addr,
 		Miner:             maddr,
 		EpochPrice:        types.NewInt(1000000),
diff --git a/cli/client.go b/cli/client.go
index 693b2b83e..63658c27d 100644
--- a/cli/client.go
+++ b/cli/client.go
@@ -18,6 +18,7 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/filecoin-project/go-multistore"
 	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-actors/actors/builtin/market"
 
@@ -142,14 +143,14 @@ var clientDropCmd = &cli.Command{
 		defer closer()
 		ctx := ReqContext(cctx)
 
-		var ids []int
+		var ids []multistore.StoreID
 		for i, s := range cctx.Args().Slice() {
 			id, err := strconv.ParseInt(s, 10, 0)
 			if err != nil {
 				return xerrors.Errorf("parsing %d-th import ID: %w", i, err)
 			}
 
-			ids = append(ids, int(id))
+			ids = append(ids, multistore.StoreID(id))
 		}
 
 		for _, id := range ids {
diff --git a/go.mod b/go.mod
index 1c3bfd7eb..129abd241 100644
--- a/go.mod
+++ b/go.mod
@@ -21,11 +21,11 @@ require (
 	github.com/filecoin-project/go-bitfield v0.1.0
 	github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
 	github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
-	github.com/filecoin-project/go-data-transfer v0.5.0
+	github.com/filecoin-project/go-data-transfer v0.5.1
 	github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f
-	github.com/filecoin-project/go-fil-markets v0.5.1
+	github.com/filecoin-project/go-fil-markets v0.5.2-0.20200728055441-cf6b0a74e50c
 	github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24
-	github.com/filecoin-project/go-multistore v0.0.1
+	github.com/filecoin-project/go-multistore v0.0.2
 	github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261
 	github.com/filecoin-project/go-statestore v0.1.0
 	github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
@@ -53,7 +53,7 @@ require (
 	github.com/ipfs/go-ds-measure v0.1.0
 	github.com/ipfs/go-filestore v1.0.0
 	github.com/ipfs/go-fs-lock v0.0.1
-	github.com/ipfs/go-graphsync v0.0.6-0.20200715204712-ef06b3d32e83
+	github.com/ipfs/go-graphsync v0.0.6-0.20200721211002-c376cbe14c0a
 	github.com/ipfs/go-ipfs-blockstore v1.0.0
 	github.com/ipfs/go-ipfs-chunker v0.0.5
 	github.com/ipfs/go-ipfs-ds-help v1.0.0
diff --git a/go.sum b/go.sum
index f1ef52518..f5f6c00eb 100644
--- a/go.sum
+++ b/go.sum
@@ -239,17 +239,17 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:a
 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
-github.com/filecoin-project/go-data-transfer v0.5.0 h1:pvWlab69BD5dwheRHjjBjFB6m7CEqEZeI+aChtVqKVk=
-github.com/filecoin-project/go-data-transfer v0.5.0/go.mod h1:7yckbsPPMGuN3O1+SYNE/lowwheaUn5woGILpjN52UI=
+github.com/filecoin-project/go-data-transfer v0.5.1 h1:tDPmVVSgkit3cEG+9TFr6nwhKHdUipt9f0dEZSmvjbg=
+github.com/filecoin-project/go-data-transfer v0.5.1/go.mod h1:PRs78hp9u8T4G2Jce5NOkHB1bAqecPkvSLsMPuJckGU=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
-github.com/filecoin-project/go-fil-markets v0.5.1 h1:Y69glslNCuXnygfesCmyilTVhEEjcLK7CtAohKP9SL8=
-github.com/filecoin-project/go-fil-markets v0.5.1/go.mod h1:GKGigsFNMvKmx/+Mcn7093TdZTiCDLc7YGxQ7d6fq2s=
+github.com/filecoin-project/go-fil-markets v0.5.2-0.20200728055441-cf6b0a74e50c h1:cIoO/T2DxLDpPjIp1cvtreTlq8YBzIbLPk/MpVOrkrw=
+github.com/filecoin-project/go-fil-markets v0.5.2-0.20200728055441-cf6b0a74e50c/go.mod h1:tBjxNhgi3djW9RZacV3D/HhADZe+tgvfCeMyWWEgVlI=
 github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms=
 github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM=
-github.com/filecoin-project/go-multistore v0.0.1 h1:wXCd02azCxEcMNlDE9lksraQO+iIjFGNw01IZyf8GPA=
-github.com/filecoin-project/go-multistore v0.0.1/go.mod h1:z8NeSPWubEvrzi0XolhZ1NjTeW9ZDR779M+EDhf4QIQ=
+github.com/filecoin-project/go-multistore v0.0.2 h1:JZEddnXXt3mMzHi7bi9IH7Yi1NpGLy19J5Lk/xbxBMs=
+github.com/filecoin-project/go-multistore v0.0.2/go.mod h1:edte5g7DHqJasFNOvdm9ZS6CjdfFTPoQ6xeKs1eOBIA=
 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs=
 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6/go.mod h1:0HgYnrkeSU4lu1p+LEOeDpFsNBssa0OGGriWdA4hvaE=
 github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
@@ -528,8 +528,8 @@ github.com/ipfs/go-filestore v1.0.0 h1:QR7ekKH+q2AGiWDc7W2Q0qHuYSRZGUJqUn0GsegEP
 github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPiFOdcuu9SM=
 github.com/ipfs/go-fs-lock v0.0.1 h1:XHX8uW4jQBYWHj59XXcjg7BHlHxV9ZOYs6Y43yb7/l0=
 github.com/ipfs/go-fs-lock v0.0.1/go.mod h1:DNBekbboPKcxs1aukPSaOtFA3QfSdi5C855v0i9XJ8Y=
-github.com/ipfs/go-graphsync v0.0.6-0.20200715204712-ef06b3d32e83 h1:tkGDAwcZfzDFeBNyBWYOM02Qw0rGpA2UuCvq49T3K5o=
-github.com/ipfs/go-graphsync v0.0.6-0.20200715204712-ef06b3d32e83/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE=
+github.com/ipfs/go-graphsync v0.0.6-0.20200721211002-c376cbe14c0a h1:QViYKbSYNKtfivrYx69UFJiH7HfdE5APQBbIu5fCK3k=
+github.com/ipfs/go-graphsync v0.0.6-0.20200721211002-c376cbe14c0a/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE=
 github.com/ipfs/go-hamt-ipld v0.0.15-0.20200131012125-dd88a59d3f2e/go.mod h1:9aQJu/i/TaRDW6jqB5U217dLIDopn50wxLdHXM2CTfE=
 github.com/ipfs/go-hamt-ipld v0.0.15-0.20200204200533-99b8553ef242/go.mod h1:kq3Pi+UP3oHhAdKexE+kHHYRKMoFNuGero0R7q3hWGg=
 github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk=
@@ -634,7 +634,6 @@ github.com/ipfs/iptb v1.4.0 h1:YFYTrCkLMRwk/35IMyC6+yjoQSHTEcNcefBStLJzgvo=
 github.com/ipfs/iptb v1.4.0/go.mod h1:1rzHpCYtNp87/+hTxG5TfCVn/yMY3dKnLn8tBiMfdmg=
 github.com/ipfs/iptb-plugins v0.2.1 h1:au4HWn9/pRPbkxA08pDx2oRAs4cnbgQWgV0teYXuuGA=
 github.com/ipfs/iptb-plugins v0.2.1/go.mod h1:QXMbtIWZ+jRsW8a4h13qAKU7jcM7qaittO8wOsTP0Rs=
-github.com/ipld/go-car v0.1.1-0.20200429200904-c222d793c339/go.mod h1:eajxljm6I8o3LitnFeVEmucwZmz7+yLSiKce9yYMefg=
 github.com/ipld/go-car v0.1.1-0.20200526133713-1c7508d55aae h1:OV9dxl8iPMCOD8Vi/hvFwRh3JWPXqmkYSVxWr9JnEzM=
 github.com/ipld/go-car v0.1.1-0.20200526133713-1c7508d55aae/go.mod h1:2mvxpu4dKRnuH3mj5u6KW/tmRSCcXvy/KYiJ4nC6h4c=
 github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e h1:ZISbJlM0urTANR9KRfRaqlBmyOj5uUtxs2r4Up9IXsA=
diff --git a/node/builder.go b/node/builder.go
index 46fd260d5..16be84d2c 100644
--- a/node/builder.go
+++ b/node/builder.go
@@ -298,6 +298,7 @@ func Online() Option {
 			Override(new(*storage.Miner), modules.StorageMiner),
 			Override(new(dtypes.NetworkName), modules.StorageNetworkName),
 
+			Override(new(dtypes.StagingMultiDstore), modules.StagingMultiDatastore),
 			Override(new(dtypes.StagingBlockstore), modules.StagingBlockstore),
 			Override(new(dtypes.StagingDAG), modules.StagingDAG),
 			Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync),
diff --git a/node/impl/client/client.go b/node/impl/client/client.go
index 1e7fa8146..9c320f832 100644
--- a/node/impl/client/client.go
+++ b/node/impl/client/client.go
@@ -85,6 +85,27 @@ func (a *API) imgr() *importmgr.Mgr {
 }
 
 func (a *API) ClientStartDeal(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) {
+	var storeID *multistore.StoreID
+	if params.Data.TransferType == storagemarket.TTGraphsync {
+		importIDs := a.imgr().List()
+		for _, importID := range importIDs {
+			info, err := a.imgr().Info(importID)
+			if err != nil {
+				continue
+			}
+			if info.Labels[importmgr.LRootCid] == "" {
+				continue
+			}
+			c, err := cid.Parse(info.Labels[importmgr.LRootCid])
+			if err != nil {
+				continue
+			}
+			if c.Equals(params.Data.Root) {
+				storeID = &importID
+				break
+			}
+		}
+	}
 	exist, err := a.WalletHas(ctx, params.Wallet)
 	if err != nil {
 		return nil, xerrors.Errorf("failed getting addr from wallet: %w", params.Wallet)
@@ -133,19 +154,19 @@ func (a *API) ClientStartDeal(ctx context.Context, params *api.StartDealParams)
 		dealStart = ts.Height() + dealStartBuffer
 	}
 
-	result, err := a.SMDealClient.ProposeStorageDeal(
-		ctx,
-		params.Wallet,
-		&providerInfo,
-		params.Data,
-		dealStart,
-		calcDealExpiration(params.MinBlocksDuration, md, dealStart),
-		params.EpochPrice,
-		big.Zero(),
-		rt,
-		params.FastRetrieval,
-		params.VerifiedDeal,
-	)
+	result, err := a.SMDealClient.ProposeStorageDeal(ctx, storagemarket.ProposeStorageDealParams{
+		Addr:          params.Wallet,
+		Info:          &providerInfo,
+		Data:          params.Data,
+		StartEpoch:    dealStart,
+		EndEpoch:      calcDealExpiration(params.MinBlocksDuration, md, dealStart),
+		Price:         params.EpochPrice,
+		Collateral:    big.Zero(),
+		Rt:            rt,
+		FastRetrieval: params.FastRetrieval,
+		VerifiedDeal:  params.VerifiedDeal,
+		StoreID:       storeID,
+	})
 
 	if err != nil {
 		return nil, xerrors.Errorf("failed to start deal: %w", err)
@@ -300,7 +321,7 @@ func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (*api.ImportRes
 	}, nil
 }
 
-func (a *API) ClientRemoveImport(ctx context.Context, importID int) error {
+func (a *API) ClientRemoveImport(ctx context.Context, importID multistore.StoreID) error {
 	return a.imgr().Remove(importID)
 }
 
@@ -341,6 +362,9 @@ func (a *API) ClientImportLocal(ctx context.Context, f io.Reader) (cid.Cid, erro
 	if err != nil {
 		return cid.Undef, err
 	}
+	if err := a.imgr().AddLabel(id, "root", nd.Cid().String()); err != nil {
+		return cid.Cid{}, err
+	}
 
 	return nd.Cid(), bufferedDS.Commit()
 }
@@ -425,6 +449,15 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
 	if err != nil {
 		return xerrors.Errorf("Error in retrieval params: %s", err)
 	}
+	storeID, store, err := a.imgr().NewStore()
+	if err != nil {
+		return xerrors.Errorf("Error setting up new store: %w", err)
+	}
+
+	defer func() {
+		_ = a.imgr().Remove(storeID)
+	}()
+
 	_, err = a.Retrieval.Retrieve(
 		ctx,
 		order.Root,
@@ -432,7 +465,8 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
 		order.Total,
 		order.MinerPeerID,
 		order.Client,
-		order.Miner) // TODO: pass the store here   somehow
+		order.Miner,
+		&storeID) // TODO: should we ignore storeID if we are using the IPFS blockstore?
 	if err != nil {
 		return xerrors.Errorf("Retrieve failed: %w", err)
 	}
@@ -452,28 +486,27 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
 		return nil
 	}
 
-	rdag := merkledag.NewDAGService(blockservice.New(a.RetBstore, offline.Exchange(a.RetBstore)))
-
 	if ref.IsCAR {
 		f, err := os.OpenFile(ref.Path, os.O_CREATE|os.O_WRONLY, 0644)
 		if err != nil {
 			return err
 		}
-		err = car.WriteCar(ctx, rdag, []cid.Cid{order.Root}, f)
+		err = car.WriteCar(ctx, store.DAG, []cid.Cid{order.Root}, f)
 		if err != nil {
 			return err
 		}
 		return f.Close()
 	}
 
-	nd, err := rdag.Get(ctx, order.Root)
+	nd, err := store.DAG.Get(ctx, order.Root)
 	if err != nil {
 		return xerrors.Errorf("ClientRetrieve: %w", err)
 	}
-	file, err := unixfile.NewUnixfsFile(ctx, rdag, nd)
+	file, err := unixfile.NewUnixfsFile(ctx, store.DAG, nd)
 	if err != nil {
 		return xerrors.Errorf("ClientRetrieve: %w", err)
 	}
+
 	return files.WriteTo(file, ref.Path)
 }
 
diff --git a/node/modules/client.go b/node/modules/client.go
index 77e66a69a..fe81c853b 100644
--- a/node/modules/client.go
+++ b/node/modules/client.go
@@ -5,7 +5,6 @@ import (
 	"time"
 
 	"github.com/filecoin-project/go-multistore"
-	"github.com/filecoin-project/lotus/lib/bufbstore"
 	"golang.org/x/xerrors"
 
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
@@ -62,12 +61,8 @@ func ClientImportMgr(mds dtypes.ClientMultiDstore, ds dtypes.MetadataDS) dtypes.
 }
 
 func ClientBlockstore(imgr dtypes.ClientImportMgr) dtypes.ClientBlockstore {
-	// TODO: This isn't.. the best
-	//  - If it's easy to pass per-retrieval blockstores with markets we don't need this
-	//  - If it's not easy, we need to store this in a separate datastore on disk
-	defaultWrite := blockstore.NewBlockstore(datastore.NewMapDatastore())
-
-	return blockstore.NewIdStore(bufbstore.NewTieredBstore(imgr.Blockstore, defaultWrite))
+	// in most cases this is now unused in normal operations -- however, it's important to preserve for the IPFS use case
+	return blockstore.NewBlockstore(datastore.NewMapDatastore())
 }
 
 // RegisterClientValidator is an initialization hook that registers the client
@@ -117,9 +112,9 @@ func NewClientRequestValidator(deals dtypes.ClientDealStore) dtypes.ClientReques
 	return requestvalidation.NewUnifiedRequestValidator(nil, deals)
 }
 
-func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, r repo.LockedRepo, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, deals dtypes.ClientDatastore, scn storagemarket.StorageClientNode) (storagemarket.StorageClient, error) {
+func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, mds dtypes.ClientMultiDstore, r repo.LockedRepo, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, deals dtypes.ClientDatastore, scn storagemarket.StorageClientNode) (storagemarket.StorageClient, error) {
 	net := smnet.NewFromLibp2pHost(h)
-	c, err := storageimpl.NewClient(net, ibs, dataTransfer, discovery, deals, scn, storageimpl.DealPollingInterval(time.Second))
+	c, err := storageimpl.NewClient(net, ibs, mds, dataTransfer, discovery, deals, scn, storageimpl.DealPollingInterval(time.Second))
 	if err != nil {
 		return nil, err
 	}
@@ -136,9 +131,9 @@ func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, r
 }
 
 // RetrievalClient creates a new retrieval client attached to the client blockstore
-func RetrievalClient(h host.Host, bs dtypes.ClientBlockstore, dt dtypes.ClientDataTransfer, pmgr *paychmgr.Manager, payapi payapi.PaychAPI, resolver retrievalmarket.PeerResolver, ds dtypes.MetadataDS, chainapi full.ChainAPI) (retrievalmarket.RetrievalClient, error) {
+func RetrievalClient(h host.Host, mds dtypes.ClientMultiDstore, dt dtypes.ClientDataTransfer, pmgr *paychmgr.Manager, payapi payapi.PaychAPI, resolver retrievalmarket.PeerResolver, ds dtypes.MetadataDS, chainapi full.ChainAPI) (retrievalmarket.RetrievalClient, error) {
 	adapter := retrievaladapter.NewRetrievalClientNode(pmgr, payapi, chainapi)
 	network := rmnet.NewFromLibp2pHost(h)
 	sc := storedcounter.New(ds, datastore.NewKey("/retr"))
-	return retrievalimpl.NewClient(network, bs, dt, adapter, resolver, namespace.Wrap(ds, datastore.NewKey("/retrievals/client")), sc)
+	return retrievalimpl.NewClient(network, mds, dt, adapter, resolver, namespace.Wrap(ds, datastore.NewKey("/retrievals/client")), sc)
 }
diff --git a/node/modules/dtypes/storage.go b/node/modules/dtypes/storage.go
index b4712a37e..a4e75c4b8 100644
--- a/node/modules/dtypes/storage.go
+++ b/node/modules/dtypes/storage.go
@@ -50,3 +50,4 @@ type ProviderDataTransfer datatransfer.Manager
 type StagingDAG format.DAGService
 type StagingBlockstore blockstore.Blockstore
 type StagingGraphsync graphsync.GraphExchange
+type StagingMultiDstore *multistore.MultiStore
diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go
index 1a22fdd95..ba0178e10 100644
--- a/node/modules/storageminer.go
+++ b/node/modules/storageminer.go
@@ -39,6 +39,7 @@ import (
 	"github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask"
 	smnet "github.com/filecoin-project/go-fil-markets/storagemarket/network"
 	"github.com/filecoin-project/go-jsonrpc/auth"
+	"github.com/filecoin-project/go-multistore"
 	paramfetch "github.com/filecoin-project/go-paramfetch"
 	"github.com/filecoin-project/go-statestore"
 	"github.com/filecoin-project/go-storedcounter"
@@ -244,6 +245,26 @@ func NewProviderPieceStore(ds dtypes.MetadataDS) dtypes.ProviderPieceStore {
 	return piecestore.NewPieceStore(namespace.Wrap(ds, datastore.NewKey("/storagemarket")))
 }
 
+func StagingMultiDatastore(lc fx.Lifecycle, r repo.LockedRepo) (dtypes.StagingMultiDstore, error) {
+	ds, err := r.Datastore("/staging")
+	if err != nil {
+		return nil, xerrors.Errorf("getting datastore out of reop: %w", err)
+	}
+
+	mds, err := multistore.NewMultiDstore(ds)
+	if err != nil {
+		return nil, err
+	}
+
+	lc.Append(fx.Hook{
+		OnStop: func(ctx context.Context) error {
+			return mds.Close()
+		},
+	})
+
+	return mds, nil
+}
+
 // StagingBlockstore creates a blockstore for staging blocks for a miner
 // in a storage deal, prior to sealing
 func StagingBlockstore(r repo.LockedRepo) (dtypes.StagingBlockstore, error) {
@@ -339,7 +360,7 @@ func StorageProvider(minerAddress dtypes.MinerAddress,
 	ffiConfig *ffiwrapper.Config,
 	storedAsk *storedask.StoredAsk,
 	h host.Host, ds dtypes.MetadataDS,
-	ibs dtypes.StagingBlockstore,
+	mds dtypes.StagingMultiDstore,
 	r repo.LockedRepo,
 	pieceStore dtypes.ProviderPieceStore,
 	dataTransfer dtypes.ProviderDataTransfer,
@@ -406,7 +427,7 @@ func StorageProvider(minerAddress dtypes.MinerAddress,
 		return true, "", nil
 	})
 
-	p, err := storageimpl.NewProvider(net, namespace.Wrap(ds, datastore.NewKey("/deals/provider")), ibs, store, pieceStore, dataTransfer, spn, address.Address(minerAddress), ffiConfig.SealProofType, storedAsk, opt)
+	p, err := storageimpl.NewProvider(net, namespace.Wrap(ds, datastore.NewKey("/deals/provider")), store, mds, pieceStore, dataTransfer, spn, address.Address(minerAddress), ffiConfig.SealProofType, storedAsk, opt)
 	if err != nil {
 		return p, err
 	}
@@ -415,7 +436,7 @@ func StorageProvider(minerAddress dtypes.MinerAddress,
 }
 
 // RetrievalProvider creates a new retrieval provider attached to the provider blockstore
-func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.SectorManager, full lapi.FullNode, ds dtypes.MetadataDS, pieceStore dtypes.ProviderPieceStore, ibs dtypes.StagingBlockstore, dt dtypes.ProviderDataTransfer, onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) (retrievalmarket.RetrievalProvider, error) {
+func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.SectorManager, full lapi.FullNode, ds dtypes.MetadataDS, pieceStore dtypes.ProviderPieceStore, mds dtypes.StagingMultiDstore, dt dtypes.ProviderDataTransfer, onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) (retrievalmarket.RetrievalProvider, error) {
 	adapter := retrievaladapter.NewRetrievalProviderNode(miner, sealer, full)
 
 	maddr, err := minerAddrFromDS(ds)
@@ -448,7 +469,7 @@ func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.S
 		return true, "", nil
 	})
 
-	return retrievalimpl.NewProvider(maddr, adapter, netwk, pieceStore, ibs, dt, namespace.Wrap(ds, datastore.NewKey("/retrievals/provider")), opt)
+	return retrievalimpl.NewProvider(maddr, adapter, netwk, pieceStore, mds, dt, namespace.Wrap(ds, datastore.NewKey("/retrievals/provider")), opt)
 }
 
 func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStorage, si stores.SectorIndex, cfg *ffiwrapper.Config, sc sectorstorage.SealerConfig, urls sectorstorage.URLs, sa sectorstorage.StorageAuth) (*sectorstorage.Manager, error) {
diff --git a/node/repo/importmgr/mgr.go b/node/repo/importmgr/mgr.go
index 754f41df2..3deb75b70 100644
--- a/node/repo/importmgr/mgr.go
+++ b/node/repo/importmgr/mgr.go
@@ -42,7 +42,7 @@ type StoreMeta struct {
 	Labels map[string]string
 }
 
-func (m *Mgr) NewStore() (int, *multistore.Store, error) {
+func (m *Mgr) NewStore() (multistore.StoreID, *multistore.Store, error) {
 	id := m.mds.Next()
 	st, err := m.mds.Get(id)
 	if err != nil {
@@ -60,7 +60,7 @@ func (m *Mgr) NewStore() (int, *multistore.Store, error) {
 	return id, st, err
 }
 
-func (m *Mgr) AddLabel(id int, key, value string) error { // source, file path, data CID..
+func (m *Mgr) AddLabel(id multistore.StoreID, key, value string) error { // source, file path, data CID..
 	meta, err := m.ds.Get(datastore.NewKey(fmt.Sprintf("%d", id)))
 	if err != nil {
 		return xerrors.Errorf("getting metadata form datastore: %w", err)
@@ -81,11 +81,11 @@ func (m *Mgr) AddLabel(id int, key, value string) error { // source, file path,
 	return m.ds.Put(datastore.NewKey(fmt.Sprintf("%d", id)), meta)
 }
 
-func (m *Mgr) List() []int {
+func (m *Mgr) List() []multistore.StoreID {
 	return m.mds.List()
 }
 
-func (m *Mgr) Info(id int) (*StoreMeta, error) {
+func (m *Mgr) Info(id multistore.StoreID) (*StoreMeta, error) {
 	meta, err := m.ds.Get(datastore.NewKey(fmt.Sprintf("%d", id)))
 	if err != nil {
 		return nil, xerrors.Errorf("getting metadata form datastore: %w", err)
@@ -99,7 +99,7 @@ func (m *Mgr) Info(id int) (*StoreMeta, error) {
 	return &sm, nil
 }
 
-func (m *Mgr) Remove(id int) error {
+func (m *Mgr) Remove(id multistore.StoreID) error {
 	if err := m.mds.Delete(id); err != nil {
 		return xerrors.Errorf("removing import: %w", err)
 	}

From 86bfca696ad2b4389eac52d0813f6c61e97e02f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 28 Jul 2020 13:11:53 +0200
Subject: [PATCH 34/37] Use markets master

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 129abd241..205b72335 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,7 @@ require (
 	github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
 	github.com/filecoin-project/go-data-transfer v0.5.1
 	github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f
-	github.com/filecoin-project/go-fil-markets v0.5.2-0.20200728055441-cf6b0a74e50c
+	github.com/filecoin-project/go-fil-markets v0.5.2-0.20200728110539-caae3fda6623
 	github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24
 	github.com/filecoin-project/go-multistore v0.0.2
 	github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261
diff --git a/go.sum b/go.sum
index f5f6c00eb..fc01bb3ea 100644
--- a/go.sum
+++ b/go.sum
@@ -244,8 +244,8 @@ github.com/filecoin-project/go-data-transfer v0.5.1/go.mod h1:PRs78hp9u8T4G2Jce5
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
-github.com/filecoin-project/go-fil-markets v0.5.2-0.20200728055441-cf6b0a74e50c h1:cIoO/T2DxLDpPjIp1cvtreTlq8YBzIbLPk/MpVOrkrw=
-github.com/filecoin-project/go-fil-markets v0.5.2-0.20200728055441-cf6b0a74e50c/go.mod h1:tBjxNhgi3djW9RZacV3D/HhADZe+tgvfCeMyWWEgVlI=
+github.com/filecoin-project/go-fil-markets v0.5.2-0.20200728110539-caae3fda6623 h1:LuX25+rzM56XX+M4413ngRA+RAhm88vlFbsvbVtpcDc=
+github.com/filecoin-project/go-fil-markets v0.5.2-0.20200728110539-caae3fda6623/go.mod h1:zDhwmUy/AS/xCJOayW7Cedff9SDuSdGIWRnGXBDjcOk=
 github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms=
 github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM=
 github.com/filecoin-project/go-multistore v0.0.2 h1:JZEddnXXt3mMzHi7bi9IH7Yi1NpGLy19J5Lk/xbxBMs=

From eac0a1bba9c35f972a40884ec0bc23cf74d37710 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 28 Jul 2020 14:31:13 +0200
Subject: [PATCH 35/37] stmgr: Handle state forks with cron on null blocks
 correctly

---
 chain/stmgr/forks.go      | 21 +++++++++------------
 chain/stmgr/forks_test.go | 17 ++++++-----------
 chain/stmgr/stmgr.go      | 25 ++++++++++++++-----------
 chain/stmgr/utils.go      | 15 ++++++++++-----
 4 files changed, 39 insertions(+), 39 deletions(-)

diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go
index 7f8dc579b..c7c7526b3 100644
--- a/chain/stmgr/forks.go
+++ b/chain/stmgr/forks.go
@@ -3,23 +3,20 @@ package stmgr
 import (
 	"context"
 
+	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/ipfs/go-cid"
 )
 
-var ForksAtHeight = map[abi.ChainEpoch]func(context.Context, *StateManager, cid.Cid) (cid.Cid, error){}
+var ForksAtHeight = map[abi.ChainEpoch]func(context.Context, *StateManager, types.StateTree) error{}
 
-func (sm *StateManager) handleStateForks(ctx context.Context, pstate cid.Cid, height, parentH abi.ChainEpoch) (_ cid.Cid, err error) {
-	for i := parentH; i < height; i++ {
-		f, ok := ForksAtHeight[i]
-		if ok {
-			nstate, err := f(ctx, sm, pstate)
-			if err != nil {
-				return cid.Undef, err
-			}
-			pstate = nstate
+func (sm *StateManager) handleStateForks(ctx context.Context, st types.StateTree, height abi.ChainEpoch) (err error) {
+	f, ok := ForksAtHeight[height]
+	if ok {
+		err := f(ctx, sm, st)
+		if err != nil {
+			return err
 		}
 	}
 
-	return pstate, nil
+	return nil
 }
diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go
index 0fc28f92c..fd597fe8d 100644
--- a/chain/stmgr/forks_test.go
+++ b/chain/stmgr/forks_test.go
@@ -21,7 +21,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/aerrors"
 	"github.com/filecoin-project/lotus/chain/gen"
-	"github.com/filecoin-project/lotus/chain/state"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	. "github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -122,37 +121,33 @@ func TestForkHeightTriggers(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	stmgr.ForksAtHeight[testForkHeight] = func(ctx context.Context, sm *StateManager, pstate cid.Cid) (cid.Cid, error) {
+	stmgr.ForksAtHeight[testForkHeight] = func(ctx context.Context, sm *StateManager, st types.StateTree) error {
 		cst := cbor.NewCborStore(sm.ChainStore().Blockstore())
-		st, err := state.LoadStateTree(cst, pstate)
-		if err != nil {
-			return cid.Undef, err
-		}
 
 		act, err := st.GetActor(taddr)
 		if err != nil {
-			return cid.Undef, err
+			return err
 		}
 
 		var tas testActorState
 		if err := cst.Get(ctx, act.Head, &tas); err != nil {
-			return cid.Undef, xerrors.Errorf("in fork handler, failed to run get: %w", err)
+			return xerrors.Errorf("in fork handler, failed to run get: %w", err)
 		}
 
 		tas.HasUpgraded = 55
 
 		ns, err := cst.Put(ctx, &tas)
 		if err != nil {
-			return cid.Undef, err
+			return err
 		}
 
 		act.Head = ns
 
 		if err := st.SetActor(taddr, act); err != nil {
-			return cid.Undef, err
+			return err
 		}
 
-		return st.Flush(ctx)
+		return nil
 	}
 
 	inv.Register(builtin.PaymentChannelActorCodeID, &testActor{}, &testActorState{})
diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go
index bc9c6b846..b241e2236 100644
--- a/chain/stmgr/stmgr.go
+++ b/chain/stmgr/stmgr.go
@@ -147,7 +147,7 @@ type BlockMessages struct {
 type ExecCallback func(cid.Cid, *types.Message, *vm.ApplyRet) error
 
 func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEpoch, pstate cid.Cid, bms []BlockMessages, epoch abi.ChainEpoch, r vm.Rand, cb ExecCallback) (cid.Cid, cid.Cid, error) {
-	vmi, err := sm.newVM(pstate, parentEpoch+1, r, sm.cs.Blockstore(), sm.cs.VMSys())
+	vmi, err := sm.newVM(pstate, parentEpoch, r, sm.cs.Blockstore(), sm.cs.VMSys())
 	if err != nil {
 		return cid.Undef, cid.Undef, xerrors.Errorf("instantiating VM failed: %w", err)
 	}
@@ -185,10 +185,18 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
 		return nil
 	}
 
-	// run cron for null rounds if any
-	for i := parentEpoch + 1; i < epoch; i++ {
-		if err := runCron(); err != nil {
-			return cid.Cid{}, cid.Cid{}, err
+	for i := parentEpoch; i < epoch; i++ {
+		// handle state forks
+		err = sm.handleStateForks(ctx, vmi.StateTree(), i)
+		if err != nil {
+			return cid.Undef, cid.Undef, xerrors.Errorf("error handling state forks: %w", err)
+		}
+
+		if i > parentEpoch {
+			// run cron for null rounds if any
+			if err := runCron(); err != nil {
+				return cid.Cid{}, cid.Cid{}, err
+			}
 		}
 
 		vmi.SetBlockHeight(i + 1)
@@ -302,18 +310,13 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl
 
 	var parentEpoch abi.ChainEpoch
 	pstate := blks[0].ParentStateRoot
-	if len(blks[0].Parents) > 0 { // don't support forks on genesis
+	if len(blks[0].Parents) > 0 {
 		parent, err := sm.cs.GetBlock(blks[0].Parents[0])
 		if err != nil {
 			return cid.Undef, cid.Undef, xerrors.Errorf("getting parent block: %w", err)
 		}
 
 		parentEpoch = parent.Height
-
-		pstate, err = sm.handleStateForks(ctx, blks[0].ParentStateRoot, blks[0].Height, parent.Height)
-		if err != nil {
-			return cid.Undef, cid.Undef, xerrors.Errorf("error handling state forks: %w", err)
-		}
 	}
 
 	cids := make([]cid.Cid, len(blks))
diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go
index e30a245de..b5e47e2c9 100644
--- a/chain/stmgr/utils.go
+++ b/chain/stmgr/utils.go
@@ -439,15 +439,20 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
 		return cid.Undef, nil, err
 	}
 
-	fstate, err := sm.handleStateForks(ctx, base, height, ts.Height())
+	r := store.NewChainRand(sm.cs, ts.Cids(), height)
+	vmi, err := vm.NewVM(base, height, r, sm.cs.Blockstore(), sm.cs.VMSys())
 	if err != nil {
 		return cid.Undef, nil, err
 	}
 
-	r := store.NewChainRand(sm.cs, ts.Cids(), height)
-	vmi, err := vm.NewVM(fstate, height, r, sm.cs.Blockstore(), sm.cs.VMSys())
-	if err != nil {
-		return cid.Undef, nil, err
+	for i := ts.Height(); i < height; i++ {
+		// handle state forks
+		err = sm.handleStateForks(ctx, vmi.StateTree(), i)
+		if err != nil {
+			return cid.Undef, nil, xerrors.Errorf("error handling state forks: %w", err)
+		}
+
+		// TODO: should we also run cron here?
 	}
 
 	for i, msg := range msgs {

From ca3651932546bfe1be11b478f6c23d80c45b4bdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 28 Jul 2020 14:32:30 +0200
Subject: [PATCH 36/37] stmgr: Bigger number for cron gas limit

---
 chain/stmgr/stmgr.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go
index b241e2236..171483f9d 100644
--- a/chain/stmgr/stmgr.go
+++ b/chain/stmgr/stmgr.go
@@ -165,7 +165,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
 			Nonce:    ca.Nonce,
 			Value:    types.NewInt(0),
 			GasPrice: types.NewInt(0),
-			GasLimit: build.BlockGasLimit * 10, // Make super sure this is never too little
+			GasLimit: build.BlockGasLimit * 10000, // Make super sure this is never too little
 			Method:   builtin.MethodsCron.EpochTick,
 			Params:   nil,
 		}

From 09eb3731f25ac083670521e92915c02e06fb7305 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 28 Jul 2020 15:38:13 +0200
Subject: [PATCH 37/37] Fix duplicate vm setup in genesis

---
 chain/gen/genesis/genesis.go | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go
index 826113071..54d4d9e2c 100644
--- a/chain/gen/genesis/genesis.go
+++ b/chain/gen/genesis/genesis.go
@@ -347,11 +347,6 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci
 		return cid.Undef, err
 	}
 
-	vm, err := vm.NewVM(stateroot, 0, &fakeRand{}, cs.Blockstore(), mkFakedSigSyscalls(cs.VMSys()))
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("failed to create NewVM: %w", err)
-	}
-
 	_, err = doExecValue(ctx, vm, builtin.VerifiedRegistryActorAddr, verifregRoot, types.NewInt(0), builtin.MethodsVerifiedRegistry.AddVerifier, mustEnc(&verifreg.AddVerifierParams{
 
 		Address:   verifier,