Lotus version 0.5.6

Fix chain import

.circleci/config.yml

@@ -113,7 +113,7 @@ jobs:
   test: &test
     description: |
       Run tests with gotestsum.
-    parameters:
+    parameters: &test-params
       executor:
         type: executor
         default: golang
@@ -161,6 +161,7 @@ jobs:
           name: go test
           environment:
             LOTUS_TEST_WINDOW_POST: << parameters.winpost-test >>
+            SKIP_CONFORMANCE: "1"
           command: |
             mkdir -p /tmp/test-reports/<< parameters.test-suite-name >>
             mkdir -p /tmp/test-artifacts
@@ -191,6 +192,63 @@ jobs:
     <<: *test
   test-window-post:
     <<: *test
+  test-conformance:
+    description: |
+      Run tests using a corpus of interoperable test vectors for Filecoin 
+      implementations to test their correctness and compliance with the Filecoin
+      specifications.
+    parameters:
+      <<: *test-params
+      vectors-branch:
+        type: string
+        default: ""
+        description: |
+          Branch on github.com/filecoin-project/test-vectors to checkout and
+          test with. If empty (the default) the commit defined by the git
+          submodule is used.
+    executor: << parameters.executor >>
+    steps:
+      - install-deps
+      - prepare
+      - run:
+          command: make deps lotus
+          no_output_timeout: 30m
+      - download-params
+      - when:
+          condition:
+            not:
+              equal: [ "", << parameters.vectors-branch >> ]
+          steps:
+            - run:
+                name: checkout vectors branch
+                command: |
+                  cd extern/test-vectors
+                  git fetch
+                  git checkout origin/<< parameters.vectors-branch >>
+            - run:
+                name: go get vectors branch
+                command: go get github.com/filecoin-project/test-vectors@<< parameters.vectors-branch >>
+      - go/install-gotestsum:
+          gobin: $HOME/.local/bin
+          version: 0.5.2
+      - run:
+          name: go test
+          environment:
+            SKIP_CONFORMANCE: "0"
+          command: |
+            mkdir -p /tmp/test-reports
+            mkdir -p /tmp/test-artifacts
+            gotestsum \
+              --format pkgname-and-test-fails \
+              --junitfile /tmp/test-reports/junit.xml \
+              -- \
+              -v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/
+            go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html
+          no_output_timeout: 30m
+      - store_test_results:
+          path: /tmp/test-reports
+      - store_artifacts:
+          path: /tmp/test-artifacts/conformance-coverage.html
 
   build-macos:
     description: build darwin lotus binary
@@ -356,6 +414,14 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+$/
+      - test-conformance:
+          test-suite-name: conformance
+          packages: "./conformance"
+      - test-conformance:
+          name: test-conformance-bleeding-edge
+          test-suite-name: conformance-bleeding-edge
+          packages: "./conformance"
+          vectors-branch: master
       - build-debug
       - build-all:
           requires:

.gitmodules

@@ -5,3 +5,6 @@
 [submodule "extern/serialization-vectors"]
 	path = extern/serialization-vectors
 	url = https://github.com/filecoin-project/serialization-vectors
+[submodule "extern/test-vectors"]
+	path = extern/test-vectors
+	url = https://github.com/filecoin-project/test-vectors.git

CHANGELOG.md

@@ -1,5 +1,67 @@
 # Lotus changelog
 
+# 0.5.6 / 2020-08-29
+
+Hotfix release that fixes a panic in the sealing scheduler (https://github.com/filecoin-project/lotus/pull/3389).
+
+# 0.5.5
+
+This patch release introduces a large number of improvements to the sealing process.
+It also updates go-fil-markets to 
+[version 0.5.8](https://github.com/filecoin-project/go-fil-markets/releases/tag/v0.5.8),
+and go-libp2p-pubsub to [v0.3.5](https://github.com/libp2p/go-libp2p-pubsub/releases/tag/v0.3.5).
+
+#### Downstream upgrades
+
+- Upgrades markets to v0.5.8 (https://github.com/filecoin-project/lotus/pull/3384)
+- Upgrades go-libp2p-pubsub to v0.3.5 (https://github.com/filecoin-project/lotus/pull/3305)
+
+#### Sector sealing
+
+- The following improvements were introduced in https://github.com/filecoin-project/lotus/pull/3350.
+
+    - Allow `lotus-miner sectors remove` to remove a sector in any state.
+    - Create a separate state in the storage FSM dedicated to submitting the Commit message.
+    - Recovery for when the Deal IDs of deals in a sector get changed in a reorg.
+    - Auto-retry sending Precommit and Commit messages if they run out of gas
+    - Auto-retry sector remove tasks when they fail
+    - Compact worker windows, and allow their tasks to be executed in any order
+
+- Don't simply skip PoSt for bad sectors (https://github.com/filecoin-project/lotus/pull/3323)
+
+#### Message Pool 
+
+- Spam Protection: Track required funds for pending messages (https://github.com/filecoin-project/lotus/pull/3313)
+
+#### Chainwatch
+
+- Add more power and reward metrics (https://github.com/filecoin-project/lotus/pull/3367)
+- Fix raciness in sector deal table (https://github.com/filecoin-project/lotus/pull/3275)
+- Parallelize miner processing (https://github.com/filecoin-project/lotus/pull/3380)
+- Accept Lotus API and token (https://github.com/filecoin-project/lotus/pull/3337)
+
+# 0.5.4
+
+A patch release, containing a few nice bugfixes and improvements:
+
+- Fix parsing of peer ID in `lotus-miner actor set-peer-id` (@whyrusleeping)
+- Update dependencies, fixing several bugs (@Stebalien)
+- Fix remaining linter warnings (@Stebalien)
+- Use safe string truncation (@Ingar)
+- Allow tweaking of blocksync message window size (@whyrusleeping)
+- Add some additional gas stats to metrics (@Kubuxu)
+- Fix an edge case bug in message selection, add many tests (@vyzo)
+
+# 0.5.3
+
+Yet another hotfix release. 
+A lesson for readers, having people who have been awake for 12+ hours review
+your hotfix PR is not a good idea. Find someone who has enough slept recently
+enough to give you good code review, otherwise you'll end up quickly bumping
+versions again.
+
+- Fixed a bug in the mempool that was introduced in v0.5.2
+
 # 0.5.2 / 2020-08-24
 
 This is a hotfix release.

README.md

@@ -40,9 +40,6 @@ All work is tracked via issues. An attempt at keeping an up-to-date view on rema
 The lotus Filecoin implementation unfolds into the following packages:
 
 - [This repo](https://github.com/filecoin-project/lotus)
-- [storage-fsm](https://github.com/filecoin-project/storage-fsm)
-- [sector-storage](https://github.com/filecoin-project/sector-storage)
-- [specs-storage](https://github.com/filecoin-project/specs-storage)
 - [go-fil-markets](https://github.com/filecoin-project/go-fil-markets) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/markets-shared-components-5daa144a7046a60001c6e253/board)
 - [spec-actors](https://github.com/filecoin-project/specs-actors) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/actors-5ee6f3aa87591f0016c05685/board)
 

api/api_full.go

@@ -243,6 +243,8 @@ type FullNode interface {
 	ClientGetDealInfo(context.Context, cid.Cid) (*DealInfo, error)
 	// ClientListDeals returns information about the deals made by the local client.
 	ClientListDeals(ctx context.Context) ([]DealInfo, error)
+	// ClientGetDealUpdates returns the status of updated deals
+	ClientGetDealUpdates(ctx context.Context) (<-chan DealInfo, error)
 	// ClientHasLocal indicates whether a certain CID is locally stored.
 	ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error)
 	// ClientFindData identifies peers that have a certain file, and returns QueryOffers (one per peer).
@@ -323,7 +325,7 @@ type FullNode interface {
 	StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)
 	// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector
 	StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error)
-	// StateSectorGetInfo returns the on-chain info for the specified miner's sector
+	// StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found
 	// NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate
 	// expiration epoch
 	StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error)
@@ -427,7 +429,7 @@ type FullNode interface {
 	PaychVoucherCreate(context.Context, address.Address, types.BigInt, uint64) (*paych.SignedVoucher, error)
 	PaychVoucherAdd(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error)
 	PaychVoucherList(context.Context, address.Address) ([]*paych.SignedVoucher, error)
-	PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher) (cid.Cid, error)
+	PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error)
 }
 
 type FileRef struct {

api/api_storage.go

@@ -120,16 +120,18 @@ type SectorLog struct {
 }
 
 type SectorInfo struct {
-	SectorID  abi.SectorNumber
-	State     SectorState
-	CommD     *cid.Cid
-	CommR     *cid.Cid
-	Proof     []byte
-	Deals     []abi.DealID
-	Ticket    SealTicket
-	Seed      SealSeed
-	Retries   uint64
-	ToUpgrade bool
+	SectorID     abi.SectorNumber
+	State        SectorState
+	CommD        *cid.Cid
+	CommR        *cid.Cid
+	Proof        []byte
+	Deals        []abi.DealID
+	Ticket       SealTicket
+	Seed         SealSeed
+	PreCommitMsg *cid.Cid
+	CommitMsg    *cid.Cid
+	Retries      uint64
+	ToUpgrade    bool
 
 	LastErr string
 

api/apistruct/struct.go

@@ -139,6 +139,7 @@ type FullNodeStruct struct {
 		ClientStartDeal           func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error)                                          `perm:"admin"`
 		ClientGetDealInfo         func(context.Context, cid.Cid) (*api.DealInfo, error)                                                             `perm:"read"`
 		ClientListDeals           func(ctx context.Context) ([]api.DealInfo, error)                                                                 `perm:"write"`
+		ClientGetDealUpdates      func(ctx context.Context) (<-chan api.DealInfo, error)                                                            `perm:"read"`
 		ClientRetrieve            func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error                                       `perm:"admin"`
 		ClientRetrieveWithEvents  func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) `perm:"admin"`
 		ClientQueryAsk            func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error)              `perm:"read"`
@@ -214,7 +215,7 @@ type FullNodeStruct struct {
 		PaychVoucherAdd            func(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error)  `perm:"write"`
 		PaychVoucherCreate         func(context.Context, address.Address, big.Int, uint64) (*paych.SignedVoucher, error)                     `perm:"sign"`
 		PaychVoucherList           func(context.Context, address.Address) ([]*paych.SignedVoucher, error)                                    `perm:"write"`
-		PaychVoucherSubmit         func(context.Context, address.Address, *paych.SignedVoucher) (cid.Cid, error)                             `perm:"sign"`
+		PaychVoucherSubmit         func(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error)             `perm:"sign"`
 	}
 }
 
@@ -433,6 +434,10 @@ func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]api.DealInfo, e
 	return c.Internal.ClientListDeals(ctx)
 }
 
+func (c *FullNodeStruct) ClientGetDealUpdates(ctx context.Context) (<-chan api.DealInfo, error) {
+	return c.Internal.ClientGetDealUpdates(ctx)
+}
+
 func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error {
 	return c.Internal.ClientRetrieve(ctx, order, ref)
 }
@@ -920,8 +925,8 @@ func (c *FullNodeStruct) PaychNewPayment(ctx context.Context, from, to address.A
 	return c.Internal.PaychNewPayment(ctx, from, to, vouchers)
 }
 
-func (c *FullNodeStruct) PaychVoucherSubmit(ctx context.Context, ch address.Address, sv *paych.SignedVoucher) (cid.Cid, error) {
-	return c.Internal.PaychVoucherSubmit(ctx, ch, sv)
+func (c *FullNodeStruct) PaychVoucherSubmit(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) {
+	return c.Internal.PaychVoucherSubmit(ctx, ch, sv, secret, proof)
 }
 
 // StorageMinerStruct

api/client/client.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"context"
 	"net/http"
 	"net/url"
 	"path"
@@ -14,9 +15,9 @@ import (
 )
 
 // NewCommonRPC creates a new http jsonrpc client.
-func NewCommonRPC(addr string, requestHeader http.Header) (api.Common, jsonrpc.ClientCloser, error) {
+func NewCommonRPC(ctx context.Context, addr string, requestHeader http.Header) (api.Common, jsonrpc.ClientCloser, error) {
 	var res apistruct.CommonStruct
-	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
 		[]interface{}{
 			&res.Internal,
 		},
@@ -27,9 +28,9 @@ func NewCommonRPC(addr string, requestHeader http.Header) (api.Common, jsonrpc.C
 }
 
 // NewFullNodeRPC creates a new http jsonrpc client.
-func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
+func NewFullNodeRPC(ctx context.Context, addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
 	var res apistruct.FullNodeStruct
-	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
 		[]interface{}{
 			&res.CommonStruct.Internal,
 			&res.Internal,
@@ -39,9 +40,9 @@ func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonr
 }
 
 // NewStorageMinerRPC creates a new http jsonrpc client for miner
-func NewStorageMinerRPC(addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.StorageMiner, jsonrpc.ClientCloser, error) {
+func NewStorageMinerRPC(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.StorageMiner, jsonrpc.ClientCloser, error) {
 	var res apistruct.StorageMinerStruct
-	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
 		[]interface{}{
 			&res.CommonStruct.Internal,
 			&res.Internal,
@@ -53,7 +54,7 @@ func NewStorageMinerRPC(addr string, requestHeader http.Header, opts ...jsonrpc.
 	return &res, closer, err
 }
 
-func NewWorkerRPC(addr string, requestHeader http.Header) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
+func NewWorkerRPC(ctx context.Context, addr string, requestHeader http.Header) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
 	u, err := url.Parse(addr)
 	if err != nil {
 		return nil, nil, err
@@ -69,7 +70,7 @@ func NewWorkerRPC(addr string, requestHeader http.Header) (api.WorkerAPI, jsonrp
 	u.Path = path.Join(u.Path, "../streams/v0/push")
 
 	var res apistruct.WorkerStruct
-	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
 		[]interface{}{
 			&res.Internal,
 		},

api/test/blockminer.go

@@ -41,7 +41,7 @@ func (bm *BlockMiner) MineBlocks() {
 			nulls := atomic.SwapInt64(&bm.nulls, 0)
 			if err := bm.miner.MineOne(bm.ctx, miner.MineReq{
 				InjectNulls: abi.ChainEpoch(nulls),
-				Done:        func(bool, error) {},
+				Done:        func(bool, abi.ChainEpoch, error) {},
 			}); err != nil {
 				bm.t.Error(err)
 			}

api/test/deals.go

@@ -24,6 +24,7 @@ import (
 	"github.com/filecoin-project/lotus/build"
 	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
 	"github.com/filecoin-project/lotus/miner"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	dag "github.com/ipfs/go-merkledag"
 	dstest "github.com/ipfs/go-merkledag/test"
 	unixfile "github.com/ipfs/go-unixfs/file"
@@ -35,7 +36,7 @@ import (
 
 var MineNext = miner.MineReq{
 	InjectNulls: 0,
-	Done:        func(bool, error) {},
+	Done:        func(bool, abi.ChainEpoch, error) {},
 }
 
 func init() {

api/test/mining.go

@@ -31,22 +31,20 @@ func (ts *testSuite) testMining(t *testing.T) {
 	newHeads, err := api.ChainNotify(ctx)
 	require.NoError(t, err)
 	initHead := (<-newHeads)[0]
-	if initHead.Val.Height() != 2 {
-		<-newHeads
-	}
+	baseHeight := initHead.Val.Height()
 
 	h1, err := api.ChainHead(ctx)
 	require.NoError(t, err)
-	require.Equal(t, abi.ChainEpoch(2), h1.Height())
+	require.Equal(t, int64(h1.Height()), int64(baseHeight))
 
-	err = sn[0].MineOne(ctx, MineNext)
+	MineUntilBlock(ctx, t, apis[0], sn[0], nil)
 	require.NoError(t, err)
 
 	<-newHeads
 
 	h2, err := api.ChainHead(ctx)
 	require.NoError(t, err)
-	require.Equal(t, abi.ChainEpoch(3), h2.Height())
+	require.Greater(t, int64(h2.Height()), int64(h1.Height()))
 }
 
 func (ts *testSuite) testMiningReal(t *testing.T) {
@@ -70,7 +68,7 @@ func (ts *testSuite) testMiningReal(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, abi.ChainEpoch(2), h1.Height())
 
-	err = sn[0].MineOne(ctx, MineNext)
+	MineUntilBlock(ctx, t, apis[0], sn[0], nil)
 	require.NoError(t, err)
 
 	<-newHeads
@@ -79,7 +77,7 @@ func (ts *testSuite) testMiningReal(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, abi.ChainEpoch(3), h2.Height())
 
-	err = sn[0].MineOne(ctx, MineNext)
+	MineUntilBlock(ctx, t, apis[0], sn[0], nil)
 	require.NoError(t, err)
 
 	<-newHeads
@@ -144,7 +142,7 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo
 		complChan := minedTwo
 		for atomic.LoadInt32(&mine) != 0 {
 			wait := make(chan int)
-			mdone := func(mined bool, err error) {
+			mdone := func(mined bool, _ abi.ChainEpoch, err error) {
 				n := 0
 				if mined {
 					n = 1

api/test/util.go

@@ -3,11 +3,13 @@ package test
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/filecoin-project/specs-actors/actors/abi"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/miner"
 )
 
 func SendFunds(ctx context.Context, t *testing.T, sender TestNode, addr address.Address, amount abi.TokenAmount) {
@@ -34,3 +36,51 @@ func SendFunds(ctx context.Context, t *testing.T, sender TestNode, addr address.
 		t.Fatal("did not successfully send money")
 	}
 }
+
+func MineUntilBlock(ctx context.Context, t *testing.T, fn TestNode, sn TestStorageNode, cb func(abi.ChainEpoch)) {
+	for i := 0; i < 1000; i++ {
+		var success bool
+		var err error
+		var epoch abi.ChainEpoch
+		wait := make(chan struct{})
+		mineErr := sn.MineOne(ctx, miner.MineReq{
+			Done: func(win bool, ep abi.ChainEpoch, e error) {
+				success = win
+				err = e
+				epoch = ep
+				wait <- struct{}{}
+			},
+		})
+		if mineErr != nil {
+			t.Fatal(mineErr)
+		}
+		<-wait
+		if err != nil {
+			t.Fatal(err)
+		}
+		if success {
+			// Wait until it shows up on the given full nodes ChainHead
+			nloops := 50
+			for i := 0; i < nloops; i++ {
+				ts, err := fn.ChainHead(ctx)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if ts.Height() == epoch {
+					break
+				}
+				if i == nloops-1 {
+					t.Fatal("block never managed to sync to node")
+				}
+				time.Sleep(time.Millisecond * 10)
+			}
+
+			if cb != nil {
+				cb(epoch)
+			}
+			return
+		}
+		t.Log("did not mine block, trying again", i)
+	}
+	t.Fatal("failed to mine 1000 times in a row...")
+}

api/test/window_post.go

@@ -53,7 +53,7 @@ func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSect
 		defer close(done)
 		for mine {
 			build.Clock.Sleep(blocktime)
-			if err := sn[0].MineOne(ctx, bminer.MineReq{Done: func(bool, error) {
+			if err := sn[0].MineOne(ctx, bminer.MineReq{Done: func(bool, abi.ChainEpoch, error) {
 
 			}}); err != nil {
 				t.Error(err)
@@ -216,6 +216,10 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 		sn, err := parts[0].Sectors.First()
 		require.NoError(t, err)
 
+		all, err := parts[0].Sectors.All(2)
+		require.NoError(t, err)
+		fmt.Println("the sectors", all)
+
 		s = abi.SectorID{
 			Miner:  abi.ActorID(mid),
 			Number: abi.SectorNumber(sn),

build/version.go

@@ -25,7 +25,7 @@ func buildType() string {
 }
 
 // BuildVersion is the local build version, set by build system
-const BuildVersion = "0.5.2"
+const BuildVersion = "0.5.6"
 
 func UserVersion() string {
 	return BuildVersion + buildType() + CurrentCommit
@@ -53,7 +53,7 @@ func (ve Version) EqMajorMinor(v2 Version) bool {
 }
 
 // APIVersion is a semver version of the rpc api exposed
-var APIVersion Version = newVer(0, 12, 0)
+var APIVersion Version = newVer(0, 13, 0)
 
 //nolint:varcheck,deadcode
 const (

chain/beacon/beacon.go

@@ -37,6 +37,10 @@ func ValidateBlockValues(b RandomBeacon, h *types.BlockHeader, prevEntry types.B
 		return nil
 	}
 
+	if len(h.BeaconEntries) == 0 {
+		return xerrors.Errorf("expected to have beacon entries in this block, but didn't find any")
+	}
+
 	last := h.BeaconEntries[len(h.BeaconEntries)-1]
 	if last.Round != maxRound {
 		return xerrors.Errorf("expected final beacon entry in block to be at round %d, got %d", maxRound, last.Round)

chain/blocksync/client.go

@@ -308,6 +308,12 @@ func (client *BlockSync) GetChainMessages(
 	length uint64,
 ) ([]*CompactedMessages, error) {
 	ctx, span := trace.StartSpan(ctx, "GetChainMessages")
+	if span.IsRecordingEvents() {
+		span.AddAttributes(
+			trace.StringAttribute("tipset", fmt.Sprint(head.Cids())),
+			trace.Int64Attribute("count", int64(length)),
+		)
+	}
 	defer span.End()
 
 	req := &Request{

chain/messagepool/messagepool.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	stdbig "math/big"
 	"sort"
 	"sync"
 	"time"
@@ -38,7 +39,7 @@ import (
 
 var log = logging.Logger("messagepool")
 
-const futureDebug = false
+var futureDebug = false
 
 var rbfNumBig = types.NewInt(uint64((ReplaceByFeeRatioDefault - 1) * RbfDenom))
 var rbfDenomBig = types.NewInt(RbfDenom)
@@ -47,6 +48,10 @@ const RbfDenom = 256
 
 var RepublishInterval = pubsub.TimeCacheDuration + time.Duration(5*build.BlockDelaySecs+build.PropagationDelaySecs)*time.Second
 
+var minimumBaseFee = types.NewInt(uint64(build.MinimumBaseFee))
+
+var MaxActorPendingMessages = 1000
+
 var (
 	ErrMessageTooBig = errors.New("message too big")
 
@@ -54,12 +59,15 @@ var (
 
 	ErrNonceTooLow = errors.New("message nonce too low")
 
+	ErrGasFeeCapTooLow = errors.New("gas fee cap too low")
+
 	ErrNotEnoughFunds = errors.New("not enough funds to execute transaction")
 
 	ErrInvalidToAddr = errors.New("message had invalid to address")
 
-	ErrBroadcastAnyway  = errors.New("broadcasting message despite validation fail")
-	ErrRBFTooLowPremium = errors.New("replace by fee has too low GasPremium")
+	ErrSoftValidationFailure  = errors.New("validation failure")
+	ErrRBFTooLowPremium       = errors.New("replace by fee has too low GasPremium")
+	ErrTooManyPendingMessages = errors.New("too many pending messages for actor")
 
 	ErrTryAgain = errors.New("state inconsistency while pushing message; please try again")
 )
@@ -118,17 +126,19 @@ type MessagePool struct {
 }
 
 type msgSet struct {
-	msgs      map[uint64]*types.SignedMessage
-	nextNonce uint64
+	msgs          map[uint64]*types.SignedMessage
+	nextNonce     uint64
+	requiredFunds *stdbig.Int
 }
 
 func newMsgSet() *msgSet {
 	return &msgSet{
-		msgs: make(map[uint64]*types.SignedMessage),
+		msgs:          make(map[uint64]*types.SignedMessage),
+		requiredFunds: stdbig.NewInt(0),
 	}
 }
 
-func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool) (bool, error) {
+func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool, limit bool) (bool, error) {
 	if len(ms.msgs) == 0 || m.Message.Nonce >= ms.nextNonce {
 		ms.nextNonce = m.Message.Nonce + 1
 	}
@@ -150,12 +160,44 @@ func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool) (bool, error) {
 					ErrRBFTooLowPremium)
 			}
 		}
+
+		ms.requiredFunds.Sub(ms.requiredFunds, exms.Message.RequiredFunds().Int)
+		//ms.requiredFunds.Sub(ms.requiredFunds, exms.Message.Value.Int)
 	}
+
+	if !has && limit && len(ms.msgs) > MaxActorPendingMessages {
+		log.Errorf("too many pending messages from actor %s", m.Message.From)
+		return false, ErrTooManyPendingMessages
+	}
+
 	ms.msgs[m.Message.Nonce] = m
+	ms.requiredFunds.Add(ms.requiredFunds, m.Message.RequiredFunds().Int)
+	//ms.requiredFunds.Add(ms.requiredFunds, m.Message.Value.Int)
 
 	return !has, nil
 }
 
+func (ms *msgSet) rm(nonce uint64) {
+	m, has := ms.msgs[nonce]
+	if has {
+		ms.requiredFunds.Sub(ms.requiredFunds, m.Message.RequiredFunds().Int)
+		//ms.requiredFunds.Sub(ms.requiredFunds, m.Message.Value.Int)
+		delete(ms.msgs, nonce)
+	}
+}
+
+func (ms *msgSet) getRequiredFunds(nonce uint64) types.BigInt {
+	requiredFunds := new(stdbig.Int).Set(ms.requiredFunds)
+
+	m, has := ms.msgs[nonce]
+	if has {
+		requiredFunds.Sub(requiredFunds, m.Message.RequiredFunds().Int)
+		//requiredFunds.Sub(requiredFunds, m.Message.Value.Int)
+	}
+
+	return types.BigInt{Int: requiredFunds}
+}
+
 func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*MessagePool, error) {
 	cache, _ := lru.New2Q(build.BlsSignatureCacheSize)
 	verifcache, _ := lru.New2Q(build.VerifSigCacheSize)
@@ -257,7 +299,7 @@ func (mp *MessagePool) addLocal(m *types.SignedMessage, msgb []byte) error {
 	return nil
 }
 
-func (mp *MessagePool) verifyMsgBeforePush(m *types.SignedMessage, epoch abi.ChainEpoch) error {
+func (mp *MessagePool) verifyMsgBeforeAdd(m *types.SignedMessage, epoch abi.ChainEpoch) error {
 	minGas := vm.PricelistByEpoch(epoch).OnChainMessage(m.ChainLength())
 
 	if err := m.VMMessage().ValidForBlockInclusion(minGas.Total()); err != nil {
@@ -278,25 +320,12 @@ func (mp *MessagePool) Push(m *types.SignedMessage) (cid.Cid, error) {
 		<-mp.addSema
 	}()
 
-	mp.curTsLk.Lock()
-	curTs := mp.curTs
-	epoch := curTs.Height()
-	mp.curTsLk.Unlock()
-	if err := mp.verifyMsgBeforePush(m, epoch); err != nil {
-		return cid.Undef, err
-	}
-
 	msgb, err := m.Serialize()
 	if err != nil {
 		return cid.Undef, err
 	}
 
 	mp.curTsLk.Lock()
-	if mp.curTs != curTs {
-		mp.curTsLk.Unlock()
-		return cid.Undef, ErrTryAgain
-	}
-
 	if err := mp.addTs(m, mp.curTs); err != nil {
 		mp.curTsLk.Unlock()
 		return cid.Undef, err
@@ -319,7 +348,7 @@ func (mp *MessagePool) checkMessage(m *types.SignedMessage) error {
 		return xerrors.Errorf("mpool message too large (%dB): %w", m.Size(), ErrMessageTooBig)
 	}
 
-	// Perform syntaxtic validation, minGas=0 as we check if correctly in select messages
+	// Perform syntactic validation, minGas=0 as we check the actual mingas before we add it
 	if err := m.Message.ValidForBlockInclusion(0); err != nil {
 		return xerrors.Errorf("message not valid for block inclusion: %w", err)
 	}
@@ -332,8 +361,12 @@ func (mp *MessagePool) checkMessage(m *types.SignedMessage) error {
 		return ErrMessageValueTooHigh
 	}
 
+	if m.Message.GasFeeCap.LessThan(minimumBaseFee) {
+		return ErrGasFeeCapTooLow
+	}
+
 	if err := mp.VerifyMsgSig(m); err != nil {
-		log.Warnf("mpooladd signature verification failed: %s", err)
+		log.Warnf("signature verification failed: %s", err)
 		return err
 	}
 
@@ -393,48 +426,71 @@ func (mp *MessagePool) VerifyMsgSig(m *types.SignedMessage) error {
 	return nil
 }
 
+func (mp *MessagePool) checkBalance(m *types.SignedMessage, curTs *types.TipSet) error {
+	balance, err := mp.getStateBalance(m.Message.From, curTs)
+	if err != nil {
+		return xerrors.Errorf("failed to check sender balance: %s: %w", err, ErrSoftValidationFailure)
+	}
+
+	requiredFunds := m.Message.RequiredFunds()
+	if balance.LessThan(requiredFunds) {
+		return xerrors.Errorf("not enough funds (required: %s, balance: %s): %w", types.FIL(requiredFunds), types.FIL(balance), ErrNotEnoughFunds)
+	}
+
+	// add Value for soft failure check
+	//requiredFunds = types.BigAdd(requiredFunds, m.Message.Value)
+
+	mset, ok := mp.pending[m.Message.From]
+	if ok {
+		requiredFunds = types.BigAdd(requiredFunds, mset.getRequiredFunds(m.Message.Nonce))
+	}
+
+	if balance.LessThan(requiredFunds) {
+		// Note: we fail here for ErrSoftValidationFailure to signal a soft failure because we might
+		// be out of sync.
+		return xerrors.Errorf("not enough funds including pending messages (required: %s, balance: %s): %w", types.FIL(requiredFunds), types.FIL(balance), ErrSoftValidationFailure)
+	}
+
+	return nil
+}
+
 func (mp *MessagePool) addTs(m *types.SignedMessage, curTs *types.TipSet) error {
 	snonce, err := mp.getStateNonce(m.Message.From, curTs)
 	if err != nil {
-		return xerrors.Errorf("failed to look up actor state nonce: %s: %w", err, ErrBroadcastAnyway)
+		return xerrors.Errorf("failed to look up actor state nonce: %s: %w", err, ErrSoftValidationFailure)
 	}
 
 	if snonce > m.Message.Nonce {
 		return xerrors.Errorf("minimum expected nonce is %d: %w", snonce, ErrNonceTooLow)
 	}
 
-	balance, err := mp.getStateBalance(m.Message.From, curTs)
-	if err != nil {
-		return xerrors.Errorf("failed to check sender balance: %s: %w", err, ErrBroadcastAnyway)
-	}
-
-	if balance.LessThan(m.Message.RequiredFunds()) {
-		return xerrors.Errorf("not enough funds (required: %s, balance: %s): %w", types.FIL(m.Message.RequiredFunds()), types.FIL(balance), ErrNotEnoughFunds)
-	}
-
 	mp.lk.Lock()
 	defer mp.lk.Unlock()
 
-	return mp.addLocked(m)
+	if err := mp.verifyMsgBeforeAdd(m, curTs.Height()); err != nil {
+		return err
+	}
+
+	if err := mp.checkBalance(m, curTs); err != nil {
+		return err
+	}
+
+	return mp.addLocked(m, true)
 }
 
 func (mp *MessagePool) addSkipChecks(m *types.SignedMessage) error {
 	mp.lk.Lock()
 	defer mp.lk.Unlock()
 
-	return mp.addLocked(m)
+	return mp.addLocked(m, false)
 }
 
-func (mp *MessagePool) addLocked(m *types.SignedMessage) error {
+func (mp *MessagePool) addLocked(m *types.SignedMessage, limit bool) error {
 	log.Debugf("mpooladd: %s %d", m.Message.From, m.Message.Nonce)
 	if m.Signature.Type == crypto.SigTypeBLS {
 		mp.blsSigCache.Add(m.Cid(), m.Signature)
 	}
 
-	if m.Message.GasLimit > build.BlockGasLimit {
-		return xerrors.Errorf("given message has too high of a gas limit")
-	}
-
 	if _, err := mp.api.PutMessage(m); err != nil {
 		log.Warnf("mpooladd cs.PutMessage failed: %s", err)
 		return err
@@ -451,7 +507,7 @@ func (mp *MessagePool) addLocked(m *types.SignedMessage) error {
 		mp.pending[m.Message.From] = mset
 	}
 
-	incr, err := mset.add(m, mp)
+	incr, err := mset.add(m, mp, limit)
 	if err != nil {
 		log.Info(err)
 		return err
@@ -562,6 +618,16 @@ func (mp *MessagePool) PushWithNonce(ctx context.Context, addr address.Address,
 		return nil, err
 	}
 
+	err = mp.checkMessage(msg)
+	if err != nil {
+		return nil, err
+	}
+
+	msgb, err := msg.Serialize()
+	if err != nil {
+		return nil, err
+	}
+
 	// reacquire the locks and check state for consistency
 	mp.curTsLk.Lock()
 	defer mp.curTsLk.Unlock()
@@ -582,16 +648,15 @@ func (mp *MessagePool) PushWithNonce(ctx context.Context, addr address.Address,
 		return nil, ErrTryAgain
 	}
 
-	if err := mp.verifyMsgBeforePush(msg, mp.curTs.Height()); err != nil {
+	if err := mp.verifyMsgBeforeAdd(msg, curTs.Height()); err != nil {
 		return nil, err
 	}
 
-	msgb, err := msg.Serialize()
-	if err != nil {
+	if err := mp.checkBalance(msg, curTs); err != nil {
 		return nil, err
 	}
 
-	if err := mp.addLocked(msg); err != nil {
+	if err := mp.addLocked(msg, true); err != nil {
 		return nil, xerrors.Errorf("add locked failed: %w", err)
 	}
 	if err := mp.addLocal(msg, msgb); err != nil {
@@ -625,7 +690,7 @@ func (mp *MessagePool) remove(from address.Address, nonce uint64) {
 
 	// NB: This deletes any message with the given nonce. This makes sense
 	// as two messages with the same sender cannot have the same nonce
-	delete(mset.msgs, nonce)
+	mset.rm(nonce)
 
 	if len(mset.msgs) == 0 {
 		delete(mp.pending, from)
@@ -651,6 +716,10 @@ func (mp *MessagePool) Pending() ([]*types.SignedMessage, *types.TipSet) {
 	mp.lk.Lock()
 	defer mp.lk.Unlock()
 
+	return mp.allPending()
+}
+
+func (mp *MessagePool) allPending() ([]*types.SignedMessage, *types.TipSet) {
 	out := make([]*types.SignedMessage, 0)
 	for a := range mp.pending {
 		out = append(out, mp.pendingFor(a)...)
@@ -658,6 +727,7 @@ func (mp *MessagePool) Pending() ([]*types.SignedMessage, *types.TipSet) {
 
 	return out, mp.curTs
 }
+
 func (mp *MessagePool) PendingFor(a address.Address) ([]*types.SignedMessage, *types.TipSet) {
 	mp.curTsLk.Lock()
 	defer mp.curTsLk.Unlock()
@@ -790,7 +860,9 @@ func (mp *MessagePool) HeadChange(revert []*types.TipSet, apply []*types.TipSet)
 	}
 
 	if len(revert) > 0 && futureDebug {
-		msgs, ts := mp.Pending()
+		mp.lk.Lock()
+		msgs, ts := mp.allPending()
+		mp.lk.Unlock()
 
 		buckets := map[address.Address]*statBucket{}
 
@@ -901,8 +973,6 @@ func (mp *MessagePool) runHeadChange(from *types.TipSet, to *types.TipSet, rmsgs
 	}
 
 	for _, ts := range apply {
-		mp.curTs = ts
-
 		for _, b := range ts.Blocks() {
 			bmsgs, smsgs, err := mp.api.MessagesForBlock(b)
 			if err != nil {
@@ -975,6 +1045,7 @@ func (mp *MessagePool) Updates(ctx context.Context) (<-chan api.MpoolUpdate, err
 
 	go func() {
 		defer mp.changes.Unsub(sub, localUpdates)
+		defer close(out)
 
 		for {
 			select {
@@ -983,9 +1054,13 @@ func (mp *MessagePool) Updates(ctx context.Context) (<-chan api.MpoolUpdate, err
 				case out <- u.(api.MpoolUpdate):
 				case <-ctx.Done():
 					return
+				case <-mp.closer:
+					return
 				}
 			case <-ctx.Done():
 				return
+			case <-mp.closer:
+				return
 			}
 		}
 	}()

chain/messagepool/messagepool_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/chain/messagepool/gasguess"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/types/mock"
 	"github.com/filecoin-project/lotus/chain/wallet"
@@ -31,6 +32,8 @@ type testMpoolAPI struct {
 	balance    map[address.Address]types.BigInt
 
 	tipsets []*types.TipSet
+
+	published int
 }
 
 func newTestMpoolAPI() *testMpoolAPI {
@@ -90,10 +93,16 @@ func (tma *testMpoolAPI) PutMessage(m types.ChainMsg) (cid.Cid, error) {
 }
 
 func (tma *testMpoolAPI) PubSubPublish(string, []byte) error {
+	tma.published++
 	return nil
 }
 
 func (tma *testMpoolAPI) GetActorAfter(addr address.Address, ts *types.TipSet) (*types.Actor, error) {
+	// regression check for load bug
+	if ts == nil {
+		panic("GetActorAfter called with nil tipset")
+	}
+
 	balance, ok := tma.balance[addr]
 	if !ok {
 		balance = types.NewInt(1000e6)
@@ -281,6 +290,11 @@ func TestMessagePoolMessagesInEachBlock(t *testing.T) {
 }
 
 func TestRevertMessages(t *testing.T) {
+	futureDebug = true
+	defer func() {
+		futureDebug = false
+	}()
+
 	tma := newTestMpoolAPI()
 
 	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
@@ -359,6 +373,7 @@ func TestPruningSimple(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	tma.setBalance(sender, 1) // in FIL
 	target := mock.Address(1001)
 
 	for i := 0; i < 5; i++ {
@@ -385,3 +400,257 @@ func TestPruningSimple(t *testing.T) {
 		t.Fatal("expected only 5 messages in pool, got: ", len(msgs))
 	}
 }
+
+func TestLoadLocal(t *testing.T) {
+	tma := newTestMpoolAPI()
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(tma, ds, "mptest")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// the actors
+	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(a1, 1) // in FIL
+	tma.setBalance(a2, 1) // in FIL
+	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
+	msgs := make(map[cid.Cid]struct{})
+	for i := 0; i < 10; i++ {
+		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		cid, err := mp.Push(m)
+		if err != nil {
+			t.Fatal(err)
+		}
+		msgs[cid] = struct{}{}
+	}
+	err = mp.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mp, err = New(tma, ds, "mptest")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pmsgs, _ := mp.Pending()
+	if len(msgs) != len(pmsgs) {
+		t.Fatalf("expected %d messages, but got %d", len(msgs), len(pmsgs))
+	}
+
+	for _, m := range pmsgs {
+		cid := m.Cid()
+		_, ok := msgs[cid]
+		if !ok {
+			t.Fatal("unknown message")
+		}
+
+		delete(msgs, cid)
+	}
+
+	if len(msgs) > 0 {
+		t.Fatalf("not all messages were laoded; missing %d messages", len(msgs))
+	}
+}
+
+func TestClearAll(t *testing.T) {
+	tma := newTestMpoolAPI()
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(tma, ds, "mptest")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// the actors
+	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(a1, 1) // in FIL
+	tma.setBalance(a2, 1) // in FIL
+	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
+	for i := 0; i < 10; i++ {
+		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		_, err := mp.Push(m)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	for i := 0; i < 10; i++ {
+		m := makeTestMessage(w2, a2, a1, uint64(i), gasLimit, uint64(i+1))
+		mustAdd(t, mp, m)
+	}
+
+	mp.Clear(true)
+
+	pending, _ := mp.Pending()
+	if len(pending) > 0 {
+		t.Fatalf("cleared the mpool, but got %d pending messages", len(pending))
+	}
+}
+
+func TestClearNonLocal(t *testing.T) {
+	tma := newTestMpoolAPI()
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(tma, ds, "mptest")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// the actors
+	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(a1, 1) // in FIL
+	tma.setBalance(a2, 1) // in FIL
+
+	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
+	for i := 0; i < 10; i++ {
+		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		_, err := mp.Push(m)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	for i := 0; i < 10; i++ {
+		m := makeTestMessage(w2, a2, a1, uint64(i), gasLimit, uint64(i+1))
+		mustAdd(t, mp, m)
+	}
+
+	mp.Clear(false)
+
+	pending, _ := mp.Pending()
+	if len(pending) != 10 {
+		t.Fatalf("expected 10 pending messages, but got %d instead", len(pending))
+	}
+
+	for _, m := range pending {
+		if m.Message.From != a1 {
+			t.Fatalf("expected message from %s but got one from %s instead", a1, m.Message.From)
+		}
+	}
+}
+
+func TestUpdates(t *testing.T) {
+	tma := newTestMpoolAPI()
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(tma, ds, "mptest")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// the actors
+	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx, cancel := context.WithCancel(context.TODO())
+	defer cancel()
+
+	ch, err := mp.Updates(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
+
+	tma.setBalance(a1, 1) // in FIL
+	tma.setBalance(a2, 1) // in FIL
+
+	for i := 0; i < 10; i++ {
+		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		_, err := mp.Push(m)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		_, ok := <-ch
+		if !ok {
+			t.Fatal("expected update, but got a closed channel instead")
+		}
+	}
+
+	err = mp.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, ok := <-ch
+	if ok {
+		t.Fatal("expected closed channel, but got an update instead")
+	}
+}

chain/messagepool/repub_test.go

@@ -0,0 +1,66 @@
+package messagepool
+
+import (
+	"testing"
+	"time"
+
+	"github.com/filecoin-project/lotus/chain/messagepool/gasguess"
+	"github.com/filecoin-project/lotus/chain/wallet"
+	"github.com/filecoin-project/specs-actors/actors/builtin"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
+	"github.com/ipfs/go-datastore"
+)
+
+func TestRepubMessages(t *testing.T) {
+	tma := newTestMpoolAPI()
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(tma, ds, "mptest")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// the actors
+	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
+
+	tma.setBalance(a1, 1) // in FIL
+
+	for i := 0; i < 10; i++ {
+		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		_, err := mp.Push(m)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if tma.published != 10 {
+		t.Fatalf("expected to have published 10 messages, but got %d instead", tma.published)
+	}
+
+	mp.repubTrigger <- struct{}{}
+	time.Sleep(100 * time.Millisecond)
+
+	if tma.published != 20 {
+		t.Fatalf("expected to have published 20 messages, but got %d instead", tma.published)
+	}
+}

chain/messagepool/selection_test.go

@@ -23,6 +23,11 @@ import (
 	logging "github.com/ipfs/go-log"
 )
 
+func init() {
+	// bump this for the selection tests
+	MaxActorPendingMessages = 1000000
+}
+
 func makeTestMessage(w *wallet.Wallet, from, to address.Address, nonce uint64, gasLimit int64, gasPrice uint64) *types.SignedMessage {
 	msg := &types.Message{
 		From:       from,
@@ -649,6 +654,73 @@ func TestPriorityMessageSelection(t *testing.T) {
 	}
 }
 
+func TestPriorityMessageSelection2(t *testing.T) {
+	mp, tma := makeTestMpool()
+
+	// the actors
+	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	block := tma.nextBlock()
+	ts := mock.TipSet(block)
+	tma.applyBlock(t, block)
+
+	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
+
+	tma.setBalance(a1, 1) // in FIL
+	tma.setBalance(a2, 1) // in FIL
+
+	mp.cfg.PriorityAddrs = []address.Address{a1}
+
+	nMessages := int(2 * build.BlockGasLimit / gasLimit)
+	for i := 0; i < nMessages; i++ {
+		bias := (nMessages - i) / 3
+		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(1+i%3+bias))
+		mustAdd(t, mp, m)
+		m = makeTestMessage(w2, a2, a1, uint64(i), gasLimit, uint64(1+i%3+bias))
+		mustAdd(t, mp, m)
+	}
+
+	msgs, err := mp.SelectMessages(ts, 1.0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedMsgs := int(build.BlockGasLimit / gasLimit)
+	if len(msgs) != expectedMsgs {
+		t.Fatalf("expected %d messages but got %d", expectedMsgs, len(msgs))
+	}
+
+	// all messages must be from a1
+	nextNonce := uint64(0)
+	for _, m := range msgs {
+		if m.Message.From != a1 {
+			t.Fatal("expected messages from a1 before messages from a2")
+		}
+		if m.Message.Nonce != nextNonce {
+			t.Fatalf("expected nonce %d but got %d", nextNonce, m.Message.Nonce)
+		}
+		nextNonce++
+	}
+}
+
 func TestOptimalMessageSelection1(t *testing.T) {
 	// this test uses just a single actor sending messages with a low tq
 	// the chain depenent merging algorithm should pick messages from the actor

chain/stmgr/utils.go

@@ -157,7 +157,7 @@ func MinerSectorInfo(ctx context.Context, sm *StateManager, maddr address.Addres
 		return nil, err
 	}
 	if !ok {
-		return nil, xerrors.New("sector not found")
+		return nil, nil
 	}
 
 	return sectorInfo, nil

chain/sub/incoming.go

@@ -3,6 +3,7 @@ package sub
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"sync"
 	"time"
@@ -40,6 +41,9 @@ import (
 
 var log = logging.Logger("sub")
 
+var ErrSoftFailure = errors.New("soft validation failure")
+var ErrInsufficientPower = errors.New("incoming block's miner does not have minimum power")
+
 func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *chain.Syncer, bserv bserv.BlockService, cmgr connmgr.ConnManager) {
 	for {
 		msg, err := bsub.Next(ctx)
@@ -258,16 +262,15 @@ func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub
 
 	stats.Record(ctx, metrics.BlockReceived.M(1))
 
-	recordFailure := func(what string) {
-		ctx, _ = tag.New(ctx, tag.Insert(metrics.FailureType, what))
-		stats.Record(ctx, metrics.BlockValidationFailure.M(1))
+	recordFailureFlagPeer := func(what string) {
+		recordFailure(ctx, metrics.BlockValidationFailure, what)
 		bv.flagPeer(pid)
 	}
 
 	blk, what, err := bv.decodeAndCheckBlock(msg)
 	if err != nil {
 		log.Error("got invalid block over pubsub: ", err)
-		recordFailure(what)
+		recordFailureFlagPeer(what)
 		return pubsub.ValidationReject
 	}
 
@@ -275,7 +278,7 @@ func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub
 	err = bv.validateMsgMeta(ctx, blk)
 	if err != nil {
 		log.Warnf("error validating message metadata: %s", err)
-		recordFailure("invalid_block_meta")
+		recordFailureFlagPeer("invalid_block_meta")
 		return pubsub.ValidationReject
 	}
 
@@ -288,11 +291,12 @@ func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub
 	// if we are synced and the miner is unknown, then the block is rejcected.
 	key, err := bv.checkPowerAndGetWorkerKey(ctx, blk.Header)
 	if err != nil {
-		if bv.isChainNearSynced() {
+		if err != ErrSoftFailure && bv.isChainNearSynced() {
 			log.Warnf("received block from unknown miner or miner that doesn't meet min power over pubsub; rejecting message")
-			recordFailure("unknown_miner")
+			recordFailureFlagPeer("unknown_miner")
 			return pubsub.ValidationReject
 		}
+
 		log.Warnf("cannot validate block message; unknown miner or miner that doesn't meet min power in unsynced chain")
 		return pubsub.ValidationIgnore
 	}
@@ -300,13 +304,13 @@ func (bv *BlockValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub
 	err = sigs.CheckBlockSignature(ctx, blk.Header, key)
 	if err != nil {
 		log.Errorf("block signature verification failed: %s", err)
-		recordFailure("signature_verification_failed")
+		recordFailureFlagPeer("signature_verification_failed")
 		return pubsub.ValidationReject
 	}
 
 	if blk.Header.ElectionProof.WinCount < 1 {
 		log.Errorf("block is not claiming to be winning")
-		recordFailure("not_winning")
+		recordFailureFlagPeer("not_winning")
 		return pubsub.ValidationReject
 	}
 
@@ -473,19 +477,19 @@ func (bv *BlockValidator) checkPowerAndGetWorkerKey(ctx context.Context, bh *typ
 	baseTs := bv.chain.GetHeaviestTipSet()
 	lbts, err := stmgr.GetLookbackTipSetForRound(ctx, bv.stmgr, baseTs, bh.Height)
 	if err != nil {
-		log.Warnf("failed to load lookback tipset for incoming block")
-		return address.Undef, err
+		log.Warnf("failed to load lookback tipset for incoming block: %s", err)
+		return address.Undef, ErrSoftFailure
 	}
 
 	hmp, err := stmgr.MinerHasMinPower(ctx, bv.stmgr, bh.Miner, lbts)
 	if err != nil {
-		log.Warnf("failed to determine if incoming block's miner has minimum power")
-		return address.Undef, err
+		log.Warnf("failed to determine if incoming block's miner has minimum power: %s", err)
+		return address.Undef, ErrSoftFailure
 	}
 
 	if !hmp {
 		log.Warnf("incoming block's miner does not have minimum power")
-		return address.Undef, xerrors.New("incoming block's miner does not have minimum power")
+		return address.Undef, ErrInsufficientPower
 	}
 
 	return key, nil
@@ -541,14 +545,16 @@ func (mv *MessageValidator) Validate(ctx context.Context, pid peer.ID, msg *pubs
 		log.Debugf("failed to add message from network to message pool (From: %s, To: %s, Nonce: %d, Value: %s): %s", m.Message.From, m.Message.To, m.Message.Nonce, types.FIL(m.Message.Value), err)
 		ctx, _ = tag.New(
 			ctx,
-			tag.Insert(metrics.FailureType, "add"),
+			tag.Upsert(metrics.Local, "false"),
 		)
-		stats.Record(ctx, metrics.MessageValidationFailure.M(1))
+		recordFailure(ctx, metrics.MessageValidationFailure, "add")
 		switch {
-		case xerrors.Is(err, messagepool.ErrBroadcastAnyway):
+		case xerrors.Is(err, messagepool.ErrSoftValidationFailure):
 			fallthrough
 		case xerrors.Is(err, messagepool.ErrRBFTooLowPremium):
 			fallthrough
+		case xerrors.Is(err, messagepool.ErrTooManyPendingMessages):
+			fallthrough
 		case xerrors.Is(err, messagepool.ErrNonceTooLow):
 			return pubsub.ValidationIgnore
 		default:
@@ -560,37 +566,41 @@ func (mv *MessageValidator) Validate(ctx context.Context, pid peer.ID, msg *pubs
 }
 
 func (mv *MessageValidator) validateLocalMessage(ctx context.Context, msg *pubsub.Message) pubsub.ValidationResult {
+	ctx, _ = tag.New(
+		ctx,
+		tag.Upsert(metrics.Local, "true"),
+	)
 	// do some lightweight validation
 	stats.Record(ctx, metrics.MessagePublished.M(1))
 
 	m, err := types.DecodeSignedMessage(msg.Message.GetData())
 	if err != nil {
 		log.Warnf("failed to decode local message: %s", err)
-		stats.Record(ctx, metrics.MessageValidationFailure.M(1))
+		recordFailure(ctx, metrics.MessageValidationFailure, "decode")
 		return pubsub.ValidationIgnore
 	}
 
 	if m.Size() > 32*1024 {
 		log.Warnf("local message is too large! (%dB)", m.Size())
-		stats.Record(ctx, metrics.MessageValidationFailure.M(1))
+		recordFailure(ctx, metrics.MessageValidationFailure, "oversize")
 		return pubsub.ValidationIgnore
 	}
 
 	if m.Message.To == address.Undef {
 		log.Warn("local message has invalid destination address")
-		stats.Record(ctx, metrics.MessageValidationFailure.M(1))
+		recordFailure(ctx, metrics.MessageValidationFailure, "undef-addr")
 		return pubsub.ValidationIgnore
 	}
 
 	if !m.Message.Value.LessThan(types.TotalFilecoinInt) {
 		log.Warnf("local messages has too high value: %s", m.Message.Value)
-		stats.Record(ctx, metrics.MessageValidationFailure.M(1))
+		recordFailure(ctx, metrics.MessageValidationFailure, "value-too-high")
 		return pubsub.ValidationIgnore
 	}
 
 	if err := mv.mpool.VerifyMsgSig(m); err != nil {
 		log.Warnf("signature verification failed for local message: %s", err)
-		stats.Record(ctx, metrics.MessageValidationFailure.M(1))
+		recordFailure(ctx, metrics.MessageValidationFailure, "verify-sig")
 		return pubsub.ValidationIgnore
 	}
 
@@ -613,3 +623,11 @@ func HandleIncomingMessages(ctx context.Context, mpool *messagepool.MessagePool,
 		// Do nothing... everything happens in validate
 	}
 }
+
+func recordFailure(ctx context.Context, metric *stats.Int64Measure, failureType string) {
+	ctx, _ = tag.New(
+		ctx,
+		tag.Upsert(metrics.FailureType, failureType),
+	)
+	stats.Record(ctx, metric.M(1))
+}

chain/sync.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -52,6 +53,19 @@ import (
 //the theoretical max height based on systime are quickly rejected
 const MaxHeightDrift = 5
 
+var defaultMessageFetchWindowSize = 200
+
+func init() {
+	if s := os.Getenv("LOTUS_BSYNC_MSG_WINDOW"); s != "" {
+		val, err := strconv.Atoi(s)
+		if err != nil {
+			log.Errorf("failed to parse LOTUS_BSYNC_MSG_WINDOW: %s", err)
+			return
+		}
+		defaultMessageFetchWindowSize = val
+	}
+}
+
 var log = logging.Logger("chain")
 
 var LocalIncoming = "incoming"
@@ -109,6 +123,8 @@ type Syncer struct {
 	receiptTracker *blockReceiptTracker
 
 	verifier ffiwrapper.Verifier
+
+	windowSize int
 }
 
 // NewSyncer creates a new Syncer object.
@@ -134,6 +150,7 @@ func NewSyncer(sm *stmgr.StateManager, bsync *blocksync.BlockSync, connmgr connm
 		receiptTracker: newBlockReceiptTracker(),
 		connmgr:        connmgr,
 		verifier:       verifier,
+		windowSize:     defaultMessageFetchWindowSize,
 
 		incoming: pubsub.New(50),
 	}
@@ -641,7 +658,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
 	validationStart := build.Clock.Now()
 	defer func() {
 		stats.Record(ctx, metrics.BlockValidationDurationMilliseconds.M(metrics.SinceInMilliseconds(validationStart)))
-		log.Infow("block validation", "took", time.Since(validationStart), "height", b.Header.Height)
+		log.Infow("block validation", "took", time.Since(validationStart), "height", b.Header.Height, "age", time.Since(time.Unix(int64(b.Header.Timestamp), 0)))
 	}()
 
 	ctx, span := trace.StartSpan(ctx, "validateBlock")
@@ -1399,7 +1416,8 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS
 
 	span.AddAttributes(trace.Int64Attribute("num_headers", int64(len(headers))))
 
-	windowSize := 200
+	windowSize := syncer.windowSize
+mainLoop:
 	for i := len(headers) - 1; i >= 0; {
 		fts, err := syncer.store.TryFillTipSet(headers[i])
 		if err != nil {
@@ -1427,6 +1445,12 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS
 			nreq := batchSize - len(bstout)
 			bstips, err := syncer.Bsync.GetChainMessages(ctx, next, uint64(nreq))
 			if err != nil {
+				// TODO check errors for temporary nature
+				if windowSize > 1 {
+					windowSize /= 2
+					log.Infof("error fetching messages: %s; reducing window size to %d and trying again", err, windowSize)
+					continue mainLoop
+				}
 				return xerrors.Errorf("message processing failed: %w", err)
 			}
 
@@ -1461,9 +1485,24 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS
 				return xerrors.Errorf("message processing failed: %w", err)
 			}
 		}
+
+		if i >= windowSize {
+			newWindowSize := windowSize + 10
+			if newWindowSize > int(blocksync.MaxRequestLength) {
+				newWindowSize = int(blocksync.MaxRequestLength)
+			}
+			if newWindowSize > windowSize {
+				windowSize = newWindowSize
+				log.Infof("successfully fetched %d messages; increasing window size to %d", len(bstout), windowSize)
+			}
+		}
+
 		i -= batchSize
 	}
 
+	// remember our window size
+	syncer.windowSize = windowSize
+
 	return nil
 }
 

chain/vm/runtime.go

@@ -34,7 +34,6 @@ type Runtime struct {
 
 	vm        *VM
 	state     *state.StateTree
-	msg       *types.Message
 	vmsg      vmr.Message
 	height    abi.ChainEpoch
 	cst       cbor.IpldStore

chain/vm/vm.go

@@ -97,7 +97,6 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin addres
 		ctx:         ctx,
 		vm:          vm,
 		state:       vm.cstate,
-		msg:         msg,
 		origin:      origin,
 		originNonce: originNonce,
 		height:      vm.blockHeight,

cli/client.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -978,6 +979,10 @@ var clientListDeals = &cli.Command{
 			Usage: "use color in display output",
 			Value: true,
 		},
+		&cli.BoolFlag{
+			Name:  "watch",
+			Usage: "watch deal updates in real-time, rather than a one time list",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		api, closer, err := GetFullNodeAPI(cctx)
@@ -987,82 +992,97 @@ var clientListDeals = &cli.Command{
 		defer closer()
 		ctx := ReqContext(cctx)
 
-		head, err := api.ChainHead(ctx)
-		if err != nil {
-			return err
-		}
+		verbose := cctx.Bool("verbose")
+		color := cctx.Bool("color")
+		watch := cctx.Bool("watch")
 
 		localDeals, err := api.ClientListDeals(ctx)
 		if err != nil {
 			return err
 		}
 
-		sort.Slice(localDeals, func(i, j int) bool {
-			return localDeals[i].CreationTime.Before(localDeals[j].CreationTime)
-		})
+		if watch {
+			updates, err := api.ClientGetDealUpdates(ctx)
+			if err != nil {
+				return err
+			}
 
-		var deals []deal
-		for _, v := range localDeals {
-			if v.DealID == 0 {
-				deals = append(deals, deal{
-					LocalDeal: v,
-					OnChainDealState: market.DealState{
-						SectorStartEpoch: -1,
-						LastUpdatedEpoch: -1,
-						SlashEpoch:       -1,
-					},
-				})
-			} else {
-				onChain, err := api.StateMarketStorageDeal(ctx, v.DealID, head.Key())
+			for {
+				tm.Clear()
+				tm.MoveCursor(1, 1)
+
+				err = outputStorageDeals(ctx, tm.Screen, api, localDeals, verbose, color)
 				if err != nil {
-					deals = append(deals, deal{LocalDeal: v})
-				} else {
-					deals = append(deals, deal{
-						LocalDeal:        v,
-						OnChainDealState: onChain.State,
-					})
+					return err
+				}
+
+				tm.Flush()
+
+				select {
+				case <-ctx.Done():
+					return nil
+				case updated := <-updates:
+					var found bool
+					for i, existing := range localDeals {
+						if existing.ProposalCid.Equals(updated.ProposalCid) {
+							localDeals[i] = updated
+							found = true
+							break
+						}
+					}
+					if !found {
+						localDeals = append(localDeals, updated)
+					}
 				}
 			}
 		}
 
-		color := cctx.Bool("color")
+		return outputStorageDeals(ctx, os.Stdout, api, localDeals, cctx.Bool("verbose"), cctx.Bool("color"))
+	},
+}
 
-		if cctx.Bool("verbose") {
-			w := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0)
-			fmt.Fprintf(w, "Created\tDealCid\tDealId\tProvider\tState\tOn Chain?\tSlashed?\tPieceCID\tSize\tPrice\tDuration\tMessage\n")
-			for _, d := range deals {
-				onChain := "N"
-				if d.OnChainDealState.SectorStartEpoch != -1 {
-					onChain = fmt.Sprintf("Y (epoch %d)", d.OnChainDealState.SectorStartEpoch)
-				}
-
-				slashed := "N"
-				if d.OnChainDealState.SlashEpoch != -1 {
-					slashed = fmt.Sprintf("Y (epoch %d)", d.OnChainDealState.SlashEpoch)
-				}
-
-				price := types.FIL(types.BigMul(d.LocalDeal.PricePerEpoch, types.NewInt(d.LocalDeal.Duration)))
-				fmt.Fprintf(w, "%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n", d.LocalDeal.CreationTime.Format(time.Stamp), d.LocalDeal.ProposalCid, d.LocalDeal.DealID, d.LocalDeal.Provider, dealStateString(color, d.LocalDeal.State), onChain, slashed, d.LocalDeal.PieceCID, types.SizeStr(types.NewInt(d.LocalDeal.Size)), price, d.LocalDeal.Duration, d.LocalDeal.Message)
-			}
-			return w.Flush()
+func dealFromDealInfo(ctx context.Context, full api.FullNode, head *types.TipSet, v api.DealInfo) deal {
+	if v.DealID == 0 {
+		return deal{
+			LocalDeal: v,
+			OnChainDealState: market.DealState{
+				SectorStartEpoch: -1,
+				LastUpdatedEpoch: -1,
+				SlashEpoch:       -1,
+			},
 		}
+	}
 
-		w := tablewriter.New(tablewriter.Col("DealCid"),
-			tablewriter.Col("DealId"),
-			tablewriter.Col("Provider"),
-			tablewriter.Col("State"),
-			tablewriter.Col("On Chain?"),
-			tablewriter.Col("Slashed?"),
-			tablewriter.Col("PieceCID"),
-			tablewriter.Col("Size"),
-			tablewriter.Col("Price"),
-			tablewriter.Col("Duration"),
-			tablewriter.NewLineCol("Message"))
+	onChain, err := full.StateMarketStorageDeal(ctx, v.DealID, head.Key())
+	if err != nil {
+		return deal{LocalDeal: v}
+	}
 
+	return deal{
+		LocalDeal:        v,
+		OnChainDealState: onChain.State,
+	}
+}
+
+func outputStorageDeals(ctx context.Context, out io.Writer, full api.FullNode, localDeals []api.DealInfo, verbose bool, color bool) error {
+	sort.Slice(localDeals, func(i, j int) bool {
+		return localDeals[i].CreationTime.Before(localDeals[j].CreationTime)
+	})
+
+	head, err := full.ChainHead(ctx)
+	if err != nil {
+		return err
+	}
+
+	var deals []deal
+	for _, localDeal := range localDeals {
+		deals = append(deals, dealFromDealInfo(ctx, full, head, localDeal))
+	}
+
+	if verbose {
+		w := tabwriter.NewWriter(out, 2, 4, 2, ' ', 0)
+		fmt.Fprintf(w, "Created\tDealCid\tDealId\tProvider\tState\tOn Chain?\tSlashed?\tPieceCID\tSize\tPrice\tDuration\tMessage\n")
 		for _, d := range deals {
-			propcid := d.LocalDeal.ProposalCid.String()
-			propcid = "..." + propcid[len(propcid)-8:]
-
 			onChain := "N"
 			if d.OnChainDealState.SectorStartEpoch != -1 {
 				onChain = fmt.Sprintf("Y (epoch %d)", d.OnChainDealState.SectorStartEpoch)
@@ -1073,28 +1093,57 @@ var clientListDeals = &cli.Command{
 				slashed = fmt.Sprintf("Y (epoch %d)", d.OnChainDealState.SlashEpoch)
 			}
 
-			piece := d.LocalDeal.PieceCID.String()
-			piece = "..." + piece[len(piece)-8:]
-
 			price := types.FIL(types.BigMul(d.LocalDeal.PricePerEpoch, types.NewInt(d.LocalDeal.Duration)))
+			fmt.Fprintf(w, "%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n", d.LocalDeal.CreationTime.Format(time.Stamp), d.LocalDeal.ProposalCid, d.LocalDeal.DealID, d.LocalDeal.Provider, dealStateString(color, d.LocalDeal.State), onChain, slashed, d.LocalDeal.PieceCID, types.SizeStr(types.NewInt(d.LocalDeal.Size)), price, d.LocalDeal.Duration, d.LocalDeal.Message)
+		}
+		return w.Flush()
+	}
 
-			w.Write(map[string]interface{}{
-				"DealCid":   propcid,
-				"DealId":    d.LocalDeal.DealID,
-				"Provider":  d.LocalDeal.Provider,
-				"State":     dealStateString(color, d.LocalDeal.State),
-				"On Chain?": onChain,
-				"Slashed?":  slashed,
-				"PieceCID":  piece,
-				"Size":      types.SizeStr(types.NewInt(d.LocalDeal.Size)),
-				"Price":     price,
-				"Duration":  d.LocalDeal.Duration,
-				"Message":   d.LocalDeal.Message,
-			})
+	w := tablewriter.New(tablewriter.Col("DealCid"),
+		tablewriter.Col("DealId"),
+		tablewriter.Col("Provider"),
+		tablewriter.Col("State"),
+		tablewriter.Col("On Chain?"),
+		tablewriter.Col("Slashed?"),
+		tablewriter.Col("PieceCID"),
+		tablewriter.Col("Size"),
+		tablewriter.Col("Price"),
+		tablewriter.Col("Duration"),
+		tablewriter.NewLineCol("Message"))
+
+	for _, d := range deals {
+		propcid := ellipsis(d.LocalDeal.ProposalCid.String(), 8)
+
+		onChain := "N"
+		if d.OnChainDealState.SectorStartEpoch != -1 {
+			onChain = fmt.Sprintf("Y (epoch %d)", d.OnChainDealState.SectorStartEpoch)
 		}
 
-		return w.Flush(os.Stdout)
-	},
+		slashed := "N"
+		if d.OnChainDealState.SlashEpoch != -1 {
+			slashed = fmt.Sprintf("Y (epoch %d)", d.OnChainDealState.SlashEpoch)
+		}
+
+		piece := ellipsis(d.LocalDeal.PieceCID.String(), 8)
+
+		price := types.FIL(types.BigMul(d.LocalDeal.PricePerEpoch, types.NewInt(d.LocalDeal.Duration)))
+
+		w.Write(map[string]interface{}{
+			"DealCid":   propcid,
+			"DealId":    d.LocalDeal.DealID,
+			"Provider":  d.LocalDeal.Provider,
+			"State":     dealStateString(color, d.LocalDeal.State),
+			"On Chain?": onChain,
+			"Slashed?":  slashed,
+			"PieceCID":  piece,
+			"Size":      types.SizeStr(types.NewInt(d.LocalDeal.Size)),
+			"Price":     price,
+			"Duration":  d.LocalDeal.Duration,
+			"Message":   d.LocalDeal.Message,
+		})
+	}
+
+	return w.Flush(out)
 }
 
 func dealStateString(c bool, state storagemarket.StorageDealStatus) string {
@@ -1352,11 +1401,8 @@ func channelStatusString(useColor bool, status datatransfer.Status) string {
 }
 
 func toChannelOutput(useColor bool, otherPartyColumn string, channel lapi.DataTransferChannel) map[string]interface{} {
-	rootCid := channel.BaseCID.String()
-	rootCid = "..." + rootCid[len(rootCid)-8:]
-
-	otherParty := channel.OtherPeer.String()
-	otherParty = "..." + otherParty[len(otherParty)-8:]
+	rootCid := ellipsis(channel.BaseCID.String(), 8)
+	otherParty := ellipsis(channel.OtherPeer.String(), 8)
 
 	initiated := "N"
 	if channel.IsInitiator {
@@ -1365,7 +1411,7 @@ func toChannelOutput(useColor bool, otherPartyColumn string, channel lapi.DataTr
 
 	voucher := channel.Voucher
 	if len(voucher) > 40 {
-		voucher = "..." + voucher[len(voucher)-37:]
+		voucher = ellipsis(voucher, 37)
 	}
 
 	return map[string]interface{}{
@@ -1379,3 +1425,10 @@ func toChannelOutput(useColor bool, otherPartyColumn string, channel lapi.DataTr
 		"Message":        channel.Message,
 	}
 }
+
+func ellipsis(s string, length int) string {
+	if length > 0 && len(s) > length {
+		return "..." + s[len(s)-length:]
+	}
+	return s
+}

cli/cmd.go

@@ -12,7 +12,7 @@ import (
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/mitchellh/go-homedir"
 	"github.com/multiformats/go-multiaddr"
-	manet "github.com/multiformats/go-multiaddr-net"
+	manet "github.com/multiformats/go-multiaddr/net"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
@@ -213,7 +213,7 @@ func GetAPI(ctx *cli.Context) (api.Common, jsonrpc.ClientCloser, error) {
 		return nil, nil, err
 	}
 
-	return client.NewCommonRPC(addr, headers)
+	return client.NewCommonRPC(ctx.Context, addr, headers)
 }
 
 func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error) {
@@ -222,7 +222,7 @@ func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error
 		return nil, nil, err
 	}
 
-	return client.NewFullNodeRPC(addr, headers)
+	return client.NewFullNodeRPC(ctx.Context, addr, headers)
 }
 
 func GetStorageMinerAPI(ctx *cli.Context, opts ...jsonrpc.Option) (api.StorageMiner, jsonrpc.ClientCloser, error) {
@@ -231,7 +231,7 @@ func GetStorageMinerAPI(ctx *cli.Context, opts ...jsonrpc.Option) (api.StorageMi
 		return nil, nil, err
 	}
 
-	return client.NewStorageMinerRPC(addr, headers, opts...)
+	return client.NewStorageMinerRPC(ctx.Context, addr, headers, opts...)
 }
 
 func DaemonContext(cctx *cli.Context) context.Context {

cli/log.go

@@ -20,7 +20,7 @@ var logList = &cli.Command{
 	Name:  "list",
 	Usage: "List log systems",
 	Action: func(cctx *cli.Context) error {
-		api, closer, err := GetFullNodeAPI(cctx)
+		api, closer, err := GetAPI(cctx)
 		if err != nil {
 			return err
 		}
@@ -71,7 +71,7 @@ var logSetLevel = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
-		api, closer, err := GetFullNodeAPI(cctx)
+		api, closer, err := GetAPI(cctx)
 		if err != nil {
 			return err
 		}

cli/mpool.go

@@ -106,6 +106,7 @@ var mpoolClear = &cli.Command{
 
 		really := cctx.Bool("really-do-it")
 		if !really {
+			//nolint:golint
 			return fmt.Errorf("--really-do-it must be specified for this action to have an effect; you have been warned")
 		}
 

cli/paych.go

@@ -468,7 +468,7 @@ var paychVoucherSubmitCmd = &cli.Command{
 
 		ctx := ReqContext(cctx)
 
-		mcid, err := api.PaychVoucherSubmit(ctx, ch, sv)
+		mcid, err := api.PaychVoucherSubmit(ctx, ch, sv, nil, nil)
 		if err != nil {
 			return err
 		}

cli/paych_test.go

@@ -107,12 +107,14 @@ func TestPaymentChannelVouchers(t *testing.T) {
 	ctx := context.Background()
 	nodes, addrs := startTwoNodesOneMiner(ctx, t, blocktime)
 	paymentCreator := nodes[0]
+	paymentReceiver := nodes[1]
 	creatorAddr := addrs[0]
 	receiverAddr := addrs[1]
 
 	// Create mock CLI
 	mockCLI := newMockCLI(t)
 	creatorCLI := mockCLI.client(paymentCreator.ListenAddr)
+	receiverCLI := mockCLI.client(paymentReceiver.ListenAddr)
 
 	// creator: paych get <creator> <receiver> <amount>
 	channelAmt := "100000"
@@ -127,43 +129,123 @@ func TestPaymentChannelVouchers(t *testing.T) {
 	// creator: paych voucher create <channel> <amount>
 	// Note: implied --lane=0
 	voucherAmt1 := 100
-	vamt1 := strconv.Itoa(voucherAmt1)
-	cmd = []string{chAddr.String(), vamt1}
+	cmd = []string{chAddr.String(), strconv.Itoa(voucherAmt1)}
 	voucher1 := creatorCLI.runCmd(paychVoucherCreateCmd, cmd)
 	vouchers = append(vouchers, voucherSpec{serialized: voucher1, lane: 0, amt: voucherAmt1})
 
 	// creator: paych voucher create <channel> <amount> --lane=5
 	lane5 := "--lane=5"
 	voucherAmt2 := 50
-	vamt2 := strconv.Itoa(voucherAmt2)
-	cmd = []string{lane5, chAddr.String(), vamt2}
+	cmd = []string{lane5, chAddr.String(), strconv.Itoa(voucherAmt2)}
 	voucher2 := creatorCLI.runCmd(paychVoucherCreateCmd, cmd)
 	vouchers = append(vouchers, voucherSpec{serialized: voucher2, lane: 5, amt: voucherAmt2})
 
 	// creator: paych voucher create <channel> <amount> --lane=5
 	voucherAmt3 := 70
-	vamt3 := strconv.Itoa(voucherAmt3)
-	cmd = []string{lane5, chAddr.String(), vamt3}
+	cmd = []string{lane5, chAddr.String(), strconv.Itoa(voucherAmt3)}
 	voucher3 := creatorCLI.runCmd(paychVoucherCreateCmd, cmd)
 	vouchers = append(vouchers, voucherSpec{serialized: voucher3, lane: 5, amt: voucherAmt3})
 
+	// creator: paych voucher create <channel> <amount> --lane=5
+	voucherAmt4 := 80
+	cmd = []string{lane5, chAddr.String(), strconv.Itoa(voucherAmt4)}
+	voucher4 := creatorCLI.runCmd(paychVoucherCreateCmd, cmd)
+	vouchers = append(vouchers, voucherSpec{serialized: voucher4, lane: 5, amt: voucherAmt4})
+
 	// creator: paych voucher list <channel> --export
 	cmd = []string{"--export", chAddr.String()}
 	list := creatorCLI.runCmd(paychVoucherListCmd, cmd)
 
-	// Check that voucher list output is correct
+	// Check that voucher list output is correct on creator
 	checkVoucherOutput(t, list, vouchers)
 
 	// creator: paych voucher best-spendable <channel>
 	cmd = []string{"--export", chAddr.String()}
 	bestSpendable := creatorCLI.runCmd(paychVoucherBestSpendableCmd, cmd)
 
-	// Check that best spendable output is correct
+	// Check that best spendable output is correct on creator
 	bestVouchers := []voucherSpec{
 		{serialized: voucher1, lane: 0, amt: voucherAmt1},
-		{serialized: voucher3, lane: 5, amt: voucherAmt3},
+		{serialized: voucher4, lane: 5, amt: voucherAmt4},
 	}
 	checkVoucherOutput(t, bestSpendable, bestVouchers)
+
+	// receiver: paych voucher add <voucher>
+	cmd = []string{chAddr.String(), voucher1}
+	receiverCLI.runCmd(paychVoucherAddCmd, cmd)
+
+	// receiver: paych voucher add <voucher>
+	cmd = []string{chAddr.String(), voucher2}
+	receiverCLI.runCmd(paychVoucherAddCmd, cmd)
+
+	// receiver: paych voucher add <voucher>
+	cmd = []string{chAddr.String(), voucher3}
+	receiverCLI.runCmd(paychVoucherAddCmd, cmd)
+
+	// receiver: paych voucher add <voucher>
+	cmd = []string{chAddr.String(), voucher4}
+	receiverCLI.runCmd(paychVoucherAddCmd, cmd)
+
+	// receiver: paych voucher list <channel> --export
+	cmd = []string{"--export", chAddr.String()}
+	list = receiverCLI.runCmd(paychVoucherListCmd, cmd)
+
+	// Check that voucher list output is correct on receiver
+	checkVoucherOutput(t, list, vouchers)
+
+	// receiver: paych voucher best-spendable <channel>
+	cmd = []string{"--export", chAddr.String()}
+	bestSpendable = receiverCLI.runCmd(paychVoucherBestSpendableCmd, cmd)
+
+	// Check that best spendable output is correct on receiver
+	bestVouchers = []voucherSpec{
+		{serialized: voucher1, lane: 0, amt: voucherAmt1},
+		{serialized: voucher4, lane: 5, amt: voucherAmt4},
+	}
+	checkVoucherOutput(t, bestSpendable, bestVouchers)
+
+	// receiver: paych voucher submit <channel> <voucher>
+	cmd = []string{chAddr.String(), voucher1}
+	receiverCLI.runCmd(paychVoucherSubmitCmd, cmd)
+
+	// receiver: paych voucher best-spendable <channel>
+	cmd = []string{"--export", chAddr.String()}
+	bestSpendable = receiverCLI.runCmd(paychVoucherBestSpendableCmd, cmd)
+
+	// Check that best spendable output no longer includes submitted voucher
+	bestVouchers = []voucherSpec{
+		{serialized: voucher4, lane: 5, amt: voucherAmt4},
+	}
+	checkVoucherOutput(t, bestSpendable, bestVouchers)
+
+	// There are three vouchers in lane 5: 50, 70, 80
+	// Submit the voucher for 50. Best spendable should still be 80.
+	// receiver: paych voucher submit <channel> <voucher>
+	cmd = []string{chAddr.String(), voucher2}
+	receiverCLI.runCmd(paychVoucherSubmitCmd, cmd)
+
+	// receiver: paych voucher best-spendable <channel>
+	cmd = []string{"--export", chAddr.String()}
+	bestSpendable = receiverCLI.runCmd(paychVoucherBestSpendableCmd, cmd)
+
+	// Check that best spendable output still includes the voucher for 80
+	bestVouchers = []voucherSpec{
+		{serialized: voucher4, lane: 5, amt: voucherAmt4},
+	}
+	checkVoucherOutput(t, bestSpendable, bestVouchers)
+
+	// Submit the voucher for 80
+	// receiver: paych voucher submit <channel> <voucher>
+	cmd = []string{chAddr.String(), voucher4}
+	receiverCLI.runCmd(paychVoucherSubmitCmd, cmd)
+
+	// receiver: paych voucher best-spendable <channel>
+	cmd = []string{"--export", chAddr.String()}
+	bestSpendable = receiverCLI.runCmd(paychVoucherBestSpendableCmd, cmd)
+
+	// Check that best spendable output no longer includes submitted voucher
+	bestVouchers = []voucherSpec{}
+	checkVoucherOutput(t, bestSpendable, bestVouchers)
 }
 
 func checkVoucherOutput(t *testing.T, list string, vouchers []voucherSpec) {
@@ -171,14 +253,20 @@ func checkVoucherOutput(t *testing.T, list string, vouchers []voucherSpec) {
 	listVouchers := make(map[string]string)
 	for _, line := range lines {
 		parts := strings.Split(line, ";")
-		serialized := strings.TrimSpace(parts[1])
-		listVouchers[serialized] = strings.TrimSpace(parts[0])
+		if len(parts) == 2 {
+			serialized := strings.TrimSpace(parts[1])
+			listVouchers[serialized] = strings.TrimSpace(parts[0])
+		}
 	}
 	for _, vchr := range vouchers {
 		res, ok := listVouchers[vchr.serialized]
 		require.True(t, ok)
 		require.Regexp(t, fmt.Sprintf("Lane %d", vchr.lane), res)
 		require.Regexp(t, fmt.Sprintf("%d", vchr.amt), res)
+		delete(listVouchers, vchr.serialized)
+	}
+	for _, vchr := range listVouchers {
+		require.Fail(t, "Extra voucher "+vchr)
 	}
 }
 

cli/pprof.go

@@ -9,7 +9,7 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/node/repo"
-	manet "github.com/multiformats/go-multiaddr-net"
+	manet "github.com/multiformats/go-multiaddr/net"
 )
 
 var pprofCmd = &cli.Command{

cmd/lotus-chainwatch/main.go

@@ -26,6 +26,11 @@ func main() {
 				EnvVars: []string{"LOTUS_PATH"},
 				Value:   "~/.lotus", // TODO: Consider XDG_DATA_HOME
 			},
+			&cli.StringFlag{
+				Name:    "api",
+				EnvVars: []string{"FULLNODE_API_INFO"},
+				Value:   "",
+			},
 			&cli.StringFlag{
 				Name:    "db",
 				EnvVars: []string{"LOTUS_DB"},

cmd/lotus-chainwatch/processor/market.go

@@ -96,12 +96,6 @@ func (p *Processor) HandleMarketChanges(ctx context.Context, marketTips ActorTip
 		log.Fatalw("Failed to persist market actors", "error", err)
 	}
 
-	// we persist the dealID <--> minerID,sectorID here since the dealID needs to be stored above first
-	if err := p.storePreCommitDealInfo(p.sectorDealEvents); err != nil {
-		close(p.sectorDealEvents)
-		return err
-	}
-
 	if err := p.updateMarket(ctx, marketChanges); err != nil {
 		log.Fatalw("Failed to update market actors", "error", err)
 	}
@@ -272,48 +266,6 @@ func (p *Processor) storeMarketActorDealProposals(ctx context.Context, marketTip
 
 }
 
-func (p *Processor) storePreCommitDealInfo(dealEvents <-chan *SectorDealEvent) error {
-	tx, err := p.db.Begin()
-	if err != nil {
-		return err
-	}
-
-	if _, err := tx.Exec(`create temp table mds (like minerid_dealid_sectorid excluding constraints) on commit  drop;`); err != nil {
-		return xerrors.Errorf("Failed to create temp table for minerid_dealid_sectorid: %w", err)
-	}
-
-	stmt, err := tx.Prepare(`copy mds (deal_id, miner_id, sector_id) from STDIN`)
-	if err != nil {
-		return xerrors.Errorf("Failed to prepare minerid_dealid_sectorid statement: %w", err)
-	}
-
-	for sde := range dealEvents {
-		for _, did := range sde.DealIDs {
-			if _, err := stmt.Exec(
-				uint64(did),
-				sde.MinerID.String(),
-				sde.SectorID,
-			); err != nil {
-				return err
-			}
-		}
-	}
-
-	if err := stmt.Close(); err != nil {
-		return xerrors.Errorf("Failed to close miner sector deals statement: %w", err)
-	}
-
-	if _, err := tx.Exec(`insert into minerid_dealid_sectorid select * from mds on conflict do nothing`); err != nil {
-		return xerrors.Errorf("Failed to insert into miner deal sector table: %w", err)
-	}
-
-	if err := tx.Commit(); err != nil {
-		return xerrors.Errorf("Failed to commit miner deal sector table: %w", err)
-	}
-	return nil
-
-}
-
 func (p *Processor) updateMarketActorDealProposals(ctx context.Context, marketTip []marketActorInfo) error {
 	start := time.Now()
 	defer func() {

cmd/lotus-chainwatch/processor/messages.go

@@ -3,7 +3,6 @@ package processor
 import (
 	"context"
 	"sync"
-	"time"
 
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/xerrors"
@@ -120,10 +119,6 @@ func (p *Processor) persistMessagesAndReceipts(ctx context.Context, blocks map[c
 }
 
 func (p *Processor) storeReceipts(recs map[mrec]*types.MessageReceipt) error {
-	start := time.Now()
-	defer func() {
-		log.Debugw("Persisted Receipts", "duration", time.Since(start).String())
-	}()
 	tx, err := p.db.Begin()
 	if err != nil {
 		return err
@@ -164,10 +159,6 @@ create temp table recs (like receipts excluding constraints) on commit drop;
 }
 
 func (p *Processor) storeMsgInclusions(incls map[cid.Cid][]cid.Cid) error {
-	start := time.Now()
-	defer func() {
-		log.Debugw("Persisted Message Inclusions", "duration", time.Since(start).String())
-	}()
 	tx, err := p.db.Begin()
 	if err != nil {
 		return err
@@ -206,10 +197,6 @@ create temp table mi (like block_messages excluding constraints) on commit drop;
 }
 
 func (p *Processor) storeMessages(msgs map[cid.Cid]*types.Message) error {
-	start := time.Now()
-	defer func() {
-		log.Debugw("Persisted Messages", "duration", time.Since(start).String())
-	}()
 	tx, err := p.db.Begin()
 	if err != nil {
 		return err

cmd/lotus-chainwatch/processor/miner.go

@@ -271,7 +271,11 @@ func (p *Processor) persistMiners(ctx context.Context, miners []minerActorInfo)
 	preCommitEvents := make(chan *MinerSectorsEvent, 8)
 	sectorEvents := make(chan *MinerSectorsEvent, 8)
 	partitionEvents := make(chan *MinerSectorsEvent, 8)
-	p.sectorDealEvents = make(chan *SectorDealEvent, 8)
+	dealEvents := make(chan *SectorDealEvent, 8)
+
+	grp.Go(func() error {
+		return p.storePreCommitDealInfo(dealEvents)
+	})
 
 	grp.Go(func() error {
 		return p.storeMinerSectorEvents(ctx, sectorEvents, preCommitEvents, partitionEvents)
@@ -280,9 +284,9 @@ func (p *Processor) persistMiners(ctx context.Context, miners []minerActorInfo)
 	grp.Go(func() error {
 		defer func() {
 			close(preCommitEvents)
-			close(p.sectorDealEvents)
+			close(dealEvents)
 		}()
-		return p.storeMinerPreCommitInfo(ctx, miners, preCommitEvents, p.sectorDealEvents)
+		return p.storeMinerPreCommitInfo(ctx, miners, preCommitEvents, dealEvents)
 	})
 
 	grp.Go(func() error {
@@ -314,100 +318,107 @@ func (p *Processor) storeMinerPreCommitInfo(ctx context.Context, miners []minerA
 		return xerrors.Errorf("Failed to prepare miner precommit info statement: %w", err)
 	}
 
+	grp, _ := errgroup.WithContext(ctx)
 	for _, m := range miners {
-		minerSectors, err := adt.AsArray(p.ctxStore, m.state.Sectors)
-		if err != nil {
-			return err
-		}
-
-		changes, err := p.getMinerPreCommitChanges(ctx, m)
-		if err != nil {
-			if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) {
-				continue
-			} else {
+		m := m
+		grp.Go(func() error {
+			minerSectors, err := adt.AsArray(p.ctxStore, m.state.Sectors)
+			if err != nil {
 				return err
 			}
-		}
-		if changes == nil {
-			continue
-		}
 
-		preCommitAdded := make([]uint64, len(changes.Added))
-		for i, added := range changes.Added {
-			if len(added.Info.DealIDs) > 0 {
-				sectorDeals <- &SectorDealEvent{
-					MinerID:  m.common.addr,
-					SectorID: uint64(added.Info.SectorNumber),
-					DealIDs:  added.Info.DealIDs,
+			changes, err := p.getMinerPreCommitChanges(ctx, m)
+			if err != nil {
+				if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) {
+					return nil
 				}
-			}
-			if added.Info.ReplaceCapacity {
-				if _, err := stmt.Exec(
-					m.common.addr.String(),
-					added.Info.SectorNumber,
-					added.Info.SealedCID.String(),
-					m.common.stateroot.String(),
-					added.Info.SealRandEpoch,
-					added.Info.Expiration,
-					added.PreCommitDeposit.String(),
-					added.PreCommitEpoch,
-					added.DealWeight.String(),
-					added.VerifiedDealWeight.String(),
-					added.Info.ReplaceCapacity,
-					added.Info.ReplaceSectorDeadline,
-					added.Info.ReplaceSectorPartition,
-					added.Info.ReplaceSectorNumber,
-				); err != nil {
-					return err
-				}
-			} else {
-				if _, err := stmt.Exec(
-					m.common.addr.String(),
-					added.Info.SectorNumber,
-					added.Info.SealedCID.String(),
-					m.common.stateroot.String(),
-					added.Info.SealRandEpoch,
-					added.Info.Expiration,
-					added.PreCommitDeposit.String(),
-					added.PreCommitEpoch,
-					added.DealWeight.String(),
-					added.VerifiedDealWeight.String(),
-					added.Info.ReplaceCapacity,
-					nil, // replace deadline
-					nil, // replace partition
-					nil, // replace sector
-				); err != nil {
-					return err
-				}
-
-			}
-			preCommitAdded[i] = uint64(added.Info.SectorNumber)
-		}
-		if len(preCommitAdded) > 0 {
-			sectorEvents <- &MinerSectorsEvent{
-				MinerID:   m.common.addr,
-				StateRoot: m.common.stateroot,
-				SectorIDs: preCommitAdded,
-				Event:     PreCommitAdded,
-			}
-		}
-		var preCommitExpired []uint64
-		for _, removed := range changes.Removed {
-			var sector miner.SectorOnChainInfo
-			if found, err := minerSectors.Get(uint64(removed.Info.SectorNumber), &sector); err != nil {
 				return err
-			} else if !found {
-				preCommitExpired = append(preCommitExpired, uint64(removed.Info.SectorNumber))
 			}
-		}
-		if len(preCommitExpired) > 0 {
-			sectorEvents <- &MinerSectorsEvent{
-				MinerID:   m.common.addr,
-				StateRoot: m.common.stateroot,
-				SectorIDs: preCommitExpired,
-				Event:     PreCommitExpired,
+			if changes == nil {
+				return nil
 			}
-		}
+
+			preCommitAdded := make([]uint64, len(changes.Added))
+			for i, added := range changes.Added {
+				if len(added.Info.DealIDs) > 0 {
+					sectorDeals <- &SectorDealEvent{
+						MinerID:  m.common.addr,
+						SectorID: uint64(added.Info.SectorNumber),
+						DealIDs:  added.Info.DealIDs,
+					}
+				}
+				if added.Info.ReplaceCapacity {
+					if _, err := stmt.Exec(
+						m.common.addr.String(),
+						added.Info.SectorNumber,
+						added.Info.SealedCID.String(),
+						m.common.stateroot.String(),
+						added.Info.SealRandEpoch,
+						added.Info.Expiration,
+						added.PreCommitDeposit.String(),
+						added.PreCommitEpoch,
+						added.DealWeight.String(),
+						added.VerifiedDealWeight.String(),
+						added.Info.ReplaceCapacity,
+						added.Info.ReplaceSectorDeadline,
+						added.Info.ReplaceSectorPartition,
+						added.Info.ReplaceSectorNumber,
+					); err != nil {
+						return err
+					}
+				} else {
+					if _, err := stmt.Exec(
+						m.common.addr.String(),
+						added.Info.SectorNumber,
+						added.Info.SealedCID.String(),
+						m.common.stateroot.String(),
+						added.Info.SealRandEpoch,
+						added.Info.Expiration,
+						added.PreCommitDeposit.String(),
+						added.PreCommitEpoch,
+						added.DealWeight.String(),
+						added.VerifiedDealWeight.String(),
+						added.Info.ReplaceCapacity,
+						nil, // replace deadline
+						nil, // replace partition
+						nil, // replace sector
+					); err != nil {
+						return err
+					}
+
+				}
+				preCommitAdded[i] = uint64(added.Info.SectorNumber)
+			}
+			if len(preCommitAdded) > 0 {
+				sectorEvents <- &MinerSectorsEvent{
+					MinerID:   m.common.addr,
+					StateRoot: m.common.stateroot,
+					SectorIDs: preCommitAdded,
+					Event:     PreCommitAdded,
+				}
+			}
+			var preCommitExpired []uint64
+			for _, removed := range changes.Removed {
+				var sector miner.SectorOnChainInfo
+				if found, err := minerSectors.Get(uint64(removed.Info.SectorNumber), &sector); err != nil {
+					return err
+				} else if !found {
+					preCommitExpired = append(preCommitExpired, uint64(removed.Info.SectorNumber))
+				}
+			}
+			if len(preCommitExpired) > 0 {
+				sectorEvents <- &MinerSectorsEvent{
+					MinerID:   m.common.addr,
+					StateRoot: m.common.stateroot,
+					SectorIDs: preCommitExpired,
+					Event:     PreCommitExpired,
+				}
+			}
+			return nil
+		})
+	}
+	if err := grp.Wait(); err != nil {
+		return err
 	}
 
 	if err := stmt.Close(); err != nil {
@@ -439,67 +450,75 @@ func (p *Processor) storeMinerSectorInfo(ctx context.Context, miners []minerActo
 		return xerrors.Errorf("Failed to prepare miner sector info statement: %w", err)
 	}
 
+	grp, _ := errgroup.WithContext(ctx)
 	for _, m := range miners {
-		changes, err := p.getMinerSectorChanges(ctx, m)
-		if err != nil {
-			if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) {
-				continue
-			} else {
+		m := m
+		grp.Go(func() error {
+			changes, err := p.getMinerSectorChanges(ctx, m)
+			if err != nil {
+				if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) {
+					return nil
+				}
 				return err
 			}
-		}
-		if changes == nil {
-			continue
-		}
-		var sectorsAdded []uint64
-		var ccAdded []uint64
-		var extended []uint64
-		for _, added := range changes.Added {
-			// add the sector to the table
-			if _, err := stmt.Exec(
-				m.common.addr.String(),
-				added.SectorNumber,
-				added.SealedCID.String(),
-				m.common.stateroot.String(),
-				added.Activation.String(),
-				added.Expiration.String(),
-				added.DealWeight.String(),
-				added.VerifiedDealWeight.String(),
-				added.InitialPledge.String(),
-				added.ExpectedDayReward.String(),
-				added.ExpectedStoragePledge.String(),
-			); err != nil {
-				return err
+			if changes == nil {
+				return nil
 			}
-			if len(added.DealIDs) == 0 {
-				ccAdded = append(ccAdded, uint64(added.SectorNumber))
-			} else {
-				sectorsAdded = append(sectorsAdded, uint64(added.SectorNumber))
+			var sectorsAdded []uint64
+			var ccAdded []uint64
+			var extended []uint64
+			for _, added := range changes.Added {
+				// add the sector to the table
+				if _, err := stmt.Exec(
+					m.common.addr.String(),
+					added.SectorNumber,
+					added.SealedCID.String(),
+					m.common.stateroot.String(),
+					added.Activation.String(),
+					added.Expiration.String(),
+					added.DealWeight.String(),
+					added.VerifiedDealWeight.String(),
+					added.InitialPledge.String(),
+					added.ExpectedDayReward.String(),
+					added.ExpectedStoragePledge.String(),
+				); err != nil {
+					log.Errorw("writing miner sector changes statement", "error", err.Error())
+				}
+				if len(added.DealIDs) == 0 {
+					ccAdded = append(ccAdded, uint64(added.SectorNumber))
+				} else {
+					sectorsAdded = append(sectorsAdded, uint64(added.SectorNumber))
+				}
 			}
-		}
 
-		for _, mod := range changes.Extended {
-			extended = append(extended, uint64(mod.To.SectorNumber))
-		}
+			for _, mod := range changes.Extended {
+				extended = append(extended, uint64(mod.To.SectorNumber))
+			}
 
-		events <- &MinerSectorsEvent{
-			MinerID:   m.common.addr,
-			StateRoot: m.common.stateroot,
-			SectorIDs: ccAdded,
-			Event:     CommitCapacityAdded,
-		}
-		events <- &MinerSectorsEvent{
-			MinerID:   m.common.addr,
-			StateRoot: m.common.stateroot,
-			SectorIDs: sectorsAdded,
-			Event:     SectorAdded,
-		}
-		events <- &MinerSectorsEvent{
-			MinerID:   m.common.addr,
-			StateRoot: m.common.stateroot,
-			SectorIDs: extended,
-			Event:     SectorExtended,
-		}
+			events <- &MinerSectorsEvent{
+				MinerID:   m.common.addr,
+				StateRoot: m.common.stateroot,
+				SectorIDs: ccAdded,
+				Event:     CommitCapacityAdded,
+			}
+			events <- &MinerSectorsEvent{
+				MinerID:   m.common.addr,
+				StateRoot: m.common.stateroot,
+				SectorIDs: sectorsAdded,
+				Event:     SectorAdded,
+			}
+			events <- &MinerSectorsEvent{
+				MinerID:   m.common.addr,
+				StateRoot: m.common.stateroot,
+				SectorIDs: extended,
+				Event:     SectorExtended,
+			}
+			return nil
+		})
+	}
+
+	if err := grp.Wait(); err != nil {
+		return err
 	}
 
 	if err := stmt.Close(); err != nil {
@@ -911,6 +930,48 @@ func (p *Processor) storeMinersActorInfoState(ctx context.Context, miners []mine
 	return tx.Commit()
 }
 
+func (p *Processor) storePreCommitDealInfo(dealEvents <-chan *SectorDealEvent) error {
+	tx, err := p.db.Begin()
+	if err != nil {
+		return err
+	}
+
+	if _, err := tx.Exec(`create temp table mds (like minerid_dealid_sectorid excluding constraints) on commit  drop;`); err != nil {
+		return xerrors.Errorf("Failed to create temp table for minerid_dealid_sectorid: %w", err)
+	}
+
+	stmt, err := tx.Prepare(`copy mds (deal_id, miner_id, sector_id) from STDIN`)
+	if err != nil {
+		return xerrors.Errorf("Failed to prepare minerid_dealid_sectorid statement: %w", err)
+	}
+
+	for sde := range dealEvents {
+		for _, did := range sde.DealIDs {
+			if _, err := stmt.Exec(
+				uint64(did),
+				sde.MinerID.String(),
+				sde.SectorID,
+			); err != nil {
+				return err
+			}
+		}
+	}
+
+	if err := stmt.Close(); err != nil {
+		return xerrors.Errorf("Failed to close miner sector deals statement: %w", err)
+	}
+
+	if _, err := tx.Exec(`insert into minerid_dealid_sectorid select * from mds on conflict do nothing`); err != nil {
+		return xerrors.Errorf("Failed to insert into miner deal sector table: %w", err)
+	}
+
+	if err := tx.Commit(); err != nil {
+		return xerrors.Errorf("Failed to commit miner deal sector table: %w", err)
+	}
+	return nil
+
+}
+
 func (p *Processor) storeMinersPower(miners []minerActorInfo) error {
 	start := time.Now()
 	defer func() {

cmd/lotus-chainwatch/processor/mpool.go

@@ -47,8 +47,6 @@ func (p *Processor) subMpool(ctx context.Context) {
 			msgs[v.Message.Message.Cid()] = &v.Message.Message
 		}
 
-		log.Debugf("Processing %d mpool updates", len(msgs))
-
 		err := p.storeMessages(msgs)
 		if err != nil {
 			log.Error(err)

cmd/lotus-chainwatch/processor/power.go

@@ -7,6 +7,7 @@ import (
 
 	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 	"github.com/filecoin-project/specs-actors/actors/util/smoothing"
@@ -15,7 +16,19 @@ import (
 type powerActorInfo struct {
 	common actorInfo
 
-	epochSmoothingEstimate *smoothing.FilterEstimate
+	totalRawBytes                      big.Int
+	totalRawBytesCommitted             big.Int
+	totalQualityAdjustedBytes          big.Int
+	totalQualityAdjustedBytesCommitted big.Int
+	totalPledgeCollateral              big.Int
+
+	newRawBytes             big.Int
+	newQualityAdjustedBytes big.Int
+	newPledgeCollateral     big.Int
+	newQAPowerSmoothed      *smoothing.FilterEstimate
+
+	minerCount                  int64
+	minerCountAboveMinimumPower int64
 }
 
 func (p *Processor) setupPower() error {
@@ -25,13 +38,27 @@ func (p *Processor) setupPower() error {
 	}
 
 	if _, err := tx.Exec(`
-create table if not exists power_smoothing_estimates
+create table if not exists chain_power
 (
-    state_root text not null
-        constraint power_smoothing_estimates_pk
-        	primary key,
-	position_estimate text not null,
-	velocity_estimate text not null
+	state_root text not null
+		constraint power_smoothing_estimates_pk
+			primary key,
+
+	new_raw_bytes_power text not null,
+	new_qa_bytes_power text not null,
+	new_pledge_collateral text not null,
+
+	total_raw_bytes_power text not null,
+	total_raw_bytes_committed text not null,
+	total_qa_bytes_power text not null,
+	total_qa_bytes_committed text not null,
+	total_pledge_collateral text not null,
+
+	qa_smoothed_position_estimate text not null,
+	qa_smoothed_velocity_estimate text not null,
+
+	miner_count int not null,
+	minimum_consensus_miner_count int not null
 );
 `); err != nil {
 		return err
@@ -60,8 +87,8 @@ func (p *Processor) processPowerActors(ctx context.Context, powerTips ActorTips)
 	}()
 
 	var out []powerActorInfo
-	for tipset, powers := range powerTips {
-		for _, act := range powers {
+	for tipset, powerStates := range powerTips {
+		for _, act := range powerStates {
 			var pw powerActorInfo
 			pw.common = act
 
@@ -80,7 +107,19 @@ func (p *Processor) processPowerActors(ctx context.Context, powerTips ActorTips)
 				return nil, xerrors.Errorf("unmarshal state (@ %s): %w", pw.common.stateroot.String(), err)
 			}
 
-			pw.epochSmoothingEstimate = powerActorState.ThisEpochQAPowerSmoothed
+			pw.totalRawBytes = powerActorState.TotalRawBytePower
+			pw.totalRawBytesCommitted = powerActorState.TotalBytesCommitted
+			pw.totalQualityAdjustedBytes = powerActorState.TotalQualityAdjPower
+			pw.totalQualityAdjustedBytesCommitted = powerActorState.TotalQABytesCommitted
+			pw.totalPledgeCollateral = powerActorState.TotalPledgeCollateral
+
+			pw.newRawBytes = powerActorState.ThisEpochRawBytePower
+			pw.newQualityAdjustedBytes = powerActorState.ThisEpochQualityAdjPower
+			pw.newPledgeCollateral = powerActorState.ThisEpochPledgeCollateral
+			pw.newQAPowerSmoothed = powerActorState.ThisEpochQAPowerSmoothed
+
+			pw.minerCount = powerActorState.MinerCount
+			pw.minerCountAboveMinimumPower = powerActorState.MinerAboveMinPowerCount
 			out = append(out, pw)
 		}
 	}
@@ -88,46 +127,59 @@ func (p *Processor) processPowerActors(ctx context.Context, powerTips ActorTips)
 	return out, nil
 }
 
-func (p *Processor) persistPowerActors(ctx context.Context, powers []powerActorInfo) error {
+func (p *Processor) persistPowerActors(ctx context.Context, powerStates []powerActorInfo) error {
 	// NB: use errgroup when there is more than a single store operation
-	return p.storePowerSmoothingEstimates(powers)
+	return p.storePowerSmoothingEstimates(powerStates)
 }
 
-func (p *Processor) storePowerSmoothingEstimates(powers []powerActorInfo) error {
+func (p *Processor) storePowerSmoothingEstimates(powerStates []powerActorInfo) error {
 	tx, err := p.db.Begin()
 	if err != nil {
-		return xerrors.Errorf("begin power_smoothing_estimates tx: %w", err)
+		return xerrors.Errorf("begin chain_power tx: %w", err)
 	}
 
-	if _, err := tx.Exec(`create temp table rse (like power_smoothing_estimates) on commit drop`); err != nil {
-		return xerrors.Errorf("prep power_smoothing_estimates: %w", err)
+	if _, err := tx.Exec(`create temp table cp (like chain_power) on commit drop`); err != nil {
+		return xerrors.Errorf("prep chain_power: %w", err)
 	}
 
-	stmt, err := tx.Prepare(`copy rse (state_root, position_estimate, velocity_estimate) from stdin;`)
+	stmt, err := tx.Prepare(`copy cp (state_root, new_raw_bytes_power, new_qa_bytes_power, new_pledge_collateral, total_raw_bytes_power, total_raw_bytes_committed, total_qa_bytes_power, total_qa_bytes_committed, total_pledge_collateral, qa_smoothed_position_estimate, qa_smoothed_velocity_estimate, miner_count, minimum_consensus_miner_count) from stdin;`)
 	if err != nil {
-		return xerrors.Errorf("prepare tmp power_smoothing_estimates: %w", err)
+		return xerrors.Errorf("prepare tmp chain_power: %w", err)
 	}
 
-	for _, powerState := range powers {
+	for _, ps := range powerStates {
 		if _, err := stmt.Exec(
-			powerState.common.stateroot.String(),
-			powerState.epochSmoothingEstimate.PositionEstimate.String(),
-			powerState.epochSmoothingEstimate.VelocityEstimate.String(),
+			ps.common.stateroot.String(),
+			ps.newRawBytes.String(),
+			ps.newQualityAdjustedBytes.String(),
+			ps.newPledgeCollateral.String(),
+
+			ps.totalRawBytes.String(),
+			ps.totalRawBytesCommitted.String(),
+			ps.totalQualityAdjustedBytes.String(),
+			ps.totalQualityAdjustedBytesCommitted.String(),
+			ps.totalPledgeCollateral.String(),
+
+			ps.newQAPowerSmoothed.PositionEstimate.String(),
+			ps.newQAPowerSmoothed.VelocityEstimate.String(),
+
+			ps.minerCount,
+			ps.minerCountAboveMinimumPower,
 		); err != nil {
 			return xerrors.Errorf("failed to store smoothing estimate: %w", err)
 		}
 	}
 
 	if err := stmt.Close(); err != nil {
-		return xerrors.Errorf("close prepared power_smoothing_estimates: %w", err)
+		return xerrors.Errorf("close prepared chain_power: %w", err)
 	}
 
-	if _, err := tx.Exec(`insert into power_smoothing_estimates select * from rse on conflict do nothing`); err != nil {
-		return xerrors.Errorf("insert power_smoothing_estimates from tmp: %w", err)
+	if _, err := tx.Exec(`insert into chain_power select * from cp on conflict do nothing`); err != nil {
+		return xerrors.Errorf("insert chain_power from tmp: %w", err)
 	}
 
 	if err := tx.Commit(); err != nil {
-		return xerrors.Errorf("commit power_smoothing_estimates tx: %w", err)
+		return xerrors.Errorf("commit chain_power tx: %w", err)
 	}
 
 	return nil

cmd/lotus-chainwatch/processor/processor.go

@@ -35,9 +35,6 @@ type Processor struct {
 
 	// number of blocks processed at a time
 	batch int
-
-	// process communication channels
-	sectorDealEvents chan *SectorDealEvent
 }
 
 type ActorTips map[types.TipSetKey][]actorInfo
@@ -152,7 +149,6 @@ func (p *Processor) Start(ctx context.Context) {
 						log.Errorf("Failed to handle market changes: %w", err)
 						return
 					}
-					log.Info("Processed Market Changes")
 				}()
 
 				grp.Add(1)
@@ -162,7 +158,6 @@ func (p *Processor) Start(ctx context.Context) {
 						log.Errorf("Failed to handle miner changes: %w", err)
 						return
 					}
-					log.Info("Processed Miner Changes")
 				}()
 
 				grp.Add(1)
@@ -172,7 +167,6 @@ func (p *Processor) Start(ctx context.Context) {
 						log.Errorf("Failed to handle reward changes: %w", err)
 						return
 					}
-					log.Info("Processed Reward Changes")
 				}()
 
 				grp.Add(1)
@@ -182,7 +176,6 @@ func (p *Processor) Start(ctx context.Context) {
 						log.Errorf("Failed to handle power actor changes: %w", err)
 						return
 					}
-					log.Info("Processes Power Changes")
 				}()
 
 				grp.Add(1)
@@ -192,7 +185,6 @@ func (p *Processor) Start(ctx context.Context) {
 						log.Errorf("Failed to handle message changes: %w", err)
 						return
 					}
-					log.Info("Processed Message Changes")
 				}()
 
 				grp.Add(1)
@@ -202,7 +194,6 @@ func (p *Processor) Start(ctx context.Context) {
 						log.Errorf("Failed to handle common actor changes: %w", err)
 						return
 					}
-					log.Info("Processed CommonActor Changes")
 				}()
 
 				grp.Wait()
@@ -214,7 +205,7 @@ func (p *Processor) Start(ctx context.Context) {
 				if err := p.refreshViews(); err != nil {
 					log.Errorw("Failed to refresh views", "error", err)
 				}
-				log.Infow("Processed Batch", "duration", time.Since(loopStart).String())
+				log.Infow("Processed Batch Complete", "duration", time.Since(loopStart).String())
 			}
 		}
 	}()

cmd/lotus-chainwatch/processor/reward.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"time"
 
-	"golang.org/x/sync/errgroup"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/specs-actors/actors/abi/big"
@@ -13,20 +12,23 @@ import (
 	"github.com/filecoin-project/specs-actors/actors/builtin/reward"
 	"github.com/filecoin-project/specs-actors/actors/util/smoothing"
 
-	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
 type rewardActorInfo struct {
 	common actorInfo
 
-	// expected power in bytes during this epoch
-	baselinePower big.Int
+	cumSumBaselinePower big.Int
+	cumSumRealizedPower big.Int
 
-	// base reward in attofil for each block found during this epoch
-	baseBlockReward big.Int
+	effectiveNetworkTime   int64
+	effectiveBaselinePower big.Int
 
-	epochSmoothingEstimate *smoothing.FilterEstimate
+	newBaselinePower     big.Int
+	newBaseReward        big.Int
+	newSmoothingEstimate *smoothing.FilterEstimate
+
+	totalMinedReward big.Int
 }
 
 func (p *Processor) setupRewards() error {
@@ -36,34 +38,23 @@ func (p *Processor) setupRewards() error {
 	}
 
 	if _, err := tx.Exec(`
-/*
-* captures base block reward per miner per state root and does not
-* include penalties or gas reward
-*/
-create table if not exists base_block_rewards
-(
-	state_root text not null
-		constraint block_rewards_pk
-			primary key,
-	base_block_reward numeric not null
-);
-
 /* captures chain-specific power state for any given stateroot */
-create table if not exists chain_power
+create table if not exists chain_reward
 (
 	state_root text not null
-		constraint chain_power_pk
+		constraint chain_reward_pk
 			primary key,
-	baseline_power text not null
-);
+	cum_sum_baseline text not null,
+	cum_sum_realized text not null,
+	effective_network_time int not null,
+	effective_baseline_power text not null,
 
-create table if not exists reward_smoothing_estimates
-(
-    state_root text not null
-        constraint reward_smoothing_estimates_pk
-        	primary key,
-	position_estimate text not null,
-	velocity_estimate text not null
+	new_baseline_power text not null,
+	new_reward numeric not null,
+	new_reward_smoothed_position_estimate text not null,
+	new_reward_smoothed_velocity_estimate text not null,
+
+	total_mined_reward text not null
 );
 `); err != nil {
 		return err
@@ -113,9 +104,14 @@ func (p *Processor) processRewardActors(ctx context.Context, rewardTips ActorTip
 				return nil, xerrors.Errorf("unmarshal state (@ %s): %w", rw.common.stateroot.String(), err)
 			}
 
-			rw.baseBlockReward = rewardActorState.ThisEpochReward
-			rw.baselinePower = rewardActorState.ThisEpochBaselinePower
-			rw.epochSmoothingEstimate = rewardActorState.ThisEpochRewardSmoothed
+			rw.cumSumBaselinePower = rewardActorState.CumsumBaseline
+			rw.cumSumRealizedPower = rewardActorState.CumsumRealized
+			rw.effectiveNetworkTime = int64(rewardActorState.EffectiveNetworkTime)
+			rw.effectiveBaselinePower = rewardActorState.EffectiveBaselinePower
+			rw.newBaselinePower = rewardActorState.ThisEpochBaselinePower
+			rw.newBaseReward = rewardActorState.ThisEpochReward
+			rw.newSmoothingEstimate = rewardActorState.ThisEpochRewardSmoothed
+			rw.totalMinedReward = rewardActorState.TotalMined
 			out = append(out, rw)
 		}
 	}
@@ -145,8 +141,14 @@ func (p *Processor) processRewardActors(ctx context.Context, rewardTips ActorTip
 			return nil, err
 		}
 
-		rw.baseBlockReward = rewardActorState.ThisEpochReward
-		rw.baselinePower = rewardActorState.ThisEpochBaselinePower
+		rw.cumSumBaselinePower = rewardActorState.CumsumBaseline
+		rw.cumSumRealizedPower = rewardActorState.CumsumRealized
+		rw.effectiveNetworkTime = int64(rewardActorState.EffectiveNetworkTime)
+		rw.effectiveBaselinePower = rewardActorState.EffectiveBaselinePower
+		rw.newBaselinePower = rewardActorState.ThisEpochBaselinePower
+		rw.newBaseReward = rewardActorState.ThisEpochReward
+		rw.newSmoothingEstimate = rewardActorState.ThisEpochRewardSmoothed
+		rw.totalMinedReward = rewardActorState.TotalMined
 		out = append(out, rw)
 	}
 
@@ -159,149 +161,47 @@ func (p *Processor) persistRewardActors(ctx context.Context, rewards []rewardAct
 		log.Debugw("Persisted Reward Actors", "duration", time.Since(start).String())
 	}()
 
-	grp, ctx := errgroup.WithContext(ctx) //nolint
-
-	grp.Go(func() error {
-		if err := p.storeChainPower(rewards); err != nil {
-			return err
-		}
-		return nil
-	})
-
-	grp.Go(func() error {
-		if err := p.storeBaseBlockReward(rewards); err != nil {
-			return err
-		}
-		return nil
-	})
-
-	grp.Go(func() error {
-		if err := p.storeRewardSmoothingEstimates(rewards); err != nil {
-			return err
-		}
-		return nil
-	})
-
-	return grp.Wait()
-}
-
-func (p *Processor) storeChainPower(rewards []rewardActorInfo) error {
 	tx, err := p.db.Begin()
 	if err != nil {
-		return xerrors.Errorf("begin chain_power tx: %w", err)
+		return xerrors.Errorf("begin chain_reward tx: %w", err)
 	}
 
-	if _, err := tx.Exec(`create temp table cp (like chain_power excluding constraints) on commit drop`); err != nil {
-		return xerrors.Errorf("prep chain_power temp: %w", err)
+	if _, err := tx.Exec(`create temp table cr (like chain_reward excluding constraints) on commit drop`); err != nil {
+		return xerrors.Errorf("prep chain_reward temp: %w", err)
 	}
 
-	stmt, err := tx.Prepare(`copy cp (state_root, baseline_power) from STDIN`)
+	stmt, err := tx.Prepare(`copy cr ( state_root, cum_sum_baseline, cum_sum_realized, effective_network_time, effective_baseline_power, new_baseline_power, new_reward, new_reward_smoothed_position_estimate, new_reward_smoothed_velocity_estimate, total_mined_reward) from STDIN`)
 	if err != nil {
-		return xerrors.Errorf("prepare tmp chain_power: %w", err)
+		return xerrors.Errorf("prepare tmp chain_reward: %w", err)
 	}
 
 	for _, rewardState := range rewards {
 		if _, err := stmt.Exec(
 			rewardState.common.stateroot.String(),
-			rewardState.baselinePower.String(),
+			rewardState.cumSumBaselinePower.String(),
+			rewardState.cumSumRealizedPower.String(),
+			rewardState.effectiveNetworkTime,
+			rewardState.effectiveBaselinePower.String(),
+			rewardState.newBaselinePower.String(),
+			rewardState.newBaseReward.String(),
+			rewardState.newSmoothingEstimate.PositionEstimate.String(),
+			rewardState.newSmoothingEstimate.VelocityEstimate.String(),
+			rewardState.totalMinedReward.String(),
 		); err != nil {
 			log.Errorw("failed to store chain power", "state_root", rewardState.common.stateroot, "error", err)
 		}
 	}
 
 	if err := stmt.Close(); err != nil {
-		return xerrors.Errorf("close prepared chain_power: %w", err)
+		return xerrors.Errorf("close prepared chain_reward: %w", err)
 	}
 
-	if _, err := tx.Exec(`insert into chain_power select * from cp on conflict do nothing`); err != nil {
-		return xerrors.Errorf("insert chain_power from tmp: %w", err)
+	if _, err := tx.Exec(`insert into chain_reward select * from cr on conflict do nothing`); err != nil {
+		return xerrors.Errorf("insert chain_reward from tmp: %w", err)
 	}
 
 	if err := tx.Commit(); err != nil {
-		return xerrors.Errorf("commit chain_power tx: %w", err)
-	}
-
-	return nil
-}
-
-func (p *Processor) storeBaseBlockReward(rewards []rewardActorInfo) error {
-	tx, err := p.db.Begin()
-	if err != nil {
-		return xerrors.Errorf("begin base_block_reward tx: %w", err)
-	}
-
-	if _, err := tx.Exec(`create temp table bbr (like base_block_rewards excluding constraints) on commit drop`); err != nil {
-		return xerrors.Errorf("prep base_block_reward temp: %w", err)
-	}
-
-	stmt, err := tx.Prepare(`copy bbr (state_root, base_block_reward) from STDIN`)
-	if err != nil {
-		return xerrors.Errorf("prepare tmp base_block_reward: %w", err)
-	}
-
-	for _, rewardState := range rewards {
-		baseBlockReward := big.Div(rewardState.baseBlockReward, big.NewIntUnsigned(build.BlocksPerEpoch))
-		if _, err := stmt.Exec(
-			rewardState.common.stateroot.String(),
-			baseBlockReward.String(),
-		); err != nil {
-			log.Errorw("failed to store base block reward", "state_root", rewardState.common.stateroot, "error", err)
-		}
-	}
-
-	if err := stmt.Close(); err != nil {
-		return xerrors.Errorf("close prepared base_block_reward: %w", err)
-	}
-
-	if _, err := tx.Exec(`insert into base_block_rewards select * from bbr on conflict do nothing`); err != nil {
-		return xerrors.Errorf("insert base_block_reward from tmp: %w", err)
-	}
-
-	if err := tx.Commit(); err != nil {
-		return xerrors.Errorf("commit base_block_reward tx: %w", err)
-	}
-
-	return nil
-}
-
-func (p *Processor) storeRewardSmoothingEstimates(rewards []rewardActorInfo) error {
-	tx, err := p.db.Begin()
-	if err != nil {
-		return xerrors.Errorf("begin reward_smoothing_estimates tx: %w", err)
-	}
-
-	if _, err := tx.Exec(`create temp table rse (like reward_smoothing_estimates) on commit drop`); err != nil {
-		return xerrors.Errorf("prep reward_smoothing_estimates: %w", err)
-	}
-
-	stmt, err := tx.Prepare(`copy rse (state_root, position_estimate, velocity_estimate) from stdin;`)
-	if err != nil {
-		return xerrors.Errorf("prepare tmp reward_smoothing_estimates: %w", err)
-	}
-
-	for _, rewardState := range rewards {
-		if rewardState.epochSmoothingEstimate == nil {
-			continue
-		}
-		if _, err := stmt.Exec(
-			rewardState.common.stateroot.String(),
-			rewardState.epochSmoothingEstimate.PositionEstimate.String(),
-			rewardState.epochSmoothingEstimate.VelocityEstimate.String(),
-		); err != nil {
-			return xerrors.Errorf("failed to store smoothing estimate: %w", err)
-		}
-	}
-
-	if err := stmt.Close(); err != nil {
-		return xerrors.Errorf("close prepared reward_smoothing_estimates: %w", err)
-	}
-
-	if _, err := tx.Exec(`insert into reward_smoothing_estimates select * from rse on conflict do nothing`); err != nil {
-		return xerrors.Errorf("insert reward_smoothing_estimates from tmp: %w", err)
-	}
-
-	if err := tx.Commit(); err != nil {
-		return xerrors.Errorf("commit reward_smoothing_estimates tx: %w", err)
+		return xerrors.Errorf("commit chain_reward tx: %w", err)
 	}
 
 	return nil

cmd/lotus-chainwatch/run.go

@@ -2,20 +2,25 @@ package main
 
 import (
 	"database/sql"
+	"fmt"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
+	"strings"
 
 	_ "github.com/lib/pq"
 
-	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/go-jsonrpc"
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/lotus/api"
+	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/cmd/lotus-chainwatch/processor"
 	"github.com/filecoin-project/lotus/cmd/lotus-chainwatch/scheduler"
 	"github.com/filecoin-project/lotus/cmd/lotus-chainwatch/syncer"
+	"github.com/filecoin-project/lotus/cmd/lotus-chainwatch/util"
 )
 
 var runCmd = &cli.Command{
@@ -39,9 +44,24 @@ var runCmd = &cli.Command{
 			return err
 		}
 
-		api, closer, err := lcli.GetFullNodeAPI(cctx)
-		if err != nil {
-			return err
+		var api api.FullNode
+		var closer jsonrpc.ClientCloser
+		var err error
+		if tokenMaddr := cctx.String("api"); tokenMaddr != "" {
+			toks := strings.Split(tokenMaddr, ":")
+			if len(toks) != 2 {
+				return fmt.Errorf("invalid api tokens, expected <token>:<maddr>, got: %s", tokenMaddr)
+			}
+
+			api, closer, err = util.GetFullNodeAPIUsingCredentials(cctx.Context, toks[1], toks[0])
+			if err != nil {
+				return err
+			}
+		} else {
+			api, closer, err = lcli.GetFullNodeAPI(cctx)
+			if err != nil {
+				return err
+			}
 		}
 		defer closer()
 		ctx := lcli.ReqContext(cctx)
@@ -70,7 +90,7 @@ var runCmd = &cli.Command{
 		}
 		db.SetMaxOpenConns(1350)
 
-		sync := syncer.NewSyncer(db, api)
+		sync := syncer.NewSyncer(db, api, 1400)
 		sync.Start(ctx)
 
 		proc := processor.NewProcessor(ctx, db, api, maxBatch)

cmd/lotus-chainwatch/scheduler/refresh_top_miners_by_base_reward.go

@@ -3,7 +3,6 @@ package scheduler
 import (
 	"context"
 	"database/sql"
-	"time"
 
 	"golang.org/x/xerrors"
 )
@@ -24,9 +23,9 @@ func setupTopMinerByBaseRewardSchema(ctx context.Context, db *sql.DB) error {
 			with total_rewards_by_miner as (
 				select
 					b.miner,
-					sum(bbr.base_block_reward * b.win_count) as total_reward
+					sum(cr.new_reward * b.win_count) as total_reward
 				from blocks b
-				inner join base_block_rewards bbr on b.parentstateroot = bbr.state_root
+				inner join chain_reward cr on b.parentstateroot = cr.state_root
 				group by 1
 			) select
 				rank() over (order by total_reward desc),
@@ -43,17 +42,17 @@ func setupTopMinerByBaseRewardSchema(ctx context.Context, db *sql.DB) error {
 				b."timestamp"as current_timestamp,
 				max(b.height) as current_height
 			from blocks b
-			join base_block_rewards bbr on b.parentstateroot = bbr.state_root
-			where bbr.base_block_reward is not null
+			join chain_reward cr on b.parentstateroot = cr.state_root
+			where cr.new_reward is not null
 			group by 1
 			order by 1 desc
 			limit 1;
 	`); err != nil {
-		return xerrors.Errorf("create top_miner_by_base_reward views: %w", err)
+		return xerrors.Errorf("create top_miners_by_base_reward views: %w", err)
 	}
 
 	if err := tx.Commit(); err != nil {
-		return xerrors.Errorf("committing top_miner_by_base_reward views; %w", err)
+		return xerrors.Errorf("committing top_miners_by_base_reward views; %w", err)
 	}
 	return nil
 }
@@ -65,11 +64,6 @@ func refreshTopMinerByBaseReward(ctx context.Context, db *sql.DB) error {
 	default:
 	}
 
-	t := time.Now()
-	defer func() {
-		log.Debugw("refresh top_miners_by_base_reward", "duration", time.Since(t).String())
-	}()
-
 	_, err := db.Exec("refresh materialized view top_miners_by_base_reward;")
 	if err != nil {
 		return xerrors.Errorf("refresh top_miners_by_base_reward: %w", err)

cmd/lotus-chainwatch/scheduler/scheduler.go

@@ -40,7 +40,7 @@ func (s *Scheduler) Start(ctx context.Context) {
 
 	go func() {
 		// run once on start after schema has initialized
-		time.Sleep(5 * time.Second)
+		time.Sleep(1 * time.Minute)
 		if err := refreshTopMinerByBaseReward(ctx, s.db); err != nil {
 			log.Errorw("failed to refresh top miner", "error", err)
 		}

cmd/lotus-chainwatch/syncer/sync.go

@@ -23,14 +23,17 @@ var log = logging.Logger("syncer")
 type Syncer struct {
 	db *sql.DB
 
+	lookbackLimit uint64
+
 	headerLk sync.Mutex
 	node     api.FullNode
 }
 
-func NewSyncer(db *sql.DB, node api.FullNode) *Syncer {
+func NewSyncer(db *sql.DB, node api.FullNode, lookbackLimit uint64) *Syncer {
 	return &Syncer{
-		db:   db,
-		node: node,
+		db:            db,
+		node:          node,
+		lookbackLimit: lookbackLimit,
 	}
 }
 
@@ -148,40 +151,34 @@ create index if not exists state_heights_parentstateroot_index
 }
 
 func (s *Syncer) Start(ctx context.Context) {
+	if err := logging.SetLogLevel("syncer", "info"); err != nil {
+		log.Fatal(err)
+	}
 	log.Debug("Starting Syncer")
 
 	if err := s.setupSchemas(); err != nil {
 		log.Fatal(err)
 	}
 
-	// doing the initial sync here lets us avoid the HCCurrent case in the switch
-	head, err := s.node.ChainHead(ctx)
-	if err != nil {
-		log.Fatalw("Failed to get chain head form lotus", "error", err)
-	}
-
-	unsynced, err := s.unsyncedBlocks(ctx, head, time.Unix(0, 0))
-	if err != nil {
-		log.Fatalw("failed to gather unsynced blocks", "error", err)
-	}
-
-	if err := s.storeHeaders(unsynced, true, time.Now()); err != nil {
-		log.Fatalw("failed to store unsynced blocks", "error", err)
-	}
-
 	// continue to keep the block headers table up to date.
 	notifs, err := s.node.ChainNotify(ctx)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	lastSynced := time.Now()
+	// we need to ensure that on a restart we don't reprocess the whole flarping chain
+	blkCID, height, err := s.mostRecentlySyncedBlockHeight()
+	if err != nil {
+		log.Fatalw("failed to find most recently synced block", "error", err)
+	}
+	log.Infow("Found starting point for syncing", "blockCID", blkCID.String(), "height", height)
+	sinceEpoch := uint64(height)
 	go func() {
 		for notif := range notifs {
 			for _, change := range notif {
 				switch change.Type {
 				case store.HCApply:
-					unsynced, err := s.unsyncedBlocks(ctx, change.Val, lastSynced)
+					unsynced, err := s.unsyncedBlocks(ctx, change.Val, sinceEpoch)
 					if err != nil {
 						log.Errorw("failed to gather unsynced blocks", "error", err)
 					}
@@ -194,13 +191,13 @@ func (s *Syncer) Start(ctx context.Context) {
 						continue
 					}
 
-					if err := s.storeHeaders(unsynced, true, lastSynced); err != nil {
+					if err := s.storeHeaders(unsynced, true, time.Now()); err != nil {
 						// so this is pretty bad, need some kind of retry..
 						// for now just log an error and the blocks will be attempted again on next notifi
 						log.Errorw("failed to store unsynced blocks", "error", err)
 					}
 
-					lastSynced = time.Now()
+					sinceEpoch = uint64(change.Val.Height())
 				case store.HCRevert:
 					log.Debug("revert todo")
 				}
@@ -209,12 +206,8 @@ func (s *Syncer) Start(ctx context.Context) {
 	}()
 }
 
-func (s *Syncer) unsyncedBlocks(ctx context.Context, head *types.TipSet, since time.Time) (map[cid.Cid]*types.BlockHeader, error) {
-	// get a list of blocks we have already synced in the past 3 mins. This ensures we aren't returning the entire
-	// table every time.
-	lookback := since.Add(-(time.Minute * 3))
-	log.Debugw("Gathering unsynced blocks", "since", lookback.String())
-	hasList, err := s.syncedBlocks(lookback)
+func (s *Syncer) unsyncedBlocks(ctx context.Context, head *types.TipSet, since uint64) (map[cid.Cid]*types.BlockHeader, error) {
+	hasList, err := s.syncedBlocks(since, s.lookbackLimit)
 	if err != nil {
 		return nil, err
 	}
@@ -257,9 +250,8 @@ func (s *Syncer) unsyncedBlocks(ctx context.Context, head *types.TipSet, since t
 	return toSync, nil
 }
 
-func (s *Syncer) syncedBlocks(timestamp time.Time) (map[cid.Cid]struct{}, error) {
-	// timestamp is used to return a configurable amount of rows based on when they were last added.
-	rws, err := s.db.Query(`select cid FROM blocks_synced where synced_at > $1`, timestamp.Unix())
+func (s *Syncer) syncedBlocks(since, limit uint64) (map[cid.Cid]struct{}, error) {
+	rws, err := s.db.Query(`select bs.cid FROM blocks_synced bs left join blocks b on b.cid = bs.cid where b.height <= $1 and bs.processed_at is not null limit $2`, since, limit)
 	if err != nil {
 		return nil, xerrors.Errorf("Failed to query blocks_synced: %w", err)
 	}
@@ -281,14 +273,43 @@ func (s *Syncer) syncedBlocks(timestamp time.Time) (map[cid.Cid]struct{}, error)
 	return out, nil
 }
 
+func (s *Syncer) mostRecentlySyncedBlockHeight() (cid.Cid, int64, error) {
+	rw := s.db.QueryRow(`
+select blocks_synced.cid, b.height
+from blocks_synced
+left join blocks b on blocks_synced.cid = b.cid
+where processed_at is not null
+order by height desc
+limit 1
+`)
+
+	var c string
+	var h int64
+	if err := rw.Scan(&c, &h); err != nil {
+		if err == sql.ErrNoRows {
+			return cid.Undef, 0, nil
+		}
+		return cid.Undef, -1, err
+	}
+
+	ci, err := cid.Parse(c)
+	if err != nil {
+		return cid.Undef, -1, err
+	}
+
+	return ci, h, nil
+}
+
 func (s *Syncer) storeCirculatingSupply(ctx context.Context, tipset *types.TipSet) error {
 	supply, err := s.node.StateCirculatingSupply(ctx, tipset.Key())
 	if err != nil {
 		return err
 	}
 
-	ceInsert := `insert into chain_economics (parent_state_root, circulating_fil, vested_fil, mined_fil, burnt_fil, locked_fil)` +
-		`values ('%s', '%s', '%s', '%s', '%s', '%s');`
+	ceInsert := `insert into chain_economics (parent_state_root, circulating_fil, vested_fil, mined_fil, burnt_fil, locked_fil) ` +
+		`values ('%s', '%s', '%s', '%s', '%s', '%s') on conflict on constraint chain_economics_pk do ` +
+		`update set (circulating_fil, vested_fil, mined_fil, burnt_fil, locked_fil) = ('%[2]s', '%[3]s', '%[4]s', '%[5]s', '%[6]s') ` +
+		`where chain_economics.parent_state_root = '%[1]s';`
 
 	if _, err := s.db.Exec(fmt.Sprintf(ceInsert,
 		tipset.ParentState().String(),

cmd/lotus-chainwatch/util/api.go

@@ -0,0 +1,34 @@
+package util
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/filecoin-project/go-jsonrpc"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/api/client"
+	ma "github.com/multiformats/go-multiaddr"
+	manet "github.com/multiformats/go-multiaddr/net"
+)
+
+func GetFullNodeAPIUsingCredentials(ctx context.Context, listenAddr, token string) (api.FullNode, jsonrpc.ClientCloser, error) {
+	parsedAddr, err := ma.NewMultiaddr(listenAddr)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	_, addr, err := manet.DialArgs(parsedAddr)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return client.NewFullNodeRPC(ctx, apiURI(addr), apiHeaders(token))
+}
+func apiURI(addr string) string {
+	return "ws://" + addr + "/rpc/v0"
+}
+func apiHeaders(token string) http.Header {
+	headers := http.Header{}
+	headers.Add("Authorization", "Bearer "+token)
+	return headers
+}

cmd/lotus-fountain/main.go

@@ -1,77 +1,27 @@
 package main
 
 import (
-	"bytes"
 	"context"
 	"fmt"
-	"html/template"
-	"io"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
-	"sort"
-	"strconv"
 	"time"
 
 	rice "github.com/GeertJohan/go.rice"
-	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log/v2"
-	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
-	"github.com/filecoin-project/specs-actors/actors/builtin"
-	"github.com/filecoin-project/specs-actors/actors/builtin/power"
-
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
-	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 )
 
 var log = logging.Logger("main")
 
-var supportedSectors struct {
-	SectorSizes []struct {
-		Name    string
-		Value   uint64
-		Default bool
-	}
-}
-
-func init() {
-	for supportedSector := range miner.SupportedProofTypes {
-		sectorSize, err := supportedSector.SectorSize()
-		if err != nil {
-			panic(err)
-		}
-
-		supportedSectors.SectorSizes = append(supportedSectors.SectorSizes, struct {
-			Name    string
-			Value   uint64
-			Default bool
-		}{
-			Name:    sectorSize.ShortString(),
-			Value:   uint64(sectorSize),
-			Default: false,
-		})
-
-	}
-
-	sort.Slice(supportedSectors.SectorSizes[:], func(i, j int) bool {
-		return supportedSectors.SectorSizes[i].Value < supportedSectors.SectorSizes[j].Value
-	})
-
-	supportedSectors.SectorSizes[0].Default = true
-}
-
 func main() {
 	logging.SetLogLevel("*", "INFO")
 
@@ -144,11 +94,6 @@ var runCmd = &cli.Command{
 			return xerrors.Errorf("parsing source address (provide correct --from flag!): %w", err)
 		}
 
-		defaultMinerPeer, err := peer.Decode("12D3KooWJpBNhwgvoZ15EB1JwRTRpxgM9D2fwq6eEktrJJG74aP6")
-		if err != nil {
-			return err
-		}
-
 		h := &handler{
 			ctx:            ctx,
 			api:            nodeApi,
@@ -162,23 +107,10 @@ var runCmd = &cli.Command{
 				WalletRate:  15 * time.Minute,
 				WalletBurst: 2,
 			}),
-			minerLimiter: NewLimiter(LimiterConfig{
-				TotalRate:   500 * time.Millisecond,
-				TotalBurst:  build.BlockMessageLimit,
-				IPRate:      10 * time.Minute,
-				IPBurst:     2,
-				WalletRate:  1 * time.Hour,
-				WalletBurst: 2,
-			}),
-			defaultMinerPeer: defaultMinerPeer,
 		}
 
 		http.Handle("/", http.FileServer(rice.MustFindBox("site").HTTPBox()))
-		http.HandleFunc("/miner.html", h.minerhtml)
 		http.HandleFunc("/send", h.send)
-		http.HandleFunc("/mkminer", h.mkminer)
-		http.HandleFunc("/msgwait", h.msgwait)
-		http.HandleFunc("/msgwaitaddr", h.msgwaitaddr)
 
 		fmt.Printf("Open http://%s\n", cctx.String("front"))
 
@@ -198,41 +130,7 @@ type handler struct {
 	from           address.Address
 	sendPerRequest types.FIL
 
-	limiter      *Limiter
-	minerLimiter *Limiter
-
-	defaultMinerPeer peer.ID
-}
-
-func (h *handler) minerhtml(w http.ResponseWriter, r *http.Request) {
-	f, err := rice.MustFindBox("site").Open("_miner.html")
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-
-	tmpl, err := ioutil.ReadAll(f)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-
-	var executedTmpl bytes.Buffer
-
-	t, err := template.New("miner.html").Parse(string(tmpl))
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-	}
-	if err := t.Execute(&executedTmpl, supportedSectors); err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-
-	if _, err := io.Copy(w, &executedTmpl); err != nil {
-		log.Errorf("failed to write template to string %s", err)
-	}
-
-	return
+	limiter *Limiter
 }
 
 func (h *handler) send(w http.ResponseWriter, r *http.Request) {
@@ -287,150 +185,3 @@ func (h *handler) send(w http.ResponseWriter, r *http.Request) {
 
 	_, _ = w.Write([]byte(smsg.Cid().String()))
 }
-
-func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) {
-	owner, err := address.NewFromString(r.FormValue("address"))
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	if owner.Protocol() != address.BLS {
-		http.Error(w,
-			"Miner address must use BLS. A BLS address starts with the prefix 't3'."+
-				"Please create a BLS address by running \"lotus wallet new bls\" while connected to a Lotus node.",
-			http.StatusBadRequest)
-		return
-	}
-
-	ssize, err := strconv.ParseInt(r.FormValue("sectorSize"), 10, 64)
-	if err != nil {
-		return
-	}
-
-	log.Infof("%s: create actor start", owner)
-
-	// Limit based on wallet address
-	limiter := h.minerLimiter.GetWalletLimiter(owner.String())
-	if !limiter.Allow() {
-		http.Error(w, http.StatusText(http.StatusTooManyRequests)+": wallet limit", http.StatusTooManyRequests)
-		return
-	}
-
-	// Limit based on IP
-	reqIP := r.Header.Get("X-Real-IP")
-	if reqIP == "" {
-		h, _, err := net.SplitHostPort(r.RemoteAddr)
-		if err != nil {
-			log.Errorf("could not get ip from: %s, err: %s", r.RemoteAddr, err)
-		}
-		reqIP = h
-	}
-	limiter = h.minerLimiter.GetIPLimiter(reqIP)
-	if !limiter.Allow() {
-		http.Error(w, http.StatusText(http.StatusTooManyRequests)+": IP limit", http.StatusTooManyRequests)
-		return
-	}
-
-	// General limiter owner allow throttling all messages that can make it into the mpool
-	if !h.minerLimiter.Allow() {
-		http.Error(w, http.StatusText(http.StatusTooManyRequests)+": global limit", http.StatusTooManyRequests)
-		return
-	}
-
-	smsg, err := h.api.MpoolPushMessage(h.ctx, &types.Message{
-		Value: types.BigInt(h.sendPerRequest),
-		From:  h.from,
-		To:    owner,
-	}, nil)
-	if err != nil {
-		http.Error(w, "pushfunds: "+err.Error(), http.StatusBadRequest)
-		return
-	}
-	log.Infof("%s: push funds %s", owner, smsg.Cid())
-
-	spt, err := ffiwrapper.SealProofTypeFromSectorSize(abi.SectorSize(ssize))
-	if err != nil {
-		http.Error(w, "sealprooftype: "+err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	params, err := actors.SerializeParams(&power.CreateMinerParams{
-		Owner:         owner,
-		Worker:        owner,
-		SealProofType: spt,
-		Peer:          abi.PeerID(h.defaultMinerPeer),
-	})
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	createStorageMinerMsg := &types.Message{
-		To:    builtin.StoragePowerActorAddr,
-		From:  h.from,
-		Value: big.Zero(),
-
-		Method: builtin.MethodsPower.CreateMiner,
-		Params: params,
-	}
-
-	signed, err := h.api.MpoolPushMessage(r.Context(), createStorageMinerMsg, nil)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	log.Infof("%s: create miner msg: %s", owner, signed.Cid())
-
-	http.Redirect(w, r, fmt.Sprintf("/wait.html?f=%s&m=%s&o=%s", signed.Cid(), smsg.Cid(), owner), http.StatusSeeOther)
-}
-
-func (h *handler) msgwait(w http.ResponseWriter, r *http.Request) {
-	c, err := cid.Parse(r.FormValue("cid"))
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	mw, err := h.api.StateWaitMsg(r.Context(), c, build.MessageConfidence)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	if mw.Receipt.ExitCode != 0 {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-	w.WriteHeader(http.StatusOK)
-}
-
-func (h *handler) msgwaitaddr(w http.ResponseWriter, r *http.Request) {
-	c, err := cid.Parse(r.FormValue("cid"))
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	mw, err := h.api.StateWaitMsg(r.Context(), c, build.MessageConfidence)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	if mw.Receipt.ExitCode != 0 {
-		http.Error(w, xerrors.Errorf("create miner failed: exit code %d", mw.Receipt.ExitCode).Error(), http.StatusBadRequest)
-		return
-	}
-	w.WriteHeader(http.StatusOK)
-
-	var ma power.CreateMinerReturn
-	if err := ma.UnmarshalCBOR(bytes.NewReader(mw.Receipt.Return)); err != nil {
-		log.Errorf("%w", err)
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	fmt.Fprintf(w, "{\"addr\": \"%s\"}", ma.IDAddress)
-}

cmd/lotus-fountain/site/_miner.html

@@ -1,51 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <title>Creating Miner - Lotus Fountain</title>
-    <link rel="stylesheet" type="text/css" href="main.css">
-</head>
-<body>
-<div class="Index">
-    <div class="Index-nodes">
-        <div class="Index-node">
-            [CREATING MINER]
-        </div>
-        <div class="Index-node" id="formnd">
-            <form id="f" action='/mkminer' method='POST'>
-                <span>Enter owner/worker address:</span>
-                <input type='text' name='address' style="width: 300px" placeholder="t3...">
-                <select name="sectorSize">
-                    {{range .SectorSizes}}
-                    <option {{if .Default}}selected{{end}} value="{{ .Value }}">{{ .Name }}</option>
-                    {{end}}
-                </select>
-                <button type='submit'>Create Miner</button>
-            </form>
-        </div>
-        <div id="plswait" style="display: none" class="Index-node">
-            <b>Waiting for transaction on chain..</b>
-        </div>
-        <div class="Index-node">
-            <span>When creating miner, DO NOT REFRESH THE PAGE, wait for it to load. This can take more than 5min.</span>
-        </div>
-        <div class="Index-node">
-            <span>If you don't have an owner/worker address, you can create it by following <a target="_blank" href="https://lotu.sh/en+mining#get-started-22083">these instructions</a>.</span>
-        </div>
-    </div>
-    <div class="Index-footer">
-        <div>
-            <a href="index.html">[Back]</a>
-            <span style="float: right">Not dispensing real Filecoin tokens</span>
-        </div>
-    </div>
-</div>
-<script>
-    let f = document.getElementById('f')
-    f.onsubmit = ev => {
-        document.getElementById('plswait').style.display = 'block'
-        document.getElementById('formnd').style.display = 'none'
-    }
-
-</script>
-</body>
-</html>

cmd/lotus-fountain/site/index.html

@@ -13,9 +13,6 @@
         <div class="Index-node">
             <a href="funds.html">[Send Funds]</a>
         </div>
-        <div class="Index-node">
-            <a href="miner.html">[Create Miner]</a>
-        </div>
     </div>
     <div class="Index-footer">
         <div>

cmd/lotus-fountain/site/wait.html

@@ -1,69 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <title>Creating Miner (wait) - Lotus Fountain</title>
-    <link rel="stylesheet" type="text/css" href="main.css">
-</head>
-<body>
-<div class="Index">
-    <div class="Index-nodes">
-        <div class="Index-node">
-            [CREATING MINER]
-        </div>
-        <div class="Index-node">
-            Gas Funds:&nbsp;&nbsp;&nbsp;<span id="fcid"></span> - <span id="fstate">WAIT</span>
-        </div>
-        <div class="Index-node">
-            Miner Actor:&nbsp;<span id="mcid"></span> - <span id="mstate">WAIT</span>
-        </div>
-        <div class="Index-node" style="display: none" id="fwait">
-            New miners address is: <b id="actaddr">t</b>
-        </div>
-        <div class="Index-node" style="display: none" id="mwait">
-            <div style="padding-bottom: 1em">To initialize the miner run the following command:</div>
-            <div style="overflow-x: visible; white-space: nowrap; background: #353500">
-                <code>lotus-miner init --actor=<span id="actaddr2">t</span> --owner=<span id="owner">t3</span></code>
-            </div>
-        </div>
-    </div>
-    <div class="Index-footer">
-        <div>
-            <a href="index.html">[Back]</a>
-            <span style="float: right">Not dispensing real Filecoin tokens</span>
-        </div>
-    </div>
-</div>
-<script>
-  const params = new URLSearchParams(window.location.search);
-
-  const fcid = document.getElementById('fcid')
-  const mcid = document.getElementById('mcid')
-
-  fcid.innerText = params.get('f')
-  mcid.innerText = params.get('m')
-
-  async function waitFunds() {
-    await fetch('/msgwait?cid=' + params.get('f'))
-    document.getElementById('fstate').innerText = "OK"
-
-    document.getElementById('fwait').style.display = 'block'
-  }
-
-  async function waitMiner() {
-    const resp = await fetch('/msgwaitaddr?cid=' + params.get('f'))
-    document.getElementById('mstate').innerText = "OK"
-
-    const addr = (await resp.json()).addr
-
-    document.getElementById('actaddr').innerText = addr
-    document.getElementById('actaddr2').innerText = addr
-    document.getElementById('owner').innerText = params.get('o')
-
-    document.getElementById('mwait').style.display = 'block'
-  }
-
-  waitFunds()
-  waitMiner()
-</script>
-</body>
-</html>

cmd/lotus-pcr/main.go

@@ -146,7 +146,7 @@ var runCmd = &cli.Command{
 		}()
 
 		ctx := context.Background()
-		api, closer, err := stats.GetFullNodeAPI(cctx.String("lotus-path"))
+		api, closer, err := stats.GetFullNodeAPI(cctx.Context, cctx.String("lotus-path"))
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/lotus-stats/main.go

@@ -56,7 +56,7 @@ func main() {
 		height = h
 	}
 
-	api, closer, err := stats.GetFullNodeAPI(repo)
+	api, closer, err := stats.GetFullNodeAPI(ctx, repo)
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/lotus-storage-miner/actor.go

@@ -137,7 +137,7 @@ var actorSetPeeridCmd = &cli.Command{
 
 		ctx := lcli.ReqContext(cctx)
 
-		pid, err := peer.IDFromString(cctx.Args().Get(0))
+		pid, err := peer.Decode(cctx.Args().Get(0))
 		if err != nil {
 			return fmt.Errorf("failed to parse input as a peerId: %w", err)
 		}

cmd/lotus-storage-miner/info.go

@@ -236,8 +236,10 @@ var stateList = []stateMeta{
 	{col: 39, state: "Total"},
 	{col: color.FgGreen, state: sealing.Proving},
 
+	{col: color.FgBlue, state: sealing.Empty},
+	{col: color.FgBlue, state: sealing.WaitDeals},
+
 	{col: color.FgRed, state: sealing.UndefinedSectorState},
-	{col: color.FgYellow, state: sealing.Empty},
 	{col: color.FgYellow, state: sealing.Packing},
 	{col: color.FgYellow, state: sealing.PreCommit1},
 	{col: color.FgYellow, state: sealing.PreCommit2},
@@ -245,9 +247,13 @@ var stateList = []stateMeta{
 	{col: color.FgYellow, state: sealing.PreCommitWait},
 	{col: color.FgYellow, state: sealing.WaitSeed},
 	{col: color.FgYellow, state: sealing.Committing},
+	{col: color.FgYellow, state: sealing.SubmitCommit},
 	{col: color.FgYellow, state: sealing.CommitWait},
 	{col: color.FgYellow, state: sealing.FinalizeSector},
 
+	{col: color.FgCyan, state: sealing.Removing},
+	{col: color.FgCyan, state: sealing.Removed},
+
 	{col: color.FgRed, state: sealing.FailedUnrecoverable},
 	{col: color.FgRed, state: sealing.SealPreCommit1Failed},
 	{col: color.FgRed, state: sealing.SealPreCommit2Failed},
@@ -259,6 +265,9 @@ var stateList = []stateMeta{
 	{col: color.FgRed, state: sealing.Faulty},
 	{col: color.FgRed, state: sealing.FaultReported},
 	{col: color.FgRed, state: sealing.FaultedFinal},
+	{col: color.FgRed, state: sealing.RemoveFailed},
+	{col: color.FgRed, state: sealing.DealsExpired},
+	{col: color.FgRed, state: sealing.RecoverDealIDs},
 }
 
 func init() {

cmd/lotus-storage-miner/run.go

@@ -10,7 +10,7 @@ import (
 
 	mux "github.com/gorilla/mux"
 	"github.com/multiformats/go-multiaddr"
-	manet "github.com/multiformats/go-multiaddr-net"
+	manet "github.com/multiformats/go-multiaddr/net"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 

cmd/lotus-storage-miner/sealing.go

@@ -155,6 +155,9 @@ var sealingJobsCmd = &cli.Command{
 
 		// oldest first
 		sort.Slice(lines, func(i, j int) bool {
+			if lines[i].RunWait != lines[j].RunWait {
+				return lines[i].RunWait < lines[j].RunWait
+			}
 			return lines[i].Start.Before(lines[j].Start)
 		})
 
@@ -170,10 +173,14 @@ var sealingJobsCmd = &cli.Command{
 		}
 
 		tw := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0)
-		_, _ = fmt.Fprintf(tw, "ID\tSector\tWorker\tHostname\tTask\tTime\n")
+		_, _ = fmt.Fprintf(tw, "ID\tSector\tWorker\tHostname\tTask\tState\tTime\n")
 
 		for _, l := range lines {
-			_, _ = fmt.Fprintf(tw, "%d\t%d\t%d\t%s\t%s\t%s\n", l.ID, l.Sector.Number, l.wid, workerHostnames[l.wid], l.Task.Short(), time.Now().Sub(l.Start).Truncate(time.Millisecond*100))
+			state := "running"
+			if l.RunWait != 0 {
+				state = fmt.Sprintf("assigned(%d)", l.RunWait-1)
+			}
+			_, _ = fmt.Fprintf(tw, "%d\t%d\t%d\t%s\t%s\t%s\t%s\n", l.ID, l.Sector.Number, l.wid, workerHostnames[l.wid], l.Task.Short(), state, time.Now().Sub(l.Start).Truncate(time.Millisecond*100))
 		}
 
 		return tw.Flush()

cmd/lotus-storage-miner/sectors.go

@@ -97,6 +97,8 @@ var sectorsStatusCmd = &cli.Command{
 		fmt.Printf("TicketH:\t%d\n", status.Ticket.Epoch)
 		fmt.Printf("Seed:\t\t%x\n", status.Seed.Value)
 		fmt.Printf("SeedH:\t\t%d\n", status.Seed.Epoch)
+		fmt.Printf("Precommit:\t%s\n", status.PreCommitMsg)
+		fmt.Printf("Commit:\t\t%s\n", status.CommitMsg)
 		fmt.Printf("Proof:\t\t%x\n", status.Proof)
 		fmt.Printf("Deals:\t\t%v\n", status.Deals)
 		fmt.Printf("Retries:\t%d\n", status.Retries)

cmd/lotus/daemon.go

@@ -345,6 +345,16 @@ func ImportChain(r repo.Repo, fname string) error {
 		return xerrors.Errorf("importing chain failed: %w", err)
 	}
 
+	gb, err := cst.GetTipsetByHeight(context.TODO(), 0, ts, true)
+	if err != nil {
+		return err
+	}
+
+	err = cst.SetGenesis(gb.Blocks()[0])
+	if err != nil {
+		return err
+	}
+
 	stm := stmgr.NewStateManager(cst)
 
 	log.Infof("validating imported chain...")

cmd/lotus/debug_advance.go

@@ -60,7 +60,9 @@ func init() {
 				}
 
 			}
-			// TODO: beacon
+
+			mbi, err := api.MinerGetBaseInfo(ctx, addr, head.Height()+1, head.Key())
+
 			ep := &types.ElectionProof{}
 			ep.WinCount = ep.ComputeWinCount(types.NewInt(1), types.NewInt(1))
 			for ep.WinCount == 0 {
@@ -75,7 +77,7 @@ func init() {
 			uts := head.MinTimestamp() + uint64(build.BlockDelaySecs)
 			nheight := head.Height() + 1
 			blk, err := api.MinerCreateBlock(ctx, &lapi.BlockTemplate{
-				addr, head.Key(), ticket, ep, nil, msgs, nheight, uts, gen.ValidWpostForTesting,
+				addr, head.Key(), ticket, ep, mbi.BeaconEntries, msgs, nheight, uts, gen.ValidWpostForTesting,
 			})
 			if err != nil {
 				return xerrors.Errorf("creating block: %w", err)

cmd/lotus/rpc.go

@@ -12,7 +12,7 @@ import (
 	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/multiformats/go-multiaddr"
-	manet "github.com/multiformats/go-multiaddr-net"
+	manet "github.com/multiformats/go-multiaddr/net"
 	"golang.org/x/xerrors"
 
 	"contrib.go.opencensus.io/exporter/prometheus"

conformance/driver.go

@@ -0,0 +1,71 @@
+package conformance
+
+import (
+	"context"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/vm"
+	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
+	"github.com/filecoin-project/lotus/lib/blockstore"
+
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/puppet"
+
+	"github.com/filecoin-project/test-vectors/chaos"
+	"github.com/filecoin-project/test-vectors/schema"
+
+	"github.com/ipfs/go-cid"
+)
+
+var (
+	// BaseFee to use in the VM.
+	// TODO make parametrisable through vector.
+	BaseFee = abi.NewTokenAmount(100)
+)
+
+type Driver struct {
+	ctx    context.Context
+	vector *schema.TestVector
+}
+
+func NewDriver(ctx context.Context, vector *schema.TestVector) *Driver {
+	return &Driver{ctx: ctx, vector: vector}
+}
+
+// ExecuteMessage executes a conformance test vector message in a temporary VM.
+func (d *Driver) ExecuteMessage(msg *types.Message, preroot cid.Cid, bs blockstore.Blockstore, epoch abi.ChainEpoch) (*vm.ApplyRet, cid.Cid, error) {
+	vmOpts := &vm.VMOpts{
+		StateBase:      preroot,
+		Epoch:          epoch,
+		Rand:           &testRand{}, // TODO always succeeds; need more flexibility.
+		Bstore:         bs,
+		Syscalls:       mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility.
+		CircSupplyCalc: nil,
+		BaseFee:        BaseFee,
+	}
+
+	lvm, err := vm.NewVM(vmOpts)
+	if err != nil {
+		return nil, cid.Undef, err
+	}
+
+	invoker := vm.NewInvoker()
+
+	// add support for the puppet and chaos actors.
+	if puppetOn, ok := d.vector.Selector["puppet_actor"]; ok && puppetOn == "true" {
+		invoker.Register(puppet.PuppetActorCodeID, puppet.Actor{}, puppet.State{})
+	}
+	if chaosOn, ok := d.vector.Selector["chaos_actor"]; ok && chaosOn == "true" {
+		invoker.Register(chaos.ChaosActorCodeCID, chaos.Actor{}, chaos.State{})
+	}
+
+	lvm.SetInvoker(invoker)
+
+	ret, err := lvm.ApplyMessage(d.ctx, msg)
+	if err != nil {
+		return nil, cid.Undef, err
+	}
+
+	root, err := lvm.Flush(d.ctx)
+	return ret, root, err
+}

conformance/runner_test.go

@@ -0,0 +1,234 @@
+package conformance
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/vm"
+	"github.com/filecoin-project/lotus/lib/blockstore"
+
+	"github.com/filecoin-project/statediff"
+	"github.com/filecoin-project/test-vectors/schema"
+
+	"github.com/fatih/color"
+	"github.com/ipld/go-car"
+)
+
+const (
+	// EnvSkipConformance, if 1, skips the conformance test suite.
+	EnvSkipConformance = "SKIP_CONFORMANCE"
+
+	// EnvCorpusRootDir is the name of the environment variable where the path
+	// to an alternative corpus location can be provided.
+	//
+	// The default is defaultCorpusRoot.
+	EnvCorpusRootDir = "CORPUS_DIR"
+
+	// defaultCorpusRoot is the directory where the test vector corpus is hosted.
+	// It is mounted on the Lotus repo as a git submodule.
+	//
+	// When running this test, the corpus root can be overridden through the
+	// -conformance.corpus CLI flag to run an alternate corpus.
+	defaultCorpusRoot = "../extern/test-vectors/corpus"
+)
+
+// ignore is a set of paths relative to root to skip.
+var ignore = map[string]struct{}{
+	".git":        {},
+	"schema.json": {},
+}
+
+// TestConformance is the entrypoint test that runs all test vectors found
+// in the corpus root directory.
+//
+// It locates all json files via a recursive walk, skipping over the ignore set,
+// as well as files beginning with _. It parses each file as a test vector, and
+// runs it via the Driver.
+func TestConformance(t *testing.T) {
+	if skip := strings.TrimSpace(os.Getenv(EnvSkipConformance)); skip == "1" {
+		t.SkipNow()
+	}
+	// corpusRoot is the effective corpus root path, taken from the `-conformance.corpus` CLI flag,
+	// falling back to defaultCorpusRoot if not provided.
+	corpusRoot := defaultCorpusRoot
+	if dir := strings.TrimSpace(os.Getenv(EnvCorpusRootDir)); dir != "" {
+		corpusRoot = dir
+	}
+
+	var vectors []string
+	err := filepath.Walk(corpusRoot+"/", func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		filename := filepath.Base(path)
+		rel, err := filepath.Rel(corpusRoot, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if _, ok := ignore[rel]; ok {
+			// skip over using the right error.
+			if info.IsDir() {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if info.IsDir() {
+			// dive into directories.
+			return nil
+		}
+		if filepath.Ext(path) != ".json" {
+			// skip if not .json.
+			return nil
+		}
+		if ignored := strings.HasPrefix(filename, "_"); ignored {
+			// ignore files starting with _.
+			t.Logf("ignoring: %s", rel)
+			return nil
+		}
+		vectors = append(vectors, rel)
+		return nil
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(vectors) == 0 {
+		t.Fatalf("no test vectors found")
+	}
+
+	// Run a test for each vector.
+	for _, v := range vectors {
+		path := filepath.Join(corpusRoot, v)
+		raw, err := ioutil.ReadFile(path)
+		if err != nil {
+			t.Fatalf("failed to read test raw file: %s", path)
+		}
+
+		var vector schema.TestVector
+		err = json.Unmarshal(raw, &vector)
+		if err != nil {
+			t.Errorf("failed to parse test vector %s: %s; skipping", path, err)
+			continue
+		}
+
+		t.Run(v, func(t *testing.T) {
+			for _, h := range vector.Hints {
+				if h == schema.HintIncorrect {
+					t.Logf("skipping vector marked as incorrect: %s", vector.Meta.ID)
+					t.SkipNow()
+				}
+			}
+
+			// dispatch the execution depending on the vector class.
+			switch vector.Class {
+			case "message":
+				executeMessageVector(t, &vector)
+			default:
+				t.Fatalf("test vector class not supported: %s", vector.Class)
+			}
+		})
+	}
+}
+
+// executeMessageVector executes a message-class test vector.
+func executeMessageVector(t *testing.T, vector *schema.TestVector) {
+	var (
+		ctx   = context.Background()
+		epoch = vector.Pre.Epoch
+		root  = vector.Pre.StateTree.RootCID
+	)
+
+	bs := blockstore.NewTemporary()
+
+	// Read the base64-encoded CAR from the vector, and inflate the gzip.
+	buf := bytes.NewReader(vector.CAR)
+	r, err := gzip.NewReader(buf)
+	if err != nil {
+		t.Fatalf("failed to inflate gzipped CAR: %s", err)
+	}
+	defer r.Close() // nolint
+
+	// Load the CAR embedded in the test vector into the Blockstore.
+	_, err = car.LoadCar(bs, r)
+	if err != nil {
+		t.Fatalf("failed to load state tree car from test vector: %s", err)
+	}
+
+	// Create a new Driver.
+	driver := NewDriver(ctx, vector)
+
+	// Apply every message.
+	for i, m := range vector.ApplyMessages {
+		msg, err := types.DecodeMessage(m.Bytes)
+		if err != nil {
+			t.Fatalf("failed to deserialize message: %s", err)
+		}
+
+		// add an epoch if one's set.
+		if m.Epoch != nil {
+			epoch = *m.Epoch
+		}
+
+		// Execute the message.
+		var ret *vm.ApplyRet
+		ret, root, err = driver.ExecuteMessage(msg, root, bs, epoch)
+		if err != nil {
+			t.Fatalf("fatal failure when executing message: %s", err)
+		}
+
+		// Assert that the receipt matches what the test vector expects.
+		receipt := vector.Post.Receipts[i]
+		if expected, actual := receipt.ExitCode, ret.ExitCode; expected != actual {
+			t.Errorf("exit code of msg %d did not match; expected: %s, got: %s", i, expected, actual)
+		}
+		if expected, actual := receipt.GasUsed, ret.GasUsed; expected != actual {
+			t.Errorf("gas used of msg %d did not match; expected: %d, got: %d", i, expected, actual)
+		}
+		if expected, actual := []byte(receipt.ReturnValue), ret.Return; !bytes.Equal(expected, actual) {
+			t.Errorf("return value of msg %d did not match; expected: %s, got: %s", i, base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(actual))
+		}
+	}
+
+	// Once all messages are applied, assert that the final state root matches
+	// the expected postcondition root.
+	if root != vector.Post.StateTree.RootCID {
+		color.NoColor = false // enable colouring.
+
+		t.Errorf("wrong post root cid; expected %v, but got %v", vector.Post.StateTree.RootCID, root)
+
+		var (
+			a  = color.New(color.FgMagenta, color.Bold).Sprint("(A) expected final state")
+			b  = color.New(color.FgYellow, color.Bold).Sprint("(B) actual final state")
+			c  = color.New(color.FgCyan, color.Bold).Sprint("(C) initial state")
+			d1 = color.New(color.FgGreen, color.Bold).Sprint("[Δ1]")
+			d2 = color.New(color.FgGreen, color.Bold).Sprint("[Δ2]")
+			d3 = color.New(color.FgGreen, color.Bold).Sprint("[Δ3]")
+		)
+
+		bold := color.New(color.Bold).SprintfFunc()
+
+		// run state diffs.
+		t.Log(bold("=== dumping 3-way diffs between %s, %s, %s ===", a, b, c))
+
+		t.Log(bold("--- %s left: %s; right: %s ---", d1, a, b))
+		t.Log(statediff.Diff(context.Background(), bs, vector.Post.StateTree.RootCID, root))
+
+		t.Log(bold("--- %s left: %s; right: %s ---", d2, c, b))
+		t.Log(statediff.Diff(context.Background(), bs, vector.Pre.StateTree.RootCID, root))
+
+		t.Log(bold("--- %s left: %s; right: %s ---", d3, c, a))
+		t.Log(statediff.Diff(context.Background(), bs, vector.Pre.StateTree.RootCID, vector.Post.StateTree.RootCID))
+	}
+}

conformance/stubs.go

@@ -0,0 +1,54 @@
+package conformance
+
+import (
+	"context"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/chain/state"
+	"github.com/filecoin-project/lotus/chain/vm"
+
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
+	"github.com/filecoin-project/specs-actors/actors/runtime"
+
+	cbor "github.com/ipfs/go-ipld-cbor"
+)
+
+type testRand struct{}
+
+var _ vm.Rand = (*testRand)(nil)
+
+func (r *testRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
+	return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes.
+}
+
+func (r *testRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
+	return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes.
+}
+
+type testSyscalls struct {
+	runtime.Syscalls
+}
+
+// TODO VerifySignature this will always succeed; but we want to be able to test failures too.
+func (fss *testSyscalls) VerifySignature(_ crypto.Signature, _ address.Address, _ []byte) error {
+	return nil
+}
+
+// TODO VerifySeal this will always succeed; but we want to be able to test failures too.
+func (fss *testSyscalls) VerifySeal(_ abi.SealVerifyInfo) error {
+	return nil
+}
+
+// TODO VerifyPoSt this will always succeed; but we want to be able to test failures too.
+func (fss *testSyscalls) VerifyPoSt(_ abi.WindowPoStVerifyInfo) error {
+	return nil
+}
+
+func mkFakedSigSyscalls(base vm.SyscallBuilder) vm.SyscallBuilder {
+	return func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime.Syscalls {
+		return &testSyscalls{
+			base(ctx, cstate, cst),
+		}
+	}
+}

documentation/en/api-methods.md

@@ -36,6 +36,7 @@
   * [ClientFindData](#ClientFindData)
   * [ClientGenCar](#ClientGenCar)
   * [ClientGetDealInfo](#ClientGetDealInfo)
+  * [ClientGetDealUpdates](#ClientGetDealUpdates)
   * [ClientHasLocal](#ClientHasLocal)
   * [ClientImport](#ClientImport)
   * [ClientListDataTransfers](#ClientListDataTransfers)
@@ -197,7 +198,7 @@ Response:
 ```json
 {
   "Version": "string value",
-  "APIVersion": 3072,
+  "APIVersion": 3328,
   "BlockDelay": 42
 }
 ```
@@ -915,6 +916,42 @@ Response:
 }
 ```
 
+### ClientGetDealUpdates
+ClientGetDealUpdates returns the status of updated deals
+
+
+Perms: read
+
+Inputs: `null`
+
+Response:
+```json
+{
+  "ProposalCid": {
+    "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+  },
+  "State": 42,
+  "Message": "string value",
+  "Provider": "t01234",
+  "DataRef": {
+    "TransferType": "string value",
+    "Root": {
+      "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+    },
+    "PieceCid": null,
+    "PieceSize": 1024
+  },
+  "PieceCID": {
+    "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+  },
+  "Size": 42,
+  "PricePerEpoch": "0",
+  "Duration": 42,
+  "DealID": 5432,
+  "CreationTime": "0001-01-01T00:00:00Z"
+}
+```
+
 ### ClientHasLocal
 ClientHasLocal indicates whether a certain CID is locally stored.
 
@@ -2437,7 +2474,9 @@ Inputs:
       "Type": 2,
       "Data": "Ynl0ZSBhcnJheQ=="
     }
-  }
+  },
+  "Ynl0ZSBhcnJheQ==",
+  "Ynl0ZSBhcnJheQ=="
 ]
 ```
 
@@ -3603,7 +3642,7 @@ Response:
 ```
 
 ### StateSectorGetInfo
-StateSectorGetInfo returns the on-chain info for the specified miner's sector
+StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found
 NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate
 expiration epoch
 

documentation/en/environment-vars.md

@@ -48,6 +48,11 @@ block heights that are very far from the current chain height, you may want to
 increase this.
 
 
+### `LOTUS_BSYNC_MSG_WINDOW`
+
+Set the initial maximum window size for message fetching blocksync requests. If
+you have a slower internet connection and are having trouble syncing, you might
+try lowering this down to 10-20 for a 'poor' internet connection.
 
 ## Lotus Miner
 

documentation/en/mining.md

@@ -33,26 +33,6 @@ With your wallet address:
 - Press the Request button.
 - Run `/lotus-miner init --owner=<blsAddress> --worker=<blsAddress>`
 
-The task will be complete when you see:
-
-```sh
-New miners address is: <YOUR_NEW_MINING_ADDRESS>
-```
-
-## Initialize the miner
-
-In a CLI window, use the following command to start your miner:
-
-```sh
-lotus-miner init --actor=ACTOR_VALUE_RECEIVED --owner=OWNER_VALUE_RECEIVED
-```
-
-Example
-
-```sh
-lotus-miner init --actor=t01424 --owner=t3spmep2xxsl33o4gxk7yjxcobyohzgj3vejzerug25iinbznpzob6a6kexcbeix73th6vjtzfq7boakfdtd6a
-```
-
 You will have to wait some time for this operation to complete.
 
 ## Mining

extern/sector-storage/manager.go

@@ -6,8 +6,7 @@ import (
 	"io"
 	"net/http"
 
-	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
-
+	"github.com/hashicorp/go-multierror"
 	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/mitchellh/go-homedir"
@@ -17,6 +16,7 @@ import (
 	"github.com/filecoin-project/specs-storage/storage"
 
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
+	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
 	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
@@ -463,25 +463,19 @@ func (m *Manager) Remove(ctx context.Context, sector abi.SectorID) error {
 		return xerrors.Errorf("acquiring sector lock: %w", err)
 	}
 
-	unsealed := stores.FTUnsealed
-	{
-		unsealedStores, err := m.index.StorageFindSector(ctx, sector, stores.FTUnsealed, 0, false)
-		if err != nil {
-			return xerrors.Errorf("finding unsealed sector: %w", err)
-		}
+	var err error
 
-		if len(unsealedStores) == 0 { // can be already removed
-			unsealed = stores.FTNone
-		}
+	if rerr := m.storage.Remove(ctx, sector, stores.FTSealed, true); rerr != nil {
+		err = multierror.Append(err, xerrors.Errorf("removing sector (sealed): %w", rerr))
+	}
+	if rerr := m.storage.Remove(ctx, sector, stores.FTCache, true); rerr != nil {
+		err = multierror.Append(err, xerrors.Errorf("removing sector (cache): %w", rerr))
+	}
+	if rerr := m.storage.Remove(ctx, sector, stores.FTUnsealed, true); rerr != nil {
+		err = multierror.Append(err, xerrors.Errorf("removing sector (unsealed): %w", rerr))
 	}
 
-	selector := newExistingSelector(m.index, sector, stores.FTCache|stores.FTSealed, false)
-
-	return m.sched.Schedule(ctx, sector, sealtasks.TTFinalize, selector,
-		schedFetch(sector, stores.FTCache|stores.FTSealed|unsealed, stores.PathStorage, stores.AcquireMove),
-		func(ctx context.Context, w Worker) error {
-			return w.Remove(ctx, sector)
-		})
+	return err
 }
 
 func (m *Manager) StorageLocal(ctx context.Context) (map[stores.ID]string, error) {

extern/sector-storage/mock/mock.go

@@ -3,12 +3,12 @@ package mock
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
 	"fmt"
 	"io"
 	"math/rand"
 	"sync"
 
-	"github.com/filecoin-project/go-bitfield"
 	commcid "github.com/filecoin-project/go-fil-commcid"
 	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-storage/storage"
@@ -291,32 +291,29 @@ func (mgr *SectorMgr) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorI
 	return generateFakePoSt(si, abi.RegisteredSealProof.RegisteredWindowPoStProof, randomness), skipped, nil
 }
 
-func generateFakePoSt(sectorInfo []abi.SectorInfo, rpt func(abi.RegisteredSealProof) (abi.RegisteredPoStProof, error), randomness abi.PoStRandomness) []abi.PoStProof {
-	sectors := bitfield.New()
+func generateFakePoStProof(sectorInfo []abi.SectorInfo, randomness abi.PoStRandomness) []byte {
+	hasher := sha256.New()
+	_, _ = hasher.Write(randomness)
 	for _, info := range sectorInfo {
-		sectors.Set(uint64(info.SectorNumber))
+		err := info.MarshalCBOR(hasher)
+		if err != nil {
+			panic(err)
+		}
 	}
+	return hasher.Sum(nil)
 
+}
+
+func generateFakePoSt(sectorInfo []abi.SectorInfo, rpt func(abi.RegisteredSealProof) (abi.RegisteredPoStProof, error), randomness abi.PoStRandomness) []abi.PoStProof {
 	wp, err := rpt(sectorInfo[0].SealProof)
 	if err != nil {
 		panic(err)
 	}
 
-	var proofBuf bytes.Buffer
-
-	_, err = proofBuf.Write(randomness)
-	if err != nil {
-		panic(err)
-	}
-
-	if err := sectors.MarshalCBOR(&proofBuf); err != nil {
-		panic(err)
-	}
-
 	return []abi.PoStProof{
 		{
 			PoStProof:  wp,
-			ProofBytes: proofBuf.Bytes(),
+			ProofBytes: generateFakePoStProof(sectorInfo, randomness),
 		},
 	}
 }
@@ -412,36 +409,10 @@ func (m mockVerif) VerifyWindowPoSt(ctx context.Context, info abi.WindowPoStVeri
 
 	proof := info.Proofs[0]
 
-	if !bytes.Equal(proof.ProofBytes[:len(info.Randomness)], info.Randomness) {
-		return false, xerrors.Errorf("bad randomness")
+	expected := generateFakePoStProof(info.ChallengedSectors, info.Randomness)
+	if !bytes.Equal(proof.ProofBytes, expected) {
+		return false, xerrors.Errorf("bad proof")
 	}
-
-	sectors := bitfield.New()
-	if err := sectors.UnmarshalCBOR(bytes.NewReader(proof.ProofBytes[len(info.Randomness):])); err != nil {
-		return false, xerrors.Errorf("unmarshaling sectors bitfield from \"proof\": %w", err)
-	}
-
-	challenged := bitfield.New()
-	for _, sector := range info.ChallengedSectors {
-		challenged.Set(uint64(sector.SectorNumber))
-	}
-
-	{
-		b1, err := sectors.MarshalJSON()
-		if err != nil {
-			return false, err
-		}
-
-		b2, err := challenged.MarshalJSON()
-		if err != nil {
-			return false, err
-		}
-
-		if !bytes.Equal(b1, b2) {
-			return false, xerrors.Errorf("proven and challenged sector sets didn't match: %s != %s", string(b1), string(b2))
-		}
-	}
-
 	return true, nil
 }
 

extern/sector-storage/request_queue.go

@@ -7,6 +7,11 @@ type requestQueue []*workerRequest
 func (q requestQueue) Len() int { return len(q) }
 
 func (q requestQueue) Less(i, j int) bool {
+	oneMuchLess, muchLess := q[i].taskType.MuchLess(q[j].taskType)
+	if oneMuchLess {
+		return muchLess
+	}
+
 	if q[i].priority != q[j].priority {
 		return q[i].priority > q[j].priority
 	}

extern/sector-storage/resources.go

@@ -22,17 +22,17 @@ func (r Resources) MultiThread() bool {
 
 var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{
 	sealtasks.TTAddPiece: {
-		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ // This is probably a bit conservative
-			MaxMemory: 64 << 30,
-			MinMemory: 64 << 30,
+		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
+			MaxMemory: 8 << 30,
+			MinMemory: 8 << 30,
 
 			Threads: 1,
 
 			BaseMinMemory: 1 << 30,
 		},
-		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{ // This is probably a bit conservative
-			MaxMemory: 32 << 30,
-			MinMemory: 32 << 30,
+		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
+			MaxMemory: 4 << 30,
+			MinMemory: 4 << 30,
 
 			Threads: 1,
 

extern/sector-storage/sched.go

@@ -21,6 +21,7 @@ type schedPrioCtxKey int
 var SchedPriorityKey schedPrioCtxKey
 var DefaultSchedPriority = 0
 var SelectorTimeout = 5 * time.Second
+var InitWait = 3 * time.Second
 
 var (
 	SchedWindows = 2
@@ -85,6 +86,9 @@ type workerHandle struct {
 
 	lk sync.Mutex
 
+	wndLk         sync.Mutex
+	activeWindows []*schedWindow
+
 	// stats / tracking
 	wt *workTracker
 
@@ -123,6 +127,8 @@ type workerRequest struct {
 	prepare WorkerAction
 	work    WorkerAction
 
+	start time.Time
+
 	index int // The index of the item in the heap.
 
 	indexHeap int
@@ -147,7 +153,7 @@ func newScheduler(spt abi.RegisteredSealProof) *scheduler {
 		workerClosing: make(chan WorkerID),
 
 		schedule:       make(chan *workerRequest),
-		windowRequests: make(chan *schedWindowRequest),
+		windowRequests: make(chan *schedWindowRequest, 20),
 
 		schedQueue: &requestQueue{},
 
@@ -171,6 +177,8 @@ func (sh *scheduler) Schedule(ctx context.Context, sector abi.SectorID, taskType
 		prepare: prepare,
 		work:    work,
 
+		start: time.Now(),
+
 		ret: ret,
 		ctx: ctx,
 	}:
@@ -214,7 +222,12 @@ func (sh *scheduler) runSched() {
 
 	go sh.runWorkerWatcher()
 
+	iw := time.After(InitWait)
+	var initialised bool
+
 	for {
+		var doSched bool
+
 		select {
 		case w := <-sh.newWorkers:
 			sh.newWorker(w)
@@ -224,22 +237,47 @@ func (sh *scheduler) runSched() {
 
 		case req := <-sh.schedule:
 			sh.schedQueue.Push(req)
-			sh.trySched()
+			doSched = true
 
 			if sh.testSync != nil {
 				sh.testSync <- struct{}{}
 			}
 		case req := <-sh.windowRequests:
 			sh.openWindows = append(sh.openWindows, req)
-			sh.trySched()
-
+			doSched = true
 		case ireq := <-sh.info:
 			ireq(sh.diag())
 
+		case <-iw:
+			initialised = true
+			iw = nil
+			doSched = true
 		case <-sh.closing:
 			sh.schedClose()
 			return
 		}
+
+		if doSched && initialised {
+			// First gather any pending tasks, so we go through the scheduling loop
+			// once for every added task
+		loop:
+			for {
+				select {
+				case req := <-sh.schedule:
+					sh.schedQueue.Push(req)
+					if sh.testSync != nil {
+						sh.testSync <- struct{}{}
+					}
+				case req := <-sh.windowRequests:
+					sh.openWindows = append(sh.openWindows, req)
+				default:
+					break loop
+				}
+			}
+
+			sh.trySched()
+		}
+
 	}
 }
 
@@ -321,7 +359,7 @@ func (sh *scheduler) trySched() {
 				}
 
 				// TODO: allow bigger windows
-				if !windows[wnd].allocated.canHandleRequest(needRes, windowRequest.worker, worker.info.Resources) {
+				if !windows[wnd].allocated.canHandleRequest(needRes, windowRequest.worker, "schedAcceptable", worker.info.Resources) {
 					continue
 				}
 
@@ -392,11 +430,11 @@ func (sh *scheduler) trySched() {
 			log.Debugf("SCHED try assign sqi:%d sector %d to window %d", sqi, task.sector.Number, wnd)
 
 			// TODO: allow bigger windows
-			if !windows[wnd].allocated.canHandleRequest(needRes, wid, wr) {
+			if !windows[wnd].allocated.canHandleRequest(needRes, wid, "schedAssign", wr) {
 				continue
 			}
 
-			log.Debugf("SCHED ASSIGNED sqi:%d sector %d to window %d", sqi, task.sector.Number, wnd)
+			log.Debugf("SCHED ASSIGNED sqi:%d sector %d task %s to window %d", sqi, task.sector.Number, task.taskType, wnd)
 
 			windows[wnd].allocated.add(wr, needRes)
 
@@ -475,8 +513,6 @@ func (sh *scheduler) runWorker(wid WorkerID) {
 		taskDone := make(chan struct{}, 1)
 		windowsRequested := 0
 
-		var activeWindows []*schedWindow
-
 		ctx, cancel := context.WithCancel(context.TODO())
 		defer cancel()
 
@@ -510,7 +546,9 @@ func (sh *scheduler) runWorker(wid WorkerID) {
 
 			select {
 			case w := <-scheduledWindows:
-				activeWindows = append(activeWindows, w)
+				worker.wndLk.Lock()
+				worker.activeWindows = append(worker.activeWindows, w)
+				worker.wndLk.Unlock()
 			case <-taskDone:
 				log.Debugw("task done", "workerid", wid)
 			case <-sh.closing:
@@ -521,24 +559,38 @@ func (sh *scheduler) runWorker(wid WorkerID) {
 				return
 			}
 
+			worker.wndLk.Lock()
+
+			windowsRequested -= sh.workerCompactWindows(worker, wid)
+
 		assignLoop:
 			// process windows in order
-			for len(activeWindows) > 0 {
-				// process tasks within a window in order
-				for len(activeWindows[0].todo) > 0 {
-					todo := activeWindows[0].todo[0]
-					needRes := ResourceTable[todo.taskType][sh.spt]
+			for len(worker.activeWindows) > 0 {
+				firstWindow := worker.activeWindows[0]
 
+				// process tasks within a window, preferring tasks at lower indexes
+				for len(firstWindow.todo) > 0 {
 					sh.workersLk.RLock()
+
+					tidx := -1
+
 					worker.lk.Lock()
-					ok := worker.preparing.canHandleRequest(needRes, wid, worker.info.Resources)
+					for t, todo := range firstWindow.todo {
+						needRes := ResourceTable[todo.taskType][sh.spt]
+						if worker.preparing.canHandleRequest(needRes, wid, "startPreparing", worker.info.Resources) {
+							tidx = t
+							break
+						}
+					}
 					worker.lk.Unlock()
 
-					if !ok {
+					if tidx == -1 {
 						sh.workersLk.RUnlock()
 						break assignLoop
 					}
 
+					todo := firstWindow.todo[tidx]
+
 					log.Debugf("assign worker sector %d", todo.sector.Number)
 					err := sh.assignWorker(taskDone, wid, worker, todo)
 					sh.workersLk.RUnlock()
@@ -548,19 +600,76 @@ func (sh *scheduler) runWorker(wid WorkerID) {
 						go todo.respond(xerrors.Errorf("assignWorker error: %w", err))
 					}
 
-					activeWindows[0].todo = activeWindows[0].todo[1:]
+					// Note: we're not freeing window.allocated resources here very much on purpose
+					copy(firstWindow.todo[tidx:], firstWindow.todo[tidx+1:])
+					firstWindow.todo[len(firstWindow.todo)-1] = nil
+					firstWindow.todo = firstWindow.todo[:len(firstWindow.todo)-1]
 				}
 
-				copy(activeWindows, activeWindows[1:])
-				activeWindows[len(activeWindows)-1] = nil
-				activeWindows = activeWindows[:len(activeWindows)-1]
+				copy(worker.activeWindows, worker.activeWindows[1:])
+				worker.activeWindows[len(worker.activeWindows)-1] = nil
+				worker.activeWindows = worker.activeWindows[:len(worker.activeWindows)-1]
 
 				windowsRequested--
 			}
+
+			worker.wndLk.Unlock()
 		}
 	}()
 }
 
+func (sh *scheduler) workerCompactWindows(worker *workerHandle, wid WorkerID) int {
+	// move tasks from older windows to newer windows if older windows
+	// still can fit them
+	if len(worker.activeWindows) > 1 {
+		for wi, window := range worker.activeWindows[1:] {
+			lower := worker.activeWindows[wi]
+			var moved []int
+
+			for ti, todo := range window.todo {
+				needRes := ResourceTable[todo.taskType][sh.spt]
+				if !lower.allocated.canHandleRequest(needRes, wid, "compactWindows", worker.info.Resources) {
+					continue
+				}
+
+				moved = append(moved, ti)
+				lower.todo = append(lower.todo, todo)
+				lower.allocated.add(worker.info.Resources, needRes)
+				window.allocated.free(worker.info.Resources, needRes)
+			}
+
+			if len(moved) > 0 {
+				newTodo := make([]*workerRequest, 0, len(window.todo)-len(moved))
+				for i, t := range window.todo {
+					if len(moved) > 0 && moved[0] == i {
+						moved = moved[1:]
+						continue
+					}
+
+					newTodo = append(newTodo, t)
+				}
+				window.todo = newTodo
+			}
+		}
+	}
+
+	var compacted int
+	var newWindows []*schedWindow
+
+	for _, window := range worker.activeWindows {
+		if len(window.todo) == 0 {
+			compacted++
+			continue
+		}
+
+		newWindows = append(newWindows, window)
+	}
+
+	worker.activeWindows = newWindows
+
+	return compacted
+}
+
 func (sh *scheduler) assignWorker(taskDone chan struct{}, wid WorkerID, w *workerHandle, req *workerRequest) error {
 	needRes := ResourceTable[req.taskType][sh.spt]
 

extern/sector-storage/sched_resources.go

@@ -7,7 +7,7 @@ import (
 )
 
 func (a *activeResources) withResources(id WorkerID, wr storiface.WorkerResources, r Resources, locker sync.Locker, cb func() error) error {
-	for !a.canHandleRequest(r, id, wr) {
+	for !a.canHandleRequest(r, id, "withResources", wr) {
 		if a.cond == nil {
 			a.cond = sync.NewCond(locker)
 		}
@@ -52,37 +52,37 @@ func (a *activeResources) free(wr storiface.WorkerResources, r Resources) {
 	a.memUsedMax -= r.MaxMemory
 }
 
-func (a *activeResources) canHandleRequest(needRes Resources, wid WorkerID, res storiface.WorkerResources) bool {
+func (a *activeResources) canHandleRequest(needRes Resources, wid WorkerID, caller string, res storiface.WorkerResources) bool {
 
 	// TODO: dedupe needRes.BaseMinMemory per task type (don't add if that task is already running)
 	minNeedMem := res.MemReserved + a.memUsedMin + needRes.MinMemory + needRes.BaseMinMemory
 	if minNeedMem > res.MemPhysical {
-		log.Debugf("sched: not scheduling on worker %d; not enough physical memory - need: %dM, have %dM", wid, minNeedMem/mib, res.MemPhysical/mib)
+		log.Debugf("sched: not scheduling on worker %d for %s; not enough physical memory - need: %dM, have %dM", wid, caller, minNeedMem/mib, res.MemPhysical/mib)
 		return false
 	}
 
 	maxNeedMem := res.MemReserved + a.memUsedMax + needRes.MaxMemory + needRes.BaseMinMemory
 
 	if maxNeedMem > res.MemSwap+res.MemPhysical {
-		log.Debugf("sched: not scheduling on worker %d; not enough virtual memory - need: %dM, have %dM", wid, maxNeedMem/mib, (res.MemSwap+res.MemPhysical)/mib)
+		log.Debugf("sched: not scheduling on worker %d for %s; not enough virtual memory - need: %dM, have %dM", wid, caller, maxNeedMem/mib, (res.MemSwap+res.MemPhysical)/mib)
 		return false
 	}
 
 	if needRes.MultiThread() {
 		if a.cpuUse > 0 {
-			log.Debugf("sched: not scheduling on worker %d; multicore process needs %d threads, %d in use, target %d", wid, res.CPUs, a.cpuUse, res.CPUs)
+			log.Debugf("sched: not scheduling on worker %d for %s; multicore process needs %d threads, %d in use, target %d", wid, caller, res.CPUs, a.cpuUse, res.CPUs)
 			return false
 		}
 	} else {
 		if a.cpuUse+uint64(needRes.Threads) > res.CPUs {
-			log.Debugf("sched: not scheduling on worker %d; not enough threads, need %d, %d in use, target %d", wid, needRes.Threads, a.cpuUse, res.CPUs)
+			log.Debugf("sched: not scheduling on worker %d for %s; not enough threads, need %d, %d in use, target %d", wid, caller, needRes.Threads, a.cpuUse, res.CPUs)
 			return false
 		}
 	}
 
 	if len(res.GPUs) > 0 && needRes.CanGPU {
 		if a.gpuUsed {
-			log.Debugf("sched: not scheduling on worker %d; GPU in use", wid)
+			log.Debugf("sched: not scheduling on worker %d for %s; GPU in use", wid, caller)
 			return false
 		}
 	}

extern/sector-storage/sched_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"runtime"
+	"sort"
 	"sync"
 	"testing"
 	"time"
@@ -22,6 +23,10 @@ import (
 	"github.com/filecoin-project/specs-storage/storage"
 )
 
+func init() {
+	InitWait = 10 * time.Millisecond
+}
+
 func TestWithPriority(t *testing.T) {
 	ctx := context.Background()
 
@@ -301,7 +306,8 @@ func TestSched(t *testing.T) {
 				done: map[string]chan struct{}{},
 			}
 
-			for _, task := range tasks {
+			for i, task := range tasks {
+				log.Info("TASK", i)
 				task(t, sched, index, &rm)
 			}
 
@@ -415,6 +421,45 @@ func TestSched(t *testing.T) {
 		)
 	}
 
+	diag := func() task {
+		return func(t *testing.T, s *scheduler, index *stores.Index, meta *runMeta) {
+			time.Sleep(20 * time.Millisecond)
+			for _, request := range s.diag().Requests {
+				log.Infof("!!! sDIAG: sid(%d) task(%s)", request.Sector.Number, request.TaskType)
+			}
+
+			wj := (&Manager{sched: s}).WorkerJobs()
+
+			type line struct {
+				storiface.WorkerJob
+				wid uint64
+			}
+
+			lines := make([]line, 0)
+
+			for wid, jobs := range wj {
+				for _, job := range jobs {
+					lines = append(lines, line{
+						WorkerJob: job,
+						wid:       wid,
+					})
+				}
+			}
+
+			// oldest first
+			sort.Slice(lines, func(i, j int) bool {
+				if lines[i].RunWait != lines[j].RunWait {
+					return lines[i].RunWait < lines[j].RunWait
+				}
+				return lines[i].Start.Before(lines[j].Start)
+			})
+
+			for _, l := range lines {
+				log.Infof("!!! wDIAG: rw(%d) sid(%d) t(%s)", l.RunWait, l.Sector.Number, l.Task)
+			}
+		}
+	}
+
 	// run this one a bunch of times, it had a very annoying tendency to fail randomly
 	for i := 0; i < 40; i++ {
 		t.Run("pc1-pc2-prio", testFunc([]workerSpec{
@@ -423,6 +468,8 @@ func TestSched(t *testing.T) {
 			// fill queues
 			twoPC1("w0", 0, taskStarted),
 			twoPC1("w1", 2, taskNotScheduled),
+			sched("w2", "fred", 4, sealtasks.TTPreCommit1),
+			taskNotScheduled("w2"),
 
 			// windowed
 
@@ -435,10 +482,18 @@ func TestSched(t *testing.T) {
 			sched("t3", "fred", 10, sealtasks.TTPreCommit2),
 			taskNotScheduled("t3"),
 
+			diag(),
+
 			twoPC1Act("w0", taskDone),
 			twoPC1Act("w1", taskStarted),
+			taskNotScheduled("w2"),
 
 			twoPC1Act("w1", taskDone),
+			taskStarted("w2"),
+
+			taskDone("w2"),
+
+			diag(),
 
 			taskStarted("t3"),
 			taskNotScheduled("t1"),
@@ -518,3 +573,73 @@ func BenchmarkTrySched(b *testing.B) {
 	b.Run("1w-500q", test(1, 500))
 	b.Run("200w-400q", test(200, 400))
 }
+
+func TestWindowCompact(t *testing.T) {
+	sh := scheduler{
+		spt: abi.RegisteredSealProof_StackedDrg32GiBV1,
+	}
+
+	test := func(start [][]sealtasks.TaskType, expect [][]sealtasks.TaskType) func(t *testing.T) {
+		return func(t *testing.T) {
+			wh := &workerHandle{
+				info: storiface.WorkerInfo{
+					Resources: decentWorkerResources,
+				},
+			}
+
+			for _, windowTasks := range start {
+				window := &schedWindow{}
+
+				for _, task := range windowTasks {
+					window.todo = append(window.todo, &workerRequest{taskType: task})
+					window.allocated.add(wh.info.Resources, ResourceTable[task][sh.spt])
+				}
+
+				wh.activeWindows = append(wh.activeWindows, window)
+			}
+
+			n := sh.workerCompactWindows(wh, 0)
+			require.Equal(t, len(start)-len(expect), n)
+
+			for wi, tasks := range expect {
+				var expectRes activeResources
+
+				for ti, task := range tasks {
+					require.Equal(t, task, wh.activeWindows[wi].todo[ti].taskType, "%d, %d", wi, ti)
+					expectRes.add(wh.info.Resources, ResourceTable[task][sh.spt])
+				}
+
+				require.Equal(t, expectRes.cpuUse, wh.activeWindows[wi].allocated.cpuUse, "%d", wi)
+				require.Equal(t, expectRes.gpuUsed, wh.activeWindows[wi].allocated.gpuUsed, "%d", wi)
+				require.Equal(t, expectRes.memUsedMin, wh.activeWindows[wi].allocated.memUsedMin, "%d", wi)
+				require.Equal(t, expectRes.memUsedMax, wh.activeWindows[wi].allocated.memUsedMax, "%d", wi)
+			}
+
+		}
+	}
+
+	t.Run("2-pc1-windows", test(
+		[][]sealtasks.TaskType{{sealtasks.TTPreCommit1}, {sealtasks.TTPreCommit1}},
+		[][]sealtasks.TaskType{{sealtasks.TTPreCommit1, sealtasks.TTPreCommit1}}),
+	)
+
+	t.Run("1-window", test(
+		[][]sealtasks.TaskType{{sealtasks.TTPreCommit1, sealtasks.TTPreCommit1}},
+		[][]sealtasks.TaskType{{sealtasks.TTPreCommit1, sealtasks.TTPreCommit1}}),
+	)
+
+	t.Run("2-pc2-windows", test(
+		[][]sealtasks.TaskType{{sealtasks.TTPreCommit2}, {sealtasks.TTPreCommit2}},
+		[][]sealtasks.TaskType{{sealtasks.TTPreCommit2}, {sealtasks.TTPreCommit2}}),
+	)
+
+	t.Run("2pc1-pc1ap", test(
+		[][]sealtasks.TaskType{{sealtasks.TTPreCommit1, sealtasks.TTPreCommit1}, {sealtasks.TTPreCommit1, sealtasks.TTAddPiece}},
+		[][]sealtasks.TaskType{{sealtasks.TTPreCommit1, sealtasks.TTPreCommit1, sealtasks.TTAddPiece}, {sealtasks.TTPreCommit1}}),
+	)
+
+	t.Run("2pc1-pc1appc2", test(
+		[][]sealtasks.TaskType{{sealtasks.TTPreCommit1, sealtasks.TTPreCommit1}, {sealtasks.TTPreCommit1, sealtasks.TTAddPiece, sealtasks.TTPreCommit2}},
+		[][]sealtasks.TaskType{{sealtasks.TTPreCommit1, sealtasks.TTPreCommit1, sealtasks.TTAddPiece}, {sealtasks.TTPreCommit1, sealtasks.TTPreCommit2}}),
+	)
+}

extern/sector-storage/sealtasks/task.go

@@ -17,15 +17,15 @@ const (
 )
 
 var order = map[TaskType]int{
-	TTAddPiece:     7,
-	TTPreCommit1:   6,
-	TTPreCommit2:   5,
-	TTCommit2:      4,
-	TTCommit1:      3,
-	TTFetch:        2,
-	TTFinalize:     1,
-	TTUnseal:       0,
-	TTReadUnsealed: 0,
+	TTAddPiece:     6, // least priority
+	TTPreCommit1:   5,
+	TTPreCommit2:   4,
+	TTCommit2:      3,
+	TTCommit1:      2,
+	TTUnseal:       1,
+	TTFetch:        -1,
+	TTReadUnsealed: -1,
+	TTFinalize:     -2, // most priority
 }
 
 var shortNames = map[TaskType]string{
@@ -43,6 +43,12 @@ var shortNames = map[TaskType]string{
 	TTReadUnsealed: "RD ",
 }
 
+func (a TaskType) MuchLess(b TaskType) (bool, bool) {
+	oa, ob := order[a], order[b]
+	oneNegative := oa^ob < 0
+	return oneNegative, oa < ob
+}
+
 func (a TaskType) Less(b TaskType) bool {
 	return order[a] < order[b]
 }

extern/sector-storage/stats.go

@@ -1,6 +1,8 @@
 package sectorstorage
 
-import "github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+import (
+	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+)
 
 func (m *Manager) WorkerStats() map[uint64]storiface.WorkerStats {
 	m.sched.workersLk.RLock()
@@ -29,6 +31,20 @@ func (m *Manager) WorkerJobs() map[uint64][]storiface.WorkerJob {
 
 	for id, handle := range m.sched.workers {
 		out[uint64(id)] = handle.wt.Running()
+
+		handle.wndLk.Lock()
+		for wi, window := range handle.activeWindows {
+			for _, request := range window.todo {
+				out[uint64(id)] = append(out[uint64(id)], storiface.WorkerJob{
+					ID:      0,
+					Sector:  request.sector,
+					Task:    request.taskType,
+					RunWait: wi + 1,
+					Start:   request.start,
+				})
+			}
+		}
+		handle.wndLk.Unlock()
 	}
 
 	return out

extern/sector-storage/storiface/worker.go

@@ -37,5 +37,6 @@ type WorkerJob struct {
 	Sector abi.SectorID
 	Task   sealtasks.TaskType
 
-	Start time.Time
+	RunWait int // 0 - running, 1+ - assigned
+	Start   time.Time
 }

extern/storage-sealing/cbor_gen.go

@@ -135,12 +135,34 @@ func (t *DealInfo) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{163}); err != nil {
+	if _, err := w.Write([]byte{164}); err != nil {
 		return err
 	}
 
 	scratch := make([]byte, 9)
 
+	// t.PublishCid (cid.Cid) (struct)
+	if len("PublishCid") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"PublishCid\" was too long")
+	}
+
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PublishCid"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("PublishCid")); err != nil {
+		return err
+	}
+
+	if t.PublishCid == nil {
+		if _, err := w.Write(cbg.CborNull); err != nil {
+			return err
+		}
+	} else {
+		if err := cbg.WriteCidBuf(scratch, w, *t.PublishCid); err != nil {
+			return xerrors.Errorf("failed to write cid field t.PublishCid: %w", err)
+		}
+	}
+
 	// t.DealID (abi.DealID) (uint64)
 	if len("DealID") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"DealID\" was too long")
@@ -224,7 +246,30 @@ func (t *DealInfo) UnmarshalCBOR(r io.Reader) error {
 		}
 
 		switch name {
-		// t.DealID (abi.DealID) (uint64)
+		// t.PublishCid (cid.Cid) (struct)
+		case "PublishCid":
+
+			{
+
+				b, err := br.ReadByte()
+				if err != nil {
+					return err
+				}
+				if b != cbg.CborNull[0] {
+					if err := br.UnreadByte(); err != nil {
+						return err
+					}
+
+					c, err := cbg.ReadCid(br)
+					if err != nil {
+						return xerrors.Errorf("failed to read cid field t.PublishCid: %w", err)
+					}
+
+					t.PublishCid = &c
+				}
+
+			}
+			// t.DealID (abi.DealID) (uint64)
 		case "DealID":
 
 			{
@@ -430,7 +475,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{182}); err != nil {
+	if _, err := w.Write([]byte{183}); err != nil {
 		return err
 	}
 
@@ -860,6 +905,29 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error {
 		}
 	}
 
+	// t.Return (sealing.ReturnState) (string)
+	if len("Return") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"Return\" was too long")
+	}
+
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Return"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("Return")); err != nil {
+		return err
+	}
+
+	if len(t.Return) > cbg.MaxLength {
+		return xerrors.Errorf("Value in field t.Return was too long")
+	}
+
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Return))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string(t.Return)); err != nil {
+		return err
+	}
+
 	// t.LastErr (string) (string)
 	if len("LastErr") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"LastErr\" was too long")
@@ -1362,6 +1430,17 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error {
 				}
 
 			}
+			// t.Return (sealing.ReturnState) (string)
+		case "Return":
+
+			{
+				sval, err := cbg.ReadStringBuf(br, scratch)
+				if err != nil {
+					return err
+				}
+
+				t.Return = ReturnState(sval)
+			}
 			// t.LastErr (string) (string)
 		case "LastErr":
 

extern/storage-sealing/checks.go

@@ -33,7 +33,7 @@ type ErrInvalidProof struct{ error }
 type ErrNoPrecommit struct{ error }
 type ErrCommitWaitFailed struct{ error }
 
-func checkPieces(ctx context.Context, si SectorInfo, api SealingAPI) error {
+func checkPieces(ctx context.Context, maddr address.Address, si SectorInfo, api SealingAPI) error {
 	tok, height, err := api.ChainHead(ctx)
 	if err != nil {
 		return &ErrApi{xerrors.Errorf("getting chain head: %w", err)}
@@ -55,6 +55,10 @@ func checkPieces(ctx context.Context, si SectorInfo, api SealingAPI) error {
 			return &ErrInvalidDeals{xerrors.Errorf("getting deal %d for piece %d: %w", p.DealInfo.DealID, i, err)}
 		}
 
+		if proposal.Provider != maddr {
+			return &ErrInvalidDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers deal %d with wrong provider: %s != %s", i, len(si.Pieces), si.SectorNumber, p.DealInfo.DealID, proposal.Provider, maddr)}
+		}
+
 		if proposal.PieceCID != p.Piece.PieceCID {
 			return &ErrInvalidDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers deal %d with wrong PieceCID: %x != %x", i, len(si.Pieces), si.SectorNumber, p.DealInfo.DealID, p.Piece.PieceCID, proposal.PieceCID)}
 		}
@@ -74,6 +78,10 @@ func checkPieces(ctx context.Context, si SectorInfo, api SealingAPI) error {
 // checkPrecommit checks that data commitment generated in the sealing process
 //  matches pieces, and that the seal ticket isn't expired
 func checkPrecommit(ctx context.Context, maddr address.Address, si SectorInfo, tok TipSetToken, height abi.ChainEpoch, api SealingAPI) (err error) {
+	if err := checkPieces(ctx, maddr, si, api); err != nil {
+		return err
+	}
+
 	commD, err := api.StateComputeDataCommitment(ctx, maddr, si.SectorType, si.dealIDs(), tok)
 	if err != nil {
 		return &ErrApi{xerrors.Errorf("calling StateComputeDataCommitment: %w", err)}
@@ -176,5 +184,9 @@ func (m *Sealing) checkCommit(ctx context.Context, si SectorInfo, proof []byte,
 		return &ErrInvalidProof{xerrors.New("invalid proof (compute error?)")}
 	}
 
+	if err := checkPieces(ctx, m.maddr, si, m.api); err != nil {
+		return err
+	}
+
 	return nil
 }

extern/storage-sealing/fsm.go

@@ -17,9 +17,9 @@ import (
 )
 
 func (m *Sealing) Plan(events []statemachine.Event, user interface{}) (interface{}, uint64, error) {
-	next, err := m.plan(events, user.(*SectorInfo))
+	next, processed, err := m.plan(events, user.(*SectorInfo))
 	if err != nil || next == nil {
-		return nil, uint64(len(events)), err
+		return nil, processed, err
 	}
 
 	return func(ctx statemachine.Context, si SectorInfo) error {
@@ -30,10 +30,10 @@ func (m *Sealing) Plan(events []statemachine.Event, user interface{}) (interface
 		}
 
 		return nil
-	}, uint64(len(events)), nil // TODO: This processed event count is not very correct
+	}, processed, nil // TODO: This processed event count is not very correct
 }
 
-var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *SectorInfo) error{
+var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *SectorInfo) (uint64, error){
 	// Sealing
 
 	UndefinedSectorState: planOne(
@@ -49,31 +49,39 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 	PreCommit1: planOne(
 		on(SectorPreCommit1{}, PreCommit2),
 		on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed),
-		on(SectorPackingFailed{}, PackingFailed),
+		on(SectorDealsExpired{}, DealsExpired),
+		on(SectorInvalidDealIDs{}, RecoverDealIDs),
 	),
 	PreCommit2: planOne(
 		on(SectorPreCommit2{}, PreCommitting),
 		on(SectorSealPreCommit2Failed{}, SealPreCommit2Failed),
-		on(SectorPackingFailed{}, PackingFailed),
 	),
 	PreCommitting: planOne(
 		on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed),
 		on(SectorPreCommitted{}, PreCommitWait),
 		on(SectorChainPreCommitFailed{}, PreCommitFailed),
 		on(SectorPreCommitLanded{}, WaitSeed),
+		on(SectorDealsExpired{}, DealsExpired),
+		on(SectorInvalidDealIDs{}, RecoverDealIDs),
 	),
 	PreCommitWait: planOne(
 		on(SectorChainPreCommitFailed{}, PreCommitFailed),
 		on(SectorPreCommitLanded{}, WaitSeed),
+		on(SectorRetryPreCommit{}, PreCommitting),
 	),
 	WaitSeed: planOne(
 		on(SectorSeedReady{}, Committing),
 		on(SectorChainPreCommitFailed{}, PreCommitFailed),
 	),
 	Committing: planCommitting,
+	SubmitCommit: planOne(
+		on(SectorCommitSubmitted{}, CommitWait),
+		on(SectorCommitFailed{}, CommitFailed),
+	),
 	CommitWait: planOne(
 		on(SectorProving{}, FinalizeSector),
 		on(SectorCommitFailed{}, CommitFailed),
+		on(SectorRetrySubmitCommit{}, SubmitCommit),
 	),
 
 	FinalizeSector: planOne(
@@ -95,6 +103,8 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 		on(SectorRetryWaitSeed{}, WaitSeed),
 		on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed),
 		on(SectorPreCommitLanded{}, WaitSeed),
+		on(SectorDealsExpired{}, DealsExpired),
+		on(SectorInvalidDealIDs{}, RecoverDealIDs),
 	),
 	ComputeProofFailed: planOne(
 		on(SectorRetryComputeProof{}, Committing),
@@ -109,22 +119,33 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 		on(SectorChainPreCommitFailed{}, PreCommitFailed),
 		on(SectorRetryPreCommit{}, PreCommitting),
 		on(SectorRetryCommitWait{}, CommitWait),
+		on(SectorDealsExpired{}, DealsExpired),
+		on(SectorInvalidDealIDs{}, RecoverDealIDs),
 	),
 	FinalizeFailed: planOne(
 		on(SectorRetryFinalize{}, FinalizeSector),
 	),
+	PackingFailed: planOne(), // TODO: Deprecated, remove
+	DealsExpired:  planOne(
+	// SectorRemove (global)
+	),
+	RecoverDealIDs: planOne(
+		onReturning(SectorUpdateDealIDs{}),
+	),
 
 	// Post-seal
 
 	Proving: planOne(
 		on(SectorFaultReported{}, FaultReported),
 		on(SectorFaulty{}, Faulty),
-		on(SectorRemove{}, Removing),
 	),
 	Removing: planOne(
 		on(SectorRemoved{}, Removed),
 		on(SectorRemoveFailed{}, RemoveFailed),
 	),
+	RemoveFailed: planOne(
+	// SectorRemove (global)
+	),
 	Faulty: planOne(
 		on(SectorFaultReported{}, FaultReported),
 	),
@@ -133,7 +154,7 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 	Removed:      final,
 }
 
-func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(statemachine.Context, SectorInfo) error, error) {
+func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(statemachine.Context, SectorInfo) error, uint64, error) {
 	/////
 	// First process all events
 
@@ -170,11 +191,12 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 
 	p := fsmPlanners[state.State]
 	if p == nil {
-		return nil, xerrors.Errorf("planner for state %s not found", state.State)
+		return nil, 0, xerrors.Errorf("planner for state %s not found", state.State)
 	}
 
-	if err := p(events, state); err != nil {
-		return nil, xerrors.Errorf("running planner for state %s failed: %w", state.State, err)
+	processed, err := p(events, state)
+	if err != nil {
+		return nil, 0, xerrors.Errorf("running planner for state %s failed: %w", state.State, err)
 	}
 
 	/////
@@ -182,47 +204,50 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 
 	/*
 
-			*   Empty <- incoming deals
-			|   |
-			|   v
-		    *<- WaitDeals <- incoming deals
-			|   |
-			|   v
-			*<- Packing <- incoming committed capacity
-			|   |
-			|   v
-			*<- PreCommit1 <--> SealPreCommit1Failed
-			|   |       ^          ^^
-			|   |       *----------++----\
-			|   v       v          ||    |
-			*<- PreCommit2 --------++--> SealPreCommit2Failed
-			|   |                  ||
-			|   v          /-------/|
-			*   PreCommitting <-----+---> PreCommitFailed
-			|   |                   |     ^
-			|   v                   |     |
-			*<- WaitSeed -----------+-----/
-			|   |||  ^              |
-			|   |||  \--------*-----/
-			|   |||           |
-			|   vvv      v----+----> ComputeProofFailed
-			*<- Committing    |
-			|   |        ^--> CommitFailed
-			|   v             ^
-			*<- CommitWait ---/
-			|   |
-			|   v
-			|   FinalizeSector <--> FinalizeFailed
-			|   |
-			|   v
-			*<- Proving
-			|
-			v
-			FailedUnrecoverable
+				*   Empty <- incoming deals
+				|   |
+				|   v
+			    *<- WaitDeals <- incoming deals
+				|   |
+				|   v
+				*<- Packing <- incoming committed capacity
+				|   |
+				|   v
+				*<- PreCommit1 <--> SealPreCommit1Failed
+				|   |       ^          ^^
+				|   |       *----------++----\
+				|   v       v          ||    |
+				*<- PreCommit2 --------++--> SealPreCommit2Failed
+				|   |                  ||
+				|   v          /-------/|
+				*   PreCommitting <-----+---> PreCommitFailed
+				|   |                   |     ^
+				|   v                   |     |
+				*<- WaitSeed -----------+-----/
+				|   |||  ^              |
+				|   |||  \--------*-----/
+				|   |||           |
+				|   vvv      v----+----> ComputeProofFailed
+				*<- Committing    |
+				|   |        ^--> CommitFailed
+				|   v             ^
+		        |   SubmitCommit  |
+		        |   |             |
+		        |   v             |
+				*<- CommitWait ---/
+				|   |
+				|   v
+				|   FinalizeSector <--> FinalizeFailed
+				|   |
+				|   v
+				*<- Proving
+				|
+				v
+				FailedUnrecoverable
 
-			UndefinedSectorState <- ¯\_(ツ)_/¯
-				|                     ^
-				*---------------------/
+				UndefinedSectorState <- ¯\_(ツ)_/¯
+					|                     ^
+					*---------------------/
 
 	*/
 
@@ -235,51 +260,63 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 	case WaitDeals:
 		log.Infof("Waiting for deals %d", state.SectorNumber)
 	case Packing:
-		return m.handlePacking, nil
+		return m.handlePacking, processed, nil
 	case PreCommit1:
-		return m.handlePreCommit1, nil
+		return m.handlePreCommit1, processed, nil
 	case PreCommit2:
-		return m.handlePreCommit2, nil
+		return m.handlePreCommit2, processed, nil
 	case PreCommitting:
-		return m.handlePreCommitting, nil
+		return m.handlePreCommitting, processed, nil
 	case PreCommitWait:
-		return m.handlePreCommitWait, nil
+		return m.handlePreCommitWait, processed, nil
 	case WaitSeed:
-		return m.handleWaitSeed, nil
+		return m.handleWaitSeed, processed, nil
 	case Committing:
-		return m.handleCommitting, nil
+		return m.handleCommitting, processed, nil
+	case SubmitCommit:
+		return m.handleSubmitCommit, processed, nil
 	case CommitWait:
-		return m.handleCommitWait, nil
+		return m.handleCommitWait, processed, nil
 	case FinalizeSector:
-		return m.handleFinalizeSector, nil
+		return m.handleFinalizeSector, processed, nil
 
 	// Handled failure modes
 	case SealPreCommit1Failed:
-		return m.handleSealPrecommit1Failed, nil
+		return m.handleSealPrecommit1Failed, processed, nil
 	case SealPreCommit2Failed:
-		return m.handleSealPrecommit2Failed, nil
+		return m.handleSealPrecommit2Failed, processed, nil
 	case PreCommitFailed:
-		return m.handlePreCommitFailed, nil
+		return m.handlePreCommitFailed, processed, nil
 	case ComputeProofFailed:
-		return m.handleComputeProofFailed, nil
+		return m.handleComputeProofFailed, processed, nil
 	case CommitFailed:
-		return m.handleCommitFailed, nil
+		return m.handleCommitFailed, processed, nil
 	case FinalizeFailed:
-		return m.handleFinalizeFailed, nil
+		return m.handleFinalizeFailed, processed, nil
+	case PackingFailed: // DEPRECATED: remove this for the next reset
+		state.State = DealsExpired
+		fallthrough
+	case DealsExpired:
+		return m.handleDealsExpired, processed, nil
+	case RecoverDealIDs:
+		return m.handleRecoverDealIDs, processed, nil
 
 	// Post-seal
 	case Proving:
-		return m.handleProvingSector, nil
+		return m.handleProvingSector, processed, nil
 	case Removing:
-		return m.handleRemoving, nil
+		return m.handleRemoving, processed, nil
 	case Removed:
-		return nil, nil
+		return nil, processed, nil
+
+	case RemoveFailed:
+		return m.handleRemoveFailed, processed, nil
 
 		// Faults
 	case Faulty:
-		return m.handleFaulty, nil
+		return m.handleFaulty, processed, nil
 	case FaultReported:
-		return m.handleFaultReported, nil
+		return m.handleFaultReported, processed, nil
 
 	// Fatal errors
 	case UndefinedSectorState:
@@ -290,28 +327,29 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 		log.Errorf("unexpected sector update state: %s", state.State)
 	}
 
-	return nil, nil
+	return nil, processed, nil
 }
 
-func planCommitting(events []statemachine.Event, state *SectorInfo) error {
-	for _, event := range events {
+func planCommitting(events []statemachine.Event, state *SectorInfo) (uint64, error) {
+	for i, event := range events {
 		switch e := event.User.(type) {
 		case globalMutator:
 			if e.applyGlobal(state) {
-				return nil
+				return uint64(i + 1), nil
 			}
 		case SectorCommitted: // the normal case
 			e.apply(state)
-			state.State = CommitWait
+			state.State = SubmitCommit
 		case SectorSeedReady: // seed changed :/
 			if e.SeedEpoch == state.SeedEpoch && bytes.Equal(e.SeedValue, state.SeedValue) {
 				log.Warnf("planCommitting: got SectorSeedReady, but the seed didn't change")
 				continue // or it didn't!
 			}
+
 			log.Warnf("planCommitting: commit Seed changed")
 			e.apply(state)
 			state.State = Committing
-			return nil
+			return uint64(i + 1), nil
 		case SectorComputeProofFailed:
 			state.State = ComputeProofFailed
 		case SectorSealPreCommit1Failed:
@@ -321,10 +359,10 @@ func planCommitting(events []statemachine.Event, state *SectorInfo) error {
 		case SectorRetryCommitWait:
 			state.State = CommitWait
 		default:
-			return xerrors.Errorf("planCommitting got event of unknown type %T, events: %+v", event.User, events)
+			return uint64(i), xerrors.Errorf("planCommitting got event of unknown type %T, events: %+v", event.User, events)
 		}
 	}
-	return nil
+	return uint64(len(events)), nil
 }
 
 func (m *Sealing) restartSectors(ctx context.Context) error {
@@ -365,31 +403,38 @@ func (m *Sealing) ForceSectorState(ctx context.Context, id abi.SectorNumber, sta
 	return m.sectors.Send(id, SectorForceState{state})
 }
 
-func final(events []statemachine.Event, state *SectorInfo) error {
-	return xerrors.Errorf("didn't expect any events in state %s, got %+v", state.State, events)
+func final(events []statemachine.Event, state *SectorInfo) (uint64, error) {
+	return 0, xerrors.Errorf("didn't expect any events in state %s, got %+v", state.State, events)
 }
 
-func on(mut mutator, next SectorState) func() (mutator, SectorState) {
-	return func() (mutator, SectorState) {
-		return mut, next
+func on(mut mutator, next SectorState) func() (mutator, func(*SectorInfo) error) {
+	return func() (mutator, func(*SectorInfo) error) {
+		return mut, func(state *SectorInfo) error {
+			state.State = next
+			return nil
+		}
 	}
 }
 
-func planOne(ts ...func() (mut mutator, next SectorState)) func(events []statemachine.Event, state *SectorInfo) error {
-	return func(events []statemachine.Event, state *SectorInfo) error {
-		if len(events) != 1 {
-			for _, event := range events {
-				if gm, ok := event.User.(globalMutator); ok {
-					gm.applyGlobal(state)
-					return nil
-				}
+func onReturning(mut mutator) func() (mutator, func(*SectorInfo) error) {
+	return func() (mutator, func(*SectorInfo) error) {
+		return mut, func(state *SectorInfo) error {
+			if state.Return == "" {
+				return xerrors.Errorf("return state not set")
 			}
-			return xerrors.Errorf("planner for state %s only has a plan for a single event only, got %+v", state.State, events)
-		}
 
+			state.State = SectorState(state.Return)
+			state.Return = ""
+			return nil
+		}
+	}
+}
+
+func planOne(ts ...func() (mut mutator, next func(*SectorInfo) error)) func(events []statemachine.Event, state *SectorInfo) (uint64, error) {
+	return func(events []statemachine.Event, state *SectorInfo) (uint64, error) {
 		if gm, ok := events[0].User.(globalMutator); ok {
 			gm.applyGlobal(state)
-			return nil
+			return 1, nil
 		}
 
 		for _, t := range ts {
@@ -404,15 +449,14 @@ func planOne(ts ...func() (mut mutator, next SectorState)) func(events []statema
 			}
 
 			events[0].User.(mutator).apply(state)
-			state.State = next
-			return nil
+			return 1, next(state)
 		}
 
 		_, ok := events[0].User.(Ignorable)
 		if ok {
-			return nil
+			return 1, nil
 		}
 
-		return xerrors.Errorf("planner for state %s received unexpected event %T (%+v)", state.State, events[0].User, events[0])
+		return 0, xerrors.Errorf("planner for state %s received unexpected event %T (%+v)", state.State, events[0].User, events[0])
 	}
 }

extern/storage-sealing/fsm_events.go

@@ -101,10 +101,6 @@ func (evt SectorPacked) apply(state *SectorInfo) {
 	}
 }
 
-type SectorPackingFailed struct{ error }
-
-func (evt SectorPackingFailed) apply(*SectorInfo) {}
-
 type SectorPreCommit1 struct {
 	PreCommit1Out storage.PreCommit1Out
 	TicketValue   abi.SealRandomness
@@ -191,13 +187,28 @@ type SectorCommitFailed struct{ error }
 func (evt SectorCommitFailed) FormatError(xerrors.Printer) (next error) { return evt.error }
 func (evt SectorCommitFailed) apply(*SectorInfo)                        {}
 
+type SectorRetrySubmitCommit struct{}
+
+func (evt SectorRetrySubmitCommit) apply(*SectorInfo) {}
+
+type SectorDealsExpired struct{ error }
+
+func (evt SectorDealsExpired) FormatError(xerrors.Printer) (next error) { return evt.error }
+func (evt SectorDealsExpired) apply(*SectorInfo)                        {}
+
 type SectorCommitted struct {
-	Message cid.Cid
-	Proof   []byte
+	Proof []byte
 }
 
 func (evt SectorCommitted) apply(state *SectorInfo) {
 	state.Proof = evt.Proof
+}
+
+type SectorCommitSubmitted struct {
+	Message cid.Cid
+}
+
+func (evt SectorCommitSubmitted) apply(state *SectorInfo) {
 	state.CommitMessage = &evt.Message
 }
 
@@ -256,6 +267,24 @@ type SectorRetryCommitWait struct{}
 
 func (evt SectorRetryCommitWait) apply(state *SectorInfo) {}
 
+type SectorInvalidDealIDs struct {
+	Return ReturnState
+}
+
+func (evt SectorInvalidDealIDs) apply(state *SectorInfo) {
+	state.Return = evt.Return
+}
+
+type SectorUpdateDealIDs struct {
+	Updates map[int]abi.DealID
+}
+
+func (evt SectorUpdateDealIDs) apply(state *SectorInfo) {
+	for i, id := range evt.Updates {
+		state.Pieces[i].DealInfo.DealID = id
+	}
+}
+
 // Faults
 
 type SectorFaulty struct{}
@@ -274,7 +303,10 @@ type SectorFaultedFinal struct{}
 
 type SectorRemove struct{}
 
-func (evt SectorRemove) apply(state *SectorInfo) {}
+func (evt SectorRemove) applyGlobal(state *SectorInfo) bool {
+	state.State = Removing
+	return true
+}
 
 type SectorRemoved struct{}
 

extern/storage-sealing/fsm_test.go

@@ -16,7 +16,7 @@ func init() {
 }
 
 func (t *test) planSingle(evt interface{}) {
-	_, err := t.s.plan([]statemachine.Event{{User: evt}}, t.state)
+	_, _, err := t.s.plan([]statemachine.Event{{User: evt}}, t.state)
 	require.NoError(t.t, err)
 }
 
@@ -58,6 +58,9 @@ func TestHappyPath(t *testing.T) {
 	require.Equal(m.t, m.state.State, Committing)
 
 	m.planSingle(SectorCommitted{})
+	require.Equal(m.t, m.state.State, SubmitCommit)
+
+	m.planSingle(SectorCommitSubmitted{})
 	require.Equal(m.t, m.state.State, CommitWait)
 
 	m.planSingle(SectorProving{})
@@ -98,13 +101,16 @@ func TestSeedRevert(t *testing.T) {
 	m.planSingle(SectorSeedReady{})
 	require.Equal(m.t, m.state.State, Committing)
 
-	_, err := m.s.plan([]statemachine.Event{{User: SectorSeedReady{SeedValue: nil, SeedEpoch: 5}}, {User: SectorCommitted{}}}, m.state)
+	_, _, err := m.s.plan([]statemachine.Event{{User: SectorSeedReady{SeedValue: nil, SeedEpoch: 5}}, {User: SectorCommitted{}}}, m.state)
 	require.NoError(t, err)
 	require.Equal(m.t, m.state.State, Committing)
 
 	// not changing the seed this time
-	_, err = m.s.plan([]statemachine.Event{{User: SectorSeedReady{SeedValue: nil, SeedEpoch: 5}}, {User: SectorCommitted{}}}, m.state)
+	_, _, err = m.s.plan([]statemachine.Event{{User: SectorSeedReady{SeedValue: nil, SeedEpoch: 5}}, {User: SectorCommitted{}}}, m.state)
 	require.NoError(t, err)
+	require.Equal(m.t, m.state.State, SubmitCommit)
+
+	m.planSingle(SectorCommitSubmitted{})
 	require.Equal(m.t, m.state.State, CommitWait)
 
 	m.planSingle(SectorProving{})
@@ -129,7 +135,8 @@ func TestPlanCommittingHandlesSectorCommitFailed(t *testing.T) {
 
 	events := []statemachine.Event{{User: SectorCommitFailed{}}}
 
-	require.NoError(t, planCommitting(events, m.state))
+	_, err := planCommitting(events, m.state)
+	require.NoError(t, err)
 
 	require.Equal(t, CommitFailed, m.state.State)
 }

extern/storage-sealing/sealing.go

@@ -141,7 +141,7 @@ func (m *Sealing) Stop(ctx context.Context) error {
 	return m.sectors.Stop(ctx)
 }
 func (m *Sealing) AddPieceToAnySector(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, d DealInfo) (abi.SectorNumber, abi.PaddedPieceSize, error) {
-	log.Infof("Adding piece for deal %d", d.DealID)
+	log.Infof("Adding piece for deal %d (publish msg: %s)", d.DealID, d.PublishCid)
 	if (padreader.PaddedSize(uint64(size))) != size {
 		return 0, 0, xerrors.Errorf("cannot allocate unpadded piece")
 	}
@@ -224,15 +224,23 @@ func (m *Sealing) Remove(ctx context.Context, sid abi.SectorNumber) error {
 
 // Caller should NOT hold m.unsealedInfoMap.lk
 func (m *Sealing) StartPacking(sectorID abi.SectorNumber) error {
+	// locking here ensures that when the SectorStartPacking event is sent, the sector won't be picked up anywhere else
+	m.unsealedInfoMap.lk.Lock()
+	defer m.unsealedInfoMap.lk.Unlock()
+
+	// cannot send SectorStartPacking to sectors that have already been packed, otherwise it will cause the state machine to exit
+	if _, ok := m.unsealedInfoMap.infos[sectorID]; !ok {
+		log.Warnf("call start packing, but sector %v not in unsealedInfoMap.infos, maybe have called", sectorID)
+		return nil
+	}
 	log.Infof("Starting packing sector %d", sectorID)
 	err := m.sectors.Send(uint64(sectorID), SectorStartPacking{})
 	if err != nil {
 		return err
 	}
+	log.Infof("send Starting packing event success sector %d", sectorID)
 
-	m.unsealedInfoMap.lk.Lock()
 	delete(m.unsealedInfoMap.infos, sectorID)
-	m.unsealedInfoMap.lk.Unlock()
 
 	return nil
 }

extern/storage-sealing/sector_state.go

@@ -10,12 +10,13 @@ const (
 	WaitDeals      SectorState = "WaitDeals"     // waiting for more pieces (deals) to be added to the sector
 	Packing        SectorState = "Packing"       // sector not in sealStore, and not on chain
 	PreCommit1     SectorState = "PreCommit1"    // do PreCommit1
-	PreCommit2     SectorState = "PreCommit2"    // do PreCommit1
+	PreCommit2     SectorState = "PreCommit2"    // do PreCommit2
 	PreCommitting  SectorState = "PreCommitting" // on chain pre-commit
 	PreCommitWait  SectorState = "PreCommitWait" // waiting for precommit to land on chain
 	WaitSeed       SectorState = "WaitSeed"      // waiting for seed
-	Committing     SectorState = "Committing"
-	CommitWait     SectorState = "CommitWait" // waiting for message to land on chain
+	Committing     SectorState = "Committing"    // compute PoRep
+	SubmitCommit   SectorState = "SubmitCommit"  // send commit message to the chain
+	CommitWait     SectorState = "CommitWait"    // wait for the commit message to land on chain
 	FinalizeSector SectorState = "FinalizeSector"
 	Proving        SectorState = "Proving"
 	// error modes
@@ -25,8 +26,10 @@ const (
 	PreCommitFailed      SectorState = "PreCommitFailed"
 	ComputeProofFailed   SectorState = "ComputeProofFailed"
 	CommitFailed         SectorState = "CommitFailed"
-	PackingFailed        SectorState = "PackingFailed"
+	PackingFailed        SectorState = "PackingFailed" // TODO: deprecated, remove
 	FinalizeFailed       SectorState = "FinalizeFailed"
+	DealsExpired         SectorState = "DealsExpired"
+	RecoverDealIDs       SectorState = "RecoverDealIDs"
 
 	Faulty        SectorState = "Faulty"        // sector is corrupted or gone for some reason
 	FaultReported SectorState = "FaultReported" // sector has been declared as a fault on chain

extern/storage-sealing/states_failed.go

@@ -1,12 +1,18 @@
 package sealing
 
 import (
+	"bytes"
 	"time"
 
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-statemachine"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/builtin/market"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
+	"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
+
+	"github.com/filecoin-project/lotus/extern/sector-storage/zerocomm"
 )
 
 const minRetryTime = 1 * time.Minute
@@ -81,6 +87,11 @@ func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorI
 			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired error: %w", err)})
 		case *ErrBadTicket:
 			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad expired: %w", err)})
+		case *ErrInvalidDeals:
+			log.Warnf("invalid deals in sector %d: %v", sector.SectorNumber, err)
+			return ctx.Send(SectorInvalidDealIDs{Return: RetPreCommitFailed})
+		case *ErrExpiredDeals:
+			return ctx.Send(SectorDealsExpired{xerrors.Errorf("sector deals expired: %w", err)})
 		case *ErrNoPrecommit:
 			return ctx.Send(SectorRetryPreCommit{})
 		case *ErrPrecommitOnChain:
@@ -88,6 +99,7 @@ func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorI
 		case *ErrSectorNumberAllocated:
 			log.Errorf("handlePreCommitFailed: sector number already allocated, not proceeding: %+v", err)
 			// TODO: check if the sector is committed (not sure how we'd end up here)
+			// TODO: check on-chain state, adjust local sector number counter to not give out allocated numbers
 			return nil
 		default:
 			return xerrors.Errorf("checkPrecommit sanity check error: %w", err)
@@ -157,7 +169,12 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo
 		case *ErrExpiredTicket:
 			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired error: %w", err)})
 		case *ErrBadTicket:
-			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad expired: %w", err)})
+			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad ticket: %w", err)})
+		case *ErrInvalidDeals:
+			log.Warnf("invalid deals in sector %d: %v", sector.SectorNumber, err)
+			return ctx.Send(SectorInvalidDealIDs{Return: RetCommitFailed})
+		case *ErrExpiredDeals:
+			return ctx.Send(SectorDealsExpired{xerrors.Errorf("sector deals expired: %w", err)})
 		case nil:
 			return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("no precommit: %w", err)})
 		case *ErrPrecommitOnChain:
@@ -192,6 +209,11 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo
 			return ctx.Send(SectorRetryPreCommitWait{})
 		case *ErrNoPrecommit:
 			return ctx.Send(SectorRetryPreCommit{})
+		case *ErrInvalidDeals:
+			log.Warnf("invalid deals in sector %d: %v", sector.SectorNumber, err)
+			return ctx.Send(SectorInvalidDealIDs{Return: RetCommitFailed})
+		case *ErrExpiredDeals:
+			return ctx.Send(SectorDealsExpired{xerrors.Errorf("sector deals expired: %w", err)})
 		case *ErrCommitWaitFailed:
 			if err := failedCooldown(ctx, sector); err != nil {
 				return err
@@ -221,3 +243,120 @@ func (m *Sealing) handleFinalizeFailed(ctx statemachine.Context, sector SectorIn
 
 	return ctx.Send(SectorRetryFinalize{})
 }
+
+func (m *Sealing) handleRemoveFailed(ctx statemachine.Context, sector SectorInfo) error {
+	if err := failedCooldown(ctx, sector); err != nil {
+		return err
+	}
+
+	return ctx.Send(SectorRemove{})
+}
+
+func (m *Sealing) handleDealsExpired(ctx statemachine.Context, sector SectorInfo) error {
+	// First make vary sure the sector isn't committed
+	si, err := m.api.StateSectorGetInfo(ctx.Context(), m.maddr, sector.SectorNumber, nil)
+	if err != nil {
+		return xerrors.Errorf("getting sector info: %w", err)
+	}
+	if si != nil {
+		// TODO: this should never happen, but in case it does, try to go back to
+		//  the proving state after running some checks
+		return xerrors.Errorf("sector is committed on-chain, but we're in DealsExpired")
+	}
+
+	if sector.PreCommitInfo == nil {
+		// TODO: Create a separate state which will remove those pieces, and go back to PC1
+		log.Errorf("non-precommitted sector with expired deals, can't recover from this yet")
+	}
+
+	// Not much to do here, we can't go back in time to commit this sector
+	return ctx.Send(SectorRemove{})
+}
+
+func (m *Sealing) handleRecoverDealIDs(ctx statemachine.Context, sector SectorInfo) error {
+	tok, height, err := m.api.ChainHead(ctx.Context())
+	if err != nil {
+		return xerrors.Errorf("getting chain head: %w", err)
+	}
+
+	var toFix []int
+
+	for i, p := range sector.Pieces {
+		// if no deal is associated with the piece, ensure that we added it as
+		// filler (i.e. ensure that it has a zero PieceCID)
+		if p.DealInfo == nil {
+			exp := zerocomm.ZeroPieceCommitment(p.Piece.Size.Unpadded())
+			if !p.Piece.PieceCID.Equals(exp) {
+				return xerrors.Errorf("sector %d piece %d had non-zero PieceCID %+v", sector.SectorNumber, i, p.Piece.PieceCID)
+			}
+			continue
+		}
+
+		proposal, err := m.api.StateMarketStorageDeal(ctx.Context(), p.DealInfo.DealID, tok)
+		if err != nil {
+			log.Warnf("getting deal %d for piece %d: %+v", p.DealInfo.DealID, i, err)
+			toFix = append(toFix, i)
+			continue
+		}
+
+		if proposal.Provider != m.maddr {
+			log.Warnf("piece %d (of %d) of sector %d refers deal %d with wrong provider: %s != %s", i, len(sector.Pieces), sector.SectorNumber, p.DealInfo.DealID, proposal.Provider, m.maddr)
+			toFix = append(toFix, i)
+			continue
+		}
+
+		if proposal.PieceCID != p.Piece.PieceCID {
+			log.Warnf("piece %d (of %d) of sector %d refers deal %d with wrong PieceCID: %x != %x", i, len(sector.Pieces), sector.SectorNumber, p.DealInfo.DealID, p.Piece.PieceCID, proposal.PieceCID)
+			toFix = append(toFix, i)
+			continue
+		}
+
+		if p.Piece.Size != proposal.PieceSize {
+			log.Warnf("piece %d (of %d) of sector %d refers deal %d with different size: %d != %d", i, len(sector.Pieces), sector.SectorNumber, p.DealInfo.DealID, p.Piece.Size, proposal.PieceSize)
+			toFix = append(toFix, i)
+			continue
+		}
+
+		if height >= proposal.StartEpoch {
+			// TODO: check if we are in an early enough state (before precommit), try to remove the offending pieces
+			//  (tricky as we have to 'defragment' the sector while doing that, and update piece references for retrieval)
+			return xerrors.Errorf("can't fix sector deals: piece %d (of %d) of sector %d refers expired deal %d - should start at %d, head %d", i, len(sector.Pieces), sector.SectorNumber, p.DealInfo.DealID, proposal.StartEpoch, height)
+		}
+	}
+
+	updates := map[int]abi.DealID{}
+	for _, i := range toFix {
+		p := sector.Pieces[i]
+
+		if p.DealInfo.PublishCid == nil {
+			// TODO: check if we are in an early enough state try to remove this piece
+			log.Error("can't fix sector deals: piece %d (of %d) of sector %d has nil DealInfo.PublishCid (refers to deal %d)", i, len(sector.Pieces), sector.SectorNumber, p.DealInfo.DealID)
+			// Not much to do here (and this can only happen for old spacerace sectors)
+			return ctx.Send(SectorRemove{})
+		}
+
+		ml, err := m.api.StateSearchMsg(ctx.Context(), *p.DealInfo.PublishCid)
+		if err != nil {
+			return xerrors.Errorf("looking for publish deal message %s (sector %d, piece %d): %w", *p.DealInfo.PublishCid, sector.SectorNumber, i, err)
+		}
+
+		if ml.Receipt.ExitCode != exitcode.Ok {
+			return xerrors.Errorf("looking for publish deal message %s (sector %d, piece %d): non-ok exit code: %s", *p.DealInfo.PublishCid, sector.SectorNumber, i, ml.Receipt.ExitCode)
+		}
+
+		var retval market.PublishStorageDealsReturn
+		if err := retval.UnmarshalCBOR(bytes.NewReader(ml.Receipt.Return)); err != nil {
+			return xerrors.Errorf("looking for publish deal message: unmarshaling message return: %w", err)
+		}
+
+		if len(retval.IDs) != 1 {
+			// market currently only ever sends messages with 1 deal
+			return xerrors.Errorf("can't recover dealIDs from publish deal message with more than 1 deal")
+		}
+
+		updates[i] = retval.IDs[0]
+	}
+
+	// Not much to do here, we can't go back in time to commit this sector
+	return ctx.Send(SectorUpdateDealIDs{Updates: updates})
+}

extern/storage-sealing/states_sealing.go

@@ -12,6 +12,7 @@ import (
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-actors/actors/crypto"
+	"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
 	"github.com/filecoin-project/specs-storage/storage"
 )
 
@@ -79,15 +80,16 @@ func (m *Sealing) getTicket(ctx statemachine.Context, sector SectorInfo) (abi.Se
 }
 
 func (m *Sealing) handlePreCommit1(ctx statemachine.Context, sector SectorInfo) error {
-	if err := checkPieces(ctx.Context(), sector, m.api); err != nil { // Sanity check state
+	if err := checkPieces(ctx.Context(), m.maddr, sector, m.api); err != nil { // Sanity check state
 		switch err.(type) {
 		case *ErrApi:
 			log.Errorf("handlePreCommit1: api error, not proceeding: %+v", err)
 			return nil
 		case *ErrInvalidDeals:
-			return ctx.Send(SectorPackingFailed{xerrors.Errorf("invalid dealIDs in sector: %w", err)})
+			log.Warnf("invalid deals in sector %d: %v", sector.SectorNumber, err)
+			return ctx.Send(SectorInvalidDealIDs{Return: RetPreCommit1})
 		case *ErrExpiredDeals: // Probably not much we can do here, maybe re-pack the sector?
-			return ctx.Send(SectorPackingFailed{xerrors.Errorf("expired dealIDs in sector: %w", err)})
+			return ctx.Send(SectorDealsExpired{xerrors.Errorf("expired dealIDs in sector: %w", err)})
 		default:
 			return xerrors.Errorf("checkPieces sanity check error: %w", err)
 		}
@@ -155,6 +157,11 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf
 			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired: %w", err)})
 		case *ErrBadTicket:
 			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad ticket: %w", err)})
+		case *ErrInvalidDeals:
+			log.Warnf("invalid deals in sector %d: %v", sector.SectorNumber, err)
+			return ctx.Send(SectorInvalidDealIDs{Return: RetPreCommitting})
+		case *ErrExpiredDeals:
+			return ctx.Send(SectorDealsExpired{xerrors.Errorf("sector deals expired: %w", err)})
 		case *ErrPrecommitOnChain:
 			return ctx.Send(SectorPreCommitLanded{TipSet: tok}) // we re-did precommit
 		case *ErrSectorNumberAllocated:
@@ -226,11 +233,18 @@ func (m *Sealing) handlePreCommitWait(ctx statemachine.Context, sector SectorInf
 		return ctx.Send(SectorChainPreCommitFailed{err})
 	}
 
-	if mw.Receipt.ExitCode != 0 {
+	switch mw.Receipt.ExitCode {
+	case exitcode.Ok:
+		// this is what we expect
+	case exitcode.SysErrOutOfGas:
+		// gas estimator guessed a wrong number
+		return ctx.Send(SectorRetryPreCommit{})
+	default:
 		log.Error("sector precommit failed: ", mw.Receipt.ExitCode)
 		err := xerrors.Errorf("sector precommit failed: %d", mw.Receipt.ExitCode)
 		return ctx.Send(SectorChainPreCommitFailed{err})
 	}
+
 	log.Info("precommit message landed on chain: ", sector.SectorNumber)
 
 	return ctx.Send(SectorPreCommitLanded{TipSet: mw.TipSetTok})
@@ -326,21 +340,25 @@ func (m *Sealing) handleCommitting(ctx statemachine.Context, sector SectorInfo)
 		return ctx.Send(SectorComputeProofFailed{xerrors.Errorf("computing seal proof failed(2): %w", err)})
 	}
 
+	return ctx.Send(SectorCommitted{
+		Proof: proof,
+	})
+}
+
+func (m *Sealing) handleSubmitCommit(ctx statemachine.Context, sector SectorInfo) error {
 	tok, _, err := m.api.ChainHead(ctx.Context())
 	if err != nil {
 		log.Errorf("handleCommitting: api error, not proceeding: %+v", err)
 		return nil
 	}
 
-	if err := m.checkCommit(ctx.Context(), sector, proof, tok); err != nil {
+	if err := m.checkCommit(ctx.Context(), sector, sector.Proof, tok); err != nil {
 		return ctx.Send(SectorCommitFailed{xerrors.Errorf("commit check error: %w", err)})
 	}
 
-	// TODO: Consider splitting states and persist proof for faster recovery
-
 	params := &miner.ProveCommitSectorParams{
 		SectorNumber: sector.SectorNumber,
-		Proof:        proof,
+		Proof:        sector.Proof,
 	}
 
 	enc := new(bytes.Buffer)
@@ -372,14 +390,13 @@ func (m *Sealing) handleCommitting(ctx statemachine.Context, sector SectorInfo)
 		collateral = big.Zero()
 	}
 
-	// TODO: check seed / ticket are up to date
+	// TODO: check seed / ticket / deals are up to date
 	mcid, err := m.api.SendMsg(ctx.Context(), waddr, m.maddr, builtin.MethodsMiner.ProveCommitSector, collateral, m.feeCfg.MaxCommitGasFee, enc.Bytes())
 	if err != nil {
 		return ctx.Send(SectorCommitFailed{xerrors.Errorf("pushing message to mpool: %w", err)})
 	}
 
-	return ctx.Send(SectorCommitted{
-		Proof:   proof,
+	return ctx.Send(SectorCommitSubmitted{
 		Message: mcid,
 	})
 }
@@ -395,13 +412,22 @@ func (m *Sealing) handleCommitWait(ctx statemachine.Context, sector SectorInfo)
 		return ctx.Send(SectorCommitFailed{xerrors.Errorf("failed to wait for porep inclusion: %w", err)})
 	}
 
-	if mw.Receipt.ExitCode != 0 {
+	switch mw.Receipt.ExitCode {
+	case exitcode.Ok:
+		// this is what we expect
+	case exitcode.SysErrOutOfGas:
+		// gas estimator guessed a wrong number
+		return ctx.Send(SectorRetrySubmitCommit{})
+	default:
 		return ctx.Send(SectorCommitFailed{xerrors.Errorf("submitting sector proof failed (exit=%d, msg=%s) (t:%x; s:%x(%d); p:%x)", mw.Receipt.ExitCode, sector.CommitMessage, sector.TicketValue, sector.SeedValue, sector.SeedEpoch, sector.Proof)})
 	}
 
-	_, err = m.api.StateSectorGetInfo(ctx.Context(), m.maddr, sector.SectorNumber, mw.TipSetTok)
+	si, err := m.api.StateSectorGetInfo(ctx.Context(), m.maddr, sector.SectorNumber, mw.TipSetTok)
 	if err != nil {
-		return ctx.Send(SectorCommitFailed{xerrors.Errorf("proof validation failed, sector not found in sector set after cron: %w", err)})
+		return ctx.Send(SectorCommitFailed{xerrors.Errorf("proof validation failed, calling StateSectorGetInfo: %w", err)})
+	}
+	if si == nil {
+		return ctx.Send(SectorCommitFailed{xerrors.Errorf("proof validation failed, sector not found in sector set after cron")})
 	}
 
 	return ctx.Send(SectorProving{})

extern/storage-sealing/types.go

@@ -30,6 +30,7 @@ type Piece struct {
 
 // DealInfo is a tuple of deal identity and its schedule
 type DealInfo struct {
+	PublishCid   *cid.Cid
 	DealID       abi.DealID
 	DealSchedule DealSchedule
 	KeepUnsealed bool
@@ -53,6 +54,15 @@ type Log struct {
 	Kind string
 }
 
+type ReturnState string
+
+const (
+	RetPreCommit1      = ReturnState(PreCommit1)
+	RetPreCommitting   = ReturnState(PreCommitting)
+	RetPreCommitFailed = ReturnState(PreCommitFailed)
+	RetCommitFailed    = ReturnState(CommitFailed)
+)
+
 type SectorInfo struct {
 	State        SectorState
 	SectorNumber abi.SectorNumber
@@ -90,6 +100,9 @@ type SectorInfo struct {
 	// Faults
 	FaultReportMsg *cid.Cid
 
+	// Recovery
+	Return ReturnState
+
 	// Debug
 	LastErr string
 

extern/storage-sealing/upgrade_queue.go

@@ -72,6 +72,10 @@ func (m *Sealing) tryUpgradeSector(ctx context.Context, params *miner.SectorPreC
 			log.Errorf("error calling StateSectorGetInfo for replaced sector: %+v", err)
 			return big.Zero()
 		}
+		if ri == nil {
+			log.Errorf("couldn't find sector info for sector to replace: %+v", replace)
+			return big.Zero()
+		}
 
 		if params.Expiration < ri.Expiration {
 			// TODO: Some limit on this

go.mod

@@ -7,13 +7,14 @@ replace github.com/supranational/blst => github.com/supranational/blst v0.1.2-al
 require (
 	contrib.go.opencensus.io/exporter/jaeger v0.1.0
 	contrib.go.opencensus.io/exporter/prometheus v0.1.0
+	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
 	github.com/BurntSushi/toml v0.3.1
 	github.com/GeertJohan/go.rice v1.0.0
 	github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee
-	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
 	github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129
 	github.com/coreos/go-systemd/v22 v22.0.0
+	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e
 	github.com/dgraph-io/badger/v2 v2.0.3
 	github.com/docker/go-units v0.4.0
@@ -24,26 +25,25 @@ require (
 	github.com/filecoin-project/chain-validation v0.0.6-0.20200813000554-40c22fe26eef
 	github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200716204036-cddc56607e1d
 	github.com/filecoin-project/go-address v0.0.3
-	github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161 // indirect
 	github.com/filecoin-project/go-bitfield v0.2.0
 	github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
 	github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
-	github.com/filecoin-project/go-data-transfer v0.6.2
+	github.com/filecoin-project/go-data-transfer v0.6.3
 	github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f
-	github.com/filecoin-project/go-fil-markets v0.5.7
-	github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200817153016-2ea5cbaf5ec0
+	github.com/filecoin-project/go-fil-markets v0.5.8
+	github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52
 	github.com/filecoin-project/go-multistore v0.0.3
 	github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6
 	github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261
 	github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370
 	github.com/filecoin-project/go-statestore v0.1.0
 	github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
-	github.com/filecoin-project/sector-storage v0.0.0-20200810171746-eac70842d8e0 // indirect
 	github.com/filecoin-project/specs-actors v0.9.3
 	github.com/filecoin-project/specs-storage v0.1.1-0.20200730063404-f7db367e9401
+	github.com/filecoin-project/statediff v0.0.1
+	github.com/filecoin-project/test-vectors v0.0.0-20200826113833-9ffe6524729d
 	github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
 	github.com/go-kit/kit v0.10.0
-	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.7.4
 	github.com/gorilla/websocket v1.4.2
@@ -51,7 +51,7 @@ require (
 	github.com/hashicorp/go-multierror v1.1.0
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d
-	github.com/ipfs/go-bitswap v0.2.8
+	github.com/ipfs/go-bitswap v0.2.20
 	github.com/ipfs/go-block-format v0.0.2
 	github.com/ipfs/go-blockservice v0.1.4-0.20200624145336-a978cec6e834
 	github.com/ipfs/go-cid v0.0.7
@@ -61,8 +61,8 @@ require (
 	github.com/ipfs/go-ds-leveldb v0.4.2
 	github.com/ipfs/go-ds-measure v0.1.0
 	github.com/ipfs/go-filestore v1.0.0
-	github.com/ipfs/go-fs-lock v0.0.1
-	github.com/ipfs/go-graphsync v0.1.1
+	github.com/ipfs/go-fs-lock v0.0.6
+	github.com/ipfs/go-graphsync v0.1.2
 	github.com/ipfs/go-ipfs-blockstore v1.0.1
 	github.com/ipfs/go-ipfs-chunker v0.0.5
 	github.com/ipfs/go-ipfs-ds-help v1.0.0
@@ -80,11 +80,11 @@ require (
 	github.com/ipfs/go-unixfs v0.2.4
 	github.com/ipfs/interface-go-ipfs-core v0.2.3
 	github.com/ipld/go-car v0.1.1-0.20200526133713-1c7508d55aae
-	github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e
+	github.com/ipld/go-ipld-prime v0.0.4-0.20200828224805-5ff8c8b0b6ef
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/lib/pq v1.7.0
 	github.com/libp2p/go-eventbus v0.2.1
-	github.com/libp2p/go-libp2p v0.10.3
+	github.com/libp2p/go-libp2p v0.11.0
 	github.com/libp2p/go-libp2p-connmgr v0.2.4
 	github.com/libp2p/go-libp2p-core v0.6.1
 	github.com/libp2p/go-libp2p-discovery v0.5.0
@@ -92,7 +92,7 @@ require (
 	github.com/libp2p/go-libp2p-mplex v0.2.4
 	github.com/libp2p/go-libp2p-noise v0.1.1
 	github.com/libp2p/go-libp2p-peerstore v0.2.6
-	github.com/libp2p/go-libp2p-pubsub v0.3.5-0.20200821075113-efd56962bced
+	github.com/libp2p/go-libp2p-pubsub v0.3.5
 	github.com/libp2p/go-libp2p-quic-transport v0.8.0
 	github.com/libp2p/go-libp2p-record v0.1.3
 	github.com/libp2p/go-libp2p-routing-helpers v0.2.3
@@ -100,18 +100,15 @@ require (
 	github.com/libp2p/go-libp2p-tls v0.1.3
 	github.com/libp2p/go-libp2p-yamux v0.2.8
 	github.com/libp2p/go-maddr-filter v0.1.0
-	github.com/mattn/go-isatty v0.0.12 // indirect
 	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/multiformats/go-base32 v0.0.3
-	github.com/multiformats/go-multiaddr v0.2.2
+	github.com/multiformats/go-multiaddr v0.3.1
 	github.com/multiformats/go-multiaddr-dns v0.2.0
-	github.com/multiformats/go-multiaddr-net v0.1.5
 	github.com/multiformats/go-multibase v0.0.3
 	github.com/multiformats/go-multihash v0.0.14
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/raulk/clock v1.1.0
-	github.com/stretchr/objx v0.2.0 // indirect
 	github.com/stretchr/testify v1.6.1
 	github.com/supranational/blst v0.1.1
 	github.com/syndtr/goleveldb v1.0.0
@@ -122,18 +119,15 @@ require (
 	github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d
 	github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542
 	go.opencensus.io v0.22.4
-	go.uber.org/dig v1.8.0 // indirect
+	go.uber.org/dig v1.10.0 // indirect
 	go.uber.org/fx v1.9.0
 	go.uber.org/multierr v1.5.0
 	go.uber.org/zap v1.15.0
-	go4.org v0.0.0-20190313082347-94abd6928b1d // indirect
 	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
 	golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
-	google.golang.org/api v0.25.0 // indirect
 	gotest.tools v2.2.0+incompatible
-	launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect
 )
 
 replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0
@@ -141,3 +135,5 @@ replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v
 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi
 
 replace github.com/dgraph-io/badger/v2 => github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200716180832-3ab515320794
+
+replace github.com/filecoin-project/test-vectors => ./extern/test-vectors

go.sum

@@ -33,8 +33,9 @@ dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1
 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
 github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
-github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
+github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M=
+github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -65,11 +66,9 @@ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia
 github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw=
 github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@@ -164,8 +163,9 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4=
-github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f h1:BOaYiTvg8p9vBUXpklC22XSK/mifLF7lG9jtmYYi3Tc=
 github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4=
+github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU=
+github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U=
 github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e h1:lj77EKYUpYXTd8CD/+QMIf8b6OIOTsfEBSXiAzuEHTU=
 github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk=
 github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
@@ -230,7 +230,8 @@ github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161
 github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161/go.mod h1:vgmwKBkx+ca5OIeEvstiQgzAZnb7R6QaqE1oEDSqa6g=
 github.com/filecoin-project/go-bitfield v0.0.0-20200416002808-b3ee67ec9060/go.mod h1:iodsLxOFZnqKtjj2zkgqzoGNrv6vUqj69AT/J8DKXEw=
 github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY=
-github.com/filecoin-project/go-bitfield v0.1.2 h1:TjLregCoyP1/5lm7WCM0axyV1myIHwbjGa21skuu5tk=
+github.com/filecoin-project/go-bitfield v0.0.3/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY=
+github.com/filecoin-project/go-bitfield v0.0.4-0.20200703174658-f4a5758051a1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY=
 github.com/filecoin-project/go-bitfield v0.1.2/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
 github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q=
 github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
@@ -238,21 +239,28 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:a
 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
-github.com/filecoin-project/go-data-transfer v0.6.2 h1:IgbkwcHoyWGglzfsY7P9L1GapzoiLNKuzfZY2bxER8E=
-github.com/filecoin-project/go-data-transfer v0.6.2/go.mod h1:uRYBRKVBVM12CSusBtVrzDHkVw/3DKZpkxKJVP1Ydas=
+github.com/filecoin-project/go-data-transfer v0.6.1/go.mod h1:uRYBRKVBVM12CSusBtVrzDHkVw/3DKZpkxKJVP1Ydas=
+github.com/filecoin-project/go-data-transfer v0.6.3 h1:7TLwm8nuodHYD/uiwJjKc/PGRR+LwqM8jmlZqgWuUfY=
+github.com/filecoin-project/go-data-transfer v0.6.3/go.mod h1:PmBKVXkhh67/tnEdJXQwDHl5mT+7Tbcwe1NPninqhnM=
+github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
-github.com/filecoin-project/go-fil-markets v0.5.7 h1:kzyMHqez8ssxchj5s9M1hkC3CTwRGh2MeglJGfUksQU=
-github.com/filecoin-project/go-fil-markets v0.5.7/go.mod h1:KnvFG3kSQ77vKYSY/QdrXET81wVCBByHXjG7AyxnbUw=
-github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200817153016-2ea5cbaf5ec0 h1:/GT3V+3f+H5w5odb7LcCWJ1zPw8H8m9TsGQcU0cGSHo=
+github.com/filecoin-project/go-fil-markets v0.5.6-0.20200814234959-80b1788108ac/go.mod h1:umicPCaN99ysHTiYOmwhuLxTFbOwcsI+mdw/t96vvM4=
+github.com/filecoin-project/go-fil-markets v0.5.6/go.mod h1:SJApXAKr5jyGpbzDEOhvemui0pih7hhT8r2MXJxCP1E=
+github.com/filecoin-project/go-fil-markets v0.5.8 h1:uwl0QNUVmmSlUQfxshpj21Dmhh6WKTQNhnb1GMfdp18=
+github.com/filecoin-project/go-fil-markets v0.5.8/go.mod h1:6ZX1vbZbnukbVQ8tCB/MmEizuW/bmRX7SpGAltU3KVg=
 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200817153016-2ea5cbaf5ec0/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4=
+github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 h1:FXtCp0ybqdQL9knb3OGDpkNTaBbPxgkqPeWKotUwkH0=
+github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4=
 github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI=
 github.com/filecoin-project/go-multistore v0.0.3/go.mod h1:kaNqCC4IhU4B1uyr7YWFHd23TL4KM32aChS0jNkyUvQ=
 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs=
 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6/go.mod h1:0HgYnrkeSU4lu1p+LEOeDpFsNBssa0OGGriWdA4hvaE=
+github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 h1:A256QonvzRaknIIAuWhe/M2dpV2otzs3NBhi5TWa/UA=
 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
+github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
 github.com/filecoin-project/go-statemachine v0.0.0-20200714194326-a77c3ae20989/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
 github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 h1:Jbburj7Ih2iaJ/o5Q9A+EAeTabME6YII7FLi9SKUf5c=
 github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
@@ -260,23 +268,27 @@ github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIi
 github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg=
 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8=
+github.com/filecoin-project/lotus v0.4.3-0.20200819134055-b13681df3205/go.mod h1:rooripL/X8ixwUngDPzphAv/RKZXWBprbyxxDW0EJi0=
+github.com/filecoin-project/lotus v0.4.3-0.20200820203717-d1718369a182/go.mod h1:biFZPQ/YyQGfkHUmHMiaNf2hnD6zm1+OAXPQYQ61Zkg=
+github.com/filecoin-project/sector-storage v0.0.0-20200712023225-1d67dcfa3c15/go.mod h1:salgVdX7qeXFo/xaiEQE29J4pPkjn71T0kt0n+VDBzo=
 github.com/filecoin-project/sector-storage v0.0.0-20200730050024-3ee28c3b6d9a/go.mod h1:oOawOl9Yk+qeytLzzIryjI8iRbqo+qzS6EEeElP4PWA=
 github.com/filecoin-project/sector-storage v0.0.0-20200810171746-eac70842d8e0 h1:E1fZ27fhKK05bhZItfTwqr1i05vXnEZJznQFEYwEEUU=
 github.com/filecoin-project/sector-storage v0.0.0-20200810171746-eac70842d8e0/go.mod h1:oOawOl9Yk+qeytLzzIryjI8iRbqo+qzS6EEeElP4PWA=
 github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA=
 github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y=
 github.com/filecoin-project/specs-actors v0.6.1/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=
+github.com/filecoin-project/specs-actors v0.7.3-0.20200716231407-60a2ae96d2e6/go.mod h1:JOMUa7EijvpOO4ofD1yeHNmqohkmmnhTvz/IpB6so4c=
 github.com/filecoin-project/specs-actors v0.8.2/go.mod h1:Q3ACV5kBLvqPaYbthc/J1lGMJ5OwogmD9pzdtPRMdCw=
-github.com/filecoin-project/specs-actors v0.8.7-0.20200811203034-272d022c1923 h1:+H4IG4OjTThljPkMH1ZpynxCulNdx4amEeHoP2GdQJI=
 github.com/filecoin-project/specs-actors v0.8.7-0.20200811203034-272d022c1923/go.mod h1:hukRu6vKQrrS7Nt+fC/ql4PqWLSfmAWNshD/VDtARZU=
-github.com/filecoin-project/specs-actors v0.9.2 h1:0JG0QLHw8pO6BPqPRe9eQxQW60biHAQsx1rlQ9QbzZ0=
 github.com/filecoin-project/specs-actors v0.9.2/go.mod h1:YasnVUOUha0DN5wB+twl+V8LlDKVNknRG00kTJpsfFA=
 github.com/filecoin-project/specs-actors v0.9.3 h1:Fi75G/UQ7R4eiIwnN+S6bBQ9LqKivyJdw62jJzTi6aE=
 github.com/filecoin-project/specs-actors v0.9.3/go.mod h1:YasnVUOUha0DN5wB+twl+V8LlDKVNknRG00kTJpsfFA=
-github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea h1:iixjULRQFPn7Q9KlIqfwLJnlAXO10bbkI+xy5GKGdLY=
 github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k=
 github.com/filecoin-project/specs-storage v0.1.1-0.20200730063404-f7db367e9401 h1:jLzN1hwO5WpKPu8ASbW8fs1FUCsOWNvoBXzQhv+8/E8=
 github.com/filecoin-project/specs-storage v0.1.1-0.20200730063404-f7db367e9401/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k=
+github.com/filecoin-project/statediff v0.0.1 h1:lym6d5wNnzr+5Uc/6RRWx1hgwb+tCKn2mFIK0Eb1Q18=
+github.com/filecoin-project/statediff v0.0.1/go.mod h1:qNWauolLFEzOiA4LNWermBRVNbaZHfPcPevumZeh+hE=
+github.com/filecoin-project/storage-fsm v0.0.0-20200805013058-9d9ea4e6331f/go.mod h1:1CGbd11KkHuyWPT+xwwCol1zl/jnlpiKD2L4fzKxaiI=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as=
 github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ=
@@ -284,7 +296,6 @@ github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJn
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -340,7 +351,6 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw=
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
@@ -393,7 +403,6 @@ github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORR
 github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f h1:KMlcu9X58lhTA/KrfX8Bi1LQSO4pzoVjTiL3h4Jk+Zk=
 github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg=
 github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
@@ -472,8 +481,9 @@ github.com/ipfs/go-bitswap v0.0.9/go.mod h1:kAPf5qgn2W2DrgAcscZ3HrM9qh4pH+X8Fkk3
 github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0=
 github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs=
 github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM=
-github.com/ipfs/go-bitswap v0.2.8 h1:5tQrbyyRS3DkzvcM5n+bVjdSAHLgvH7D+1LopndhUII=
 github.com/ipfs/go-bitswap v0.2.8/go.mod h1:2Yjog0GMdH8+AsxkE0DI9D2mANaUTxbVVav0pPoZoug=
+github.com/ipfs/go-bitswap v0.2.20 h1:Zfi5jDUoqxDThORUznqdeL77DdGniAzlccNJ4vr+Itc=
+github.com/ipfs/go-bitswap v0.2.20/go.mod h1:C7TwBgHnu89Q8sHsTJP7IhUqF9XYLe71P4tT5adgmYo=
 github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc=
 github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
@@ -526,19 +536,21 @@ github.com/ipfs/go-ds-measure v0.1.0 h1:vE4TyY4aeLeVgnnPBC5QzKIjKrqzha0NCujTfgvV
 github.com/ipfs/go-ds-measure v0.1.0/go.mod h1:1nDiFrhLlwArTME1Ees2XaBOl49OoCgd2A3f8EchMSY=
 github.com/ipfs/go-filestore v1.0.0 h1:QR7ekKH+q2AGiWDc7W2Q0qHuYSRZGUJqUn0GsegEPb0=
 github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPiFOdcuu9SM=
-github.com/ipfs/go-fs-lock v0.0.1 h1:XHX8uW4jQBYWHj59XXcjg7BHlHxV9ZOYs6Y43yb7/l0=
 github.com/ipfs/go-fs-lock v0.0.1/go.mod h1:DNBekbboPKcxs1aukPSaOtFA3QfSdi5C855v0i9XJ8Y=
-github.com/ipfs/go-graphsync v0.1.0 h1:RjLk7ha1tJtDXktqoxOjhvx4lDuzzIU+xQ+PEi74r3s=
+github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0=
+github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM=
 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE=
 github.com/ipfs/go-graphsync v0.1.1 h1:bFDAYS0Z48yd8ROPI6f/zIVmJxaDLA6m8cVuJPKC5fE=
 github.com/ipfs/go-graphsync v0.1.1/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE=
+github.com/ipfs/go-graphsync v0.1.2 h1:25Ll9kIXCE+DY0dicvfS3KMw+U5sd01b/FJbA7KAbhg=
+github.com/ipfs/go-graphsync v0.1.2/go.mod h1:sLXVXm1OxtE2XYPw62MuXCdAuNwkAdsbnfrmos5odbA=
 github.com/ipfs/go-hamt-ipld v0.0.15-0.20200131012125-dd88a59d3f2e/go.mod h1:9aQJu/i/TaRDW6jqB5U217dLIDopn50wxLdHXM2CTfE=
+github.com/ipfs/go-hamt-ipld v0.0.15-0.20200204200533-99b8553ef242/go.mod h1:kq3Pi+UP3oHhAdKexE+kHHYRKMoFNuGero0R7q3hWGg=
 github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk=
 github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk=
 github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
 github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw=
 github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ=
-github.com/ipfs/go-ipfs-blockstore v1.0.0 h1:pmFp5sFYsYVvMOp9X01AK3s85usVcLvkBTRsN6SnfUA=
 github.com/ipfs/go-ipfs-blockstore v1.0.0/go.mod h1:knLVdhVU9L7CC4T+T4nvGdeUIPAXlnd9zmXfp+9MIjU=
 github.com/ipfs/go-ipfs-blockstore v1.0.1 h1:fnuVj4XdZp4yExhd0CnUwAiMNJHiPnfInhiuwz4lW1w=
 github.com/ipfs/go-ipfs-blockstore v1.0.1/go.mod h1:MGNZlHNEnR4KGgPHM3/k8lBySIOK2Ve+0KjZubKlaOE=
@@ -613,7 +625,6 @@ github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4/go.mod h1:2v2nsGf
 github.com/ipfs/go-merkledag v0.0.3/go.mod h1:Oc5kIXLHokkE1hWGMBHw+oxehkAaTOqtEb7Zbh6BhLA=
 github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto=
 github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
-github.com/ipfs/go-merkledag v0.3.1 h1:3UqWINBEr3/N+r6OwgFXAddDP/8zpQX/8J7IGVOCqRQ=
 github.com/ipfs/go-merkledag v0.3.1/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M=
 github.com/ipfs/go-merkledag v0.3.2 h1:MRqj40QkrWkvPswXs4EfSslhZ4RVPRbxwX11js0t1xY=
 github.com/ipfs/go-merkledag v0.3.2/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M=
@@ -644,8 +655,12 @@ github.com/ipld/go-car v0.1.1-0.20200526133713-1c7508d55aae h1:OV9dxl8iPMCOD8Vi/
 github.com/ipld/go-car v0.1.1-0.20200526133713-1c7508d55aae/go.mod h1:2mvxpu4dKRnuH3mj5u6KW/tmRSCcXvy/KYiJ4nC6h4c=
 github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e h1:ZISbJlM0urTANR9KRfRaqlBmyOj5uUtxs2r4Up9IXsA=
 github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8=
+github.com/ipld/go-ipld-prime v0.0.4-0.20200828224805-5ff8c8b0b6ef h1:/yPelt/0CuzZsmRkYzBBnJ499JnAOGaIaAXHujx96ic=
+github.com/ipld/go-ipld-prime v0.0.4-0.20200828224805-5ff8c8b0b6ef/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8=
 github.com/ipld/go-ipld-prime-proto v0.0.0-20200428191222-c1ffdadc01e1 h1:K1Ysr7kgIlo7YQkPqdkA6H7BVdIugvuAz7OQUTJxLdE=
 github.com/ipld/go-ipld-prime-proto v0.0.0-20200428191222-c1ffdadc01e1/go.mod h1:OAV6xBmuTLsPZ+epzKkPB1e25FHk/vCtyatkdHcArLs=
+github.com/ipld/go-ipld-prime-proto v0.0.0-20200828231332-ae0aea07222b h1:ZtlW6pubN17TDaStlxgrwEXXwwUfJaXu9RobwczXato=
+github.com/ipld/go-ipld-prime-proto v0.0.0-20200828231332-ae0aea07222b/go.mod h1:OAV6xBmuTLsPZ+epzKkPB1e25FHk/vCtyatkdHcArLs=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4=
 github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
@@ -751,8 +766,9 @@ github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qD
 github.com/libp2p/go-libp2p v0.8.3/go.mod h1:EsH1A+8yoWK+L4iKcbPYu6MPluZ+CHWI9El8cTaefiM=
 github.com/libp2p/go-libp2p v0.9.2/go.mod h1:cunHNLDVus66Ct9iXXcjKRLdmHdFdHVe1TAnbubJQqQ=
 github.com/libp2p/go-libp2p v0.10.0/go.mod h1:yBJNpb+mGJdgrwbKAKrhPU0u3ogyNFTfjJ6bdM+Q/G8=
-github.com/libp2p/go-libp2p v0.10.3 h1:Bc8/VjmC+pICtK6xG8YgVutZvCdK0MsroWCHP+6AdFQ=
 github.com/libp2p/go-libp2p v0.10.3/go.mod h1:0ER6iPSaPeQjryNgOnm9bLNpMJCYmuw54xJXsVR17eE=
+github.com/libp2p/go-libp2p v0.11.0 h1:jb5mqdqYEBAybTEhD8io43Cz5LzVKuWxOK7znSN69jE=
+github.com/libp2p/go-libp2p v0.11.0/go.mod h1:3/ogJDXsbbepEfqtZKBR/DedzxJXCeK17t2Z9RE9bEE=
 github.com/libp2p/go-libp2p-autonat v0.0.2/go.mod h1:fs71q5Xk+pdnKU014o2iq1RhMs9/PMaG5zXRFNnIIT4=
 github.com/libp2p/go-libp2p-autonat v0.0.6/go.mod h1:uZneLdOkZHro35xIhpbtTzLlgYturpu4J5+0cZK3MqE=
 github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8=
@@ -812,7 +828,6 @@ github.com/libp2p/go-libp2p-core v0.6.1 h1:XS+Goh+QegCDojUZp00CaPMfiEADCrLjNZskW
 github.com/libp2p/go-libp2p-core v0.6.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8=
 github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE=
 github.com/libp2p/go-libp2p-crypto v0.0.2/go.mod h1:eETI5OUfBnvARGOHrJz2eWNyTUxEGZnBxMcbUjfIj4I=
-github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ=
 github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI=
 github.com/libp2p/go-libp2p-daemon v0.2.2/go.mod h1:kyrpsLB2JeNYR2rvXSVWyY0iZuRIMhqzWR3im9BV6NQ=
 github.com/libp2p/go-libp2p-discovery v0.0.1/go.mod h1:ZkkF9xIFRLA1xCc7bstYFkd80gBGK8Fc1JqGoU2i+zI=
@@ -860,7 +875,6 @@ github.com/libp2p/go-libp2p-noise v0.1.1 h1:vqYQWvnIcHpIoWJKC7Al4D6Hgj0H012TuXRh
 github.com/libp2p/go-libp2p-noise v0.1.1/go.mod h1:QDFLdKX7nluB7DEnlVPbz7xlLHdwHFA9HiohJRr3vwM=
 github.com/libp2p/go-libp2p-peer v0.0.1/go.mod h1:nXQvOBbwVqoP+T5Y5nCjeH4sP9IX/J0AMzcDUVruVoo=
 github.com/libp2p/go-libp2p-peer v0.1.1/go.mod h1:jkF12jGB4Gk/IOo+yomm+7oLWxF278F7UnrYUQ1Q8es=
-github.com/libp2p/go-libp2p-peer v0.2.0 h1:EQ8kMjaCUwt/Y5uLgjT8iY2qg0mGUT0N1zUjer50DsY=
 github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY=
 github.com/libp2p/go-libp2p-peerstore v0.0.1/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20=
 github.com/libp2p/go-libp2p-peerstore v0.0.6/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20=
@@ -880,10 +894,13 @@ github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1
 github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk=
 github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q=
 github.com/libp2p/go-libp2p-pubsub v0.3.2-0.20200527132641-c0712c6e92cf/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato=
-github.com/libp2p/go-libp2p-pubsub v0.3.5-0.20200821075113-efd56962bced h1:mgcPX7FRZNpIhg2i8B91T4fO0UEpeuhRS0YX5uMAic4=
-github.com/libp2p/go-libp2p-pubsub v0.3.5-0.20200821075113-efd56962bced/go.mod h1:DTMSVmZZfXodB/pvdTGrY2eHPZ9W2ev7hzTH83OKHrI=
+github.com/libp2p/go-libp2p-pubsub v0.3.4/go.mod h1:DTMSVmZZfXodB/pvdTGrY2eHPZ9W2ev7hzTH83OKHrI=
+github.com/libp2p/go-libp2p-pubsub v0.3.5-0.20200820194335-bfc96c2cd081/go.mod h1:DTMSVmZZfXodB/pvdTGrY2eHPZ9W2ev7hzTH83OKHrI=
+github.com/libp2p/go-libp2p-pubsub v0.3.5 h1:iF75GWpcxKEUQU8tTkgLy69qIQvfhL+t6U6ndQrB6ho=
+github.com/libp2p/go-libp2p-pubsub v0.3.5/go.mod h1:DTMSVmZZfXodB/pvdTGrY2eHPZ9W2ev7hzTH83OKHrI=
 github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU=
 github.com/libp2p/go-libp2p-quic-transport v0.5.0/go.mod h1:IEcuC5MLxvZ5KuHKjRu+dr3LjCT1Be3rcD/4d8JrX8M=
+github.com/libp2p/go-libp2p-quic-transport v0.7.1/go.mod h1:TD31to4E5exogR/GWHClXCfkktigjAl5rXSt7HoxNvY=
 github.com/libp2p/go-libp2p-quic-transport v0.8.0 h1:mHA94K2+TD0e9XtjWx/P5jGGZn0GdQ4OFYwNllagv4E=
 github.com/libp2p/go-libp2p-quic-transport v0.8.0/go.mod h1:F2FG/6Bzz0U6essUVxDzE0s9CrY4XGLbl7QEmDNvU7A=
 github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q=
@@ -901,7 +918,6 @@ github.com/libp2p/go-libp2p-secio v0.0.3/go.mod h1:hS7HQ00MgLhRO/Wyu1bTX6ctJKhVp
 github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8=
 github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g=
 github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8=
-github.com/libp2p/go-libp2p-secio v0.2.2 h1:rLLPvShPQAcY6eNurKNZq3eZjPWfU9kXF2eI9jIYdrg=
 github.com/libp2p/go-libp2p-secio v0.2.2/go.mod h1:wP3bS+m5AUnFA+OFO7Er03uO1mncHG0uVwGrwvjYlNY=
 github.com/libp2p/go-libp2p-swarm v0.0.1/go.mod h1:mh+KZxkbd3lQnveQ3j2q60BM1Cw2mX36XXQqwfPOShs=
 github.com/libp2p/go-libp2p-swarm v0.0.6/go.mod h1:s5GZvzg9xXe8sbeESuFpjt8CJPTCa8mhEusweJqyFy8=
@@ -972,12 +988,14 @@ github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO
 github.com/libp2p/go-openssl v0.0.5/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc=
 github.com/libp2p/go-openssl v0.0.7 h1:eCAzdLejcNVBzP/iZM9vqHnQm+XyCEbSSIheIPRGNsw=
 github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc=
-github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FWpFLw=
 github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA=
+github.com/libp2p/go-reuseport v0.0.2 h1:XSG94b1FJfGA01BUrT82imejHQyTxO4jEWqheyCXYvU=
+github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ=
 github.com/libp2p/go-reuseport-transport v0.0.1/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs=
 github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs=
-github.com/libp2p/go-reuseport-transport v0.0.3 h1:zzOeXnTooCkRvoH+bSXEfXhn76+LAiwoneM0gnXjF2M=
 github.com/libp2p/go-reuseport-transport v0.0.3/go.mod h1:Spv+MPft1exxARzP2Sruj2Wb5JSyHNncjf1Oi2dEbzM=
+github.com/libp2p/go-reuseport-transport v0.0.4 h1:OZGz0RB620QDGpv300n1zaOcKGGAoGVf8h9txtt/1uM=
+github.com/libp2p/go-reuseport-transport v0.0.4/go.mod h1:trPa7r/7TJK/d+0hdBLOCGvpQQVOU74OXbNCIMkufGw=
 github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k=
 github.com/libp2p/go-sockaddr v0.1.0 h1:Y4s3/jNoryVRKEBrkJ576F17CPOaMIzUeCsg7dlTDj0=
 github.com/libp2p/go-sockaddr v0.1.0/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k=
@@ -991,8 +1009,9 @@ github.com/libp2p/go-tcp-transport v0.0.1/go.mod h1:mnjg0o0O5TmXUaUIanYPUqkW4+u6
 github.com/libp2p/go-tcp-transport v0.0.4/go.mod h1:+E8HvC8ezEVOxIo3V5vCK9l1y/19K427vCzQ+xHKH/o=
 github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc=
 github.com/libp2p/go-tcp-transport v0.1.1/go.mod h1:3HzGvLbx6etZjnFlERyakbaYPdfjg2pWP97dFZworkY=
-github.com/libp2p/go-tcp-transport v0.2.0 h1:YoThc549fzmNJIh7XjHVtMIFaEDRtIrtWciG5LyYAPo=
 github.com/libp2p/go-tcp-transport v0.2.0/go.mod h1:vX2U0CnWimU4h0SGSEsg++AzvBcroCGYw28kh94oLe0=
+github.com/libp2p/go-tcp-transport v0.2.1 h1:ExZiVQV+h+qL16fzCWtd1HSzPsqWottJ8KXwWaVi8Ns=
+github.com/libp2p/go-tcp-transport v0.2.1/go.mod h1:zskiJ70MEfWz2MKxvFB/Pv+tPIB1PpPUrHIWQ8aFw7M=
 github.com/libp2p/go-testutil v0.0.1/go.mod h1:iAcJc/DKJQanJ5ws2V+u5ywdL2n12X1WbbEG+Jjy69I=
 github.com/libp2p/go-testutil v0.1.0/go.mod h1:81b2n5HypcVyrCg/MJx4Wgfp/VHojytjVe/gLzZ2Ehc=
 github.com/libp2p/go-ws-transport v0.0.1/go.mod h1:p3bKjDWHEgtuKKj+2OdPYs5dAPIjtpQGHF2tJfGz7Ww=
@@ -1016,6 +1035,7 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw=
 github.com/lucas-clemente/quic-go v0.16.0/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE=
+github.com/lucas-clemente/quic-go v0.17.3/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE=
 github.com/lucas-clemente/quic-go v0.18.0 h1:JhQDdqxdwdmGdKsKgXi1+coHRoGhvU6z0rNzOJqZ/4o=
 github.com/lucas-clemente/quic-go v0.18.0/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg=
 github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/AQ+Pg=
@@ -1027,7 +1047,6 @@ github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN
 github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
 github.com/marten-seemann/qpack v0.2.0/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
 github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
-github.com/marten-seemann/qtls v0.9.1 h1:O0YKQxNVPaiFgMng0suWEOY2Sb4LT2sRn9Qimq3Z1IQ=
 github.com/marten-seemann/qtls v0.9.1/go.mod h1:T1MmAdDPyISzxlK6kjRr0pcZFBVd1OZbBb/j3cvzHhk=
 github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl55b2pc=
 github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs=
@@ -1064,6 +1083,7 @@ github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nr
 github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/miekg/dns v1.1.30/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
 github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
@@ -1102,8 +1122,10 @@ github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lg
 github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo=
 github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4=
 github.com/multiformats/go-multiaddr v0.2.1/go.mod h1:s/Apk6IyxfvMjDafnhJgJ3/46z7tZ04iMk5wP4QMGGE=
-github.com/multiformats/go-multiaddr v0.2.2 h1:XZLDTszBIJe6m0zF6ITBrEcZR73OPUhCBBS9rYAuUzI=
 github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u0xW5UouOmQQrn6a3Y=
+github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI=
+github.com/multiformats/go-multiaddr v0.3.1 h1:1bxa+W7j9wZKTZREySx1vPMs2TqrYWjVZ7zE6/XLG1I=
+github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWzmBD8uLLL4bXW0XfGc=
 github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
 github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
 github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
@@ -1119,8 +1141,9 @@ github.com/multiformats/go-multiaddr-net v0.1.1/go.mod h1:5JNbcfBOP4dnhoZOv10JJV
 github.com/multiformats/go-multiaddr-net v0.1.2/go.mod h1:QsWt3XK/3hwvNxZJp92iMQKME1qHfpYmyIjFVsSOY6Y=
 github.com/multiformats/go-multiaddr-net v0.1.3/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA=
 github.com/multiformats/go-multiaddr-net v0.1.4/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA=
-github.com/multiformats/go-multiaddr-net v0.1.5 h1:QoRKvu0xHN1FCFJcMQLbG/yQE2z441L5urvG3+qyz7g=
 github.com/multiformats/go-multiaddr-net v0.1.5/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA=
+github.com/multiformats/go-multiaddr-net v0.2.0 h1:MSXRGN0mFymt6B1yo/6BPnIRpLPEnKgQNvVfCX5VDJk=
+github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5rnK7YaOIEOPVDI9tsJbEA=
 github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
 github.com/multiformats/go-multibase v0.0.2/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
 github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
@@ -1169,7 +1192,6 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
-github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
@@ -1177,7 +1199,6 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
-github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg=
 github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
@@ -1271,6 +1292,7 @@ github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNue
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
@@ -1403,9 +1425,7 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200504204219-64967432584d/go.mod h1:W
 github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20200715143311-227fab5a2377/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20200723185710-6a3894a6352b/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
-github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c h1:BMg3YUwLEUIYBJoYZVhA4ZDTciXRj6r7ffOCshWrsoE=
 github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
-github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c h1:otRnI08JoahNBxUFqX3372Ab9GnTj8L5J9iP5ImyxGU=
 github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20200814224545-656e08ce49ee h1:U7zWWvvAjT76EiuWPSOiZlQDnaQYPxPoxugTtTAcJK0=
 github.com/whyrusleeping/cbor-gen v0.0.0-20200814224545-656e08ce49ee/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
@@ -1431,7 +1451,12 @@ github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d/go.mod h1:g7c
 github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow=
 github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg=
 github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8=
+github.com/willscott/go-cmp v0.5.2-0.20200812183318-8affb9542345 h1:IJVAwIctqDFOrO0C2qzksXmANviyHJzrklU27e1ltzE=
+github.com/willscott/go-cmp v0.5.2-0.20200812183318-8affb9542345/go.mod h1:D7hA8H5pyQx7Y5Em7IWx1R4vNJzfon3gpG9nxjkITjQ=
 github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829 h1:wb7xrDzfkLgPHsSEBm+VSx6aDdi64VtV0xvP0E6j8bk=
 github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829/go.mod h1:h/1PEBwj7Ym/8kOuMWvO2ujZ6Lt+TMbySEXNhjjR87I=
@@ -1472,8 +1497,9 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
-go.uber.org/dig v1.8.0 h1:1rR6hnL/bu1EVcjnRDN5kx1vbIjEJDTGhSQ2B3ddpcI=
 go.uber.org/dig v1.8.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw=
+go.uber.org/dig v1.10.0 h1:yLmDDj9/zuDjv3gz8GQGviXMs9TfysIUMUilCpgzUJY=
+go.uber.org/dig v1.10.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw=
 go.uber.org/fx v1.9.0 h1:7OAz8ucp35AU8eydejpYG7QrbE8rLKzGhHbZlJi5LYY=
 go.uber.org/fx v1.9.0/go.mod h1:mFdUyAUuJ3w4jAckiKSKbldsxy1ojpAMJ+dVZg5Y0Aw=
 go.uber.org/goleak v1.0.0 h1:qsup4IcBdlmsnGfqyLl4Ntn3C2XCCuKAE7DwHpScyUo=
@@ -1492,8 +1518,9 @@ go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM=
 go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
 go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
 go4.org v0.0.0-20190218023631-ce4c26f7be8e/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
-go4.org v0.0.0-20190313082347-94abd6928b1d h1:JkRdGP3zvTtTbabWSAC6n67ka30y7gOzWAah4XYJSfw=
 go4.org v0.0.0-20190313082347-94abd6928b1d/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
+go4.org v0.0.0-20200411211856-f5505b9728dd h1:BNJlw5kRTzdmyfh5U8F93HA2OwkP7ZGwA51eJ/0wKOU=
+go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg=
 golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -1516,6 +1543,7 @@ golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -1523,7 +1551,7 @@ golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200427165652-729f1e841bcc/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM=
+golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -1558,7 +1586,6 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -1603,9 +1630,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200519113804-d87ec0cfa476/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
 golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
@@ -1676,6 +1701,7 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1737,6 +1763,7 @@ golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200108195415-316d2f248479/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -1746,13 +1773,13 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200216192241-b320d3a0f5a2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4 h1:kDtqNkeBrZb8B+atrj50B5XLHpzXXqcCdZPP/ApQ5NY=
 golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
 golang.org/x/tools v0.0.0-20200711155855-7342f9734a7d h1:F3OmlXCzYtG9YE6tXDnUOlJBzVzHF8EcmZ1yTJlcgIk=
 golang.org/x/tools v0.0.0-20200711155855-7342f9734a7d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200827010519-17fd2f27a9e3 h1:r3P/5xOq/dK1991B65Oy6E1fRF/2d/fSYZJ/fXGVfJc=
+golang.org/x/tools v0.0.0-20200827010519-17fd2f27a9e3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1838,7 +1865,6 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1863,7 +1889,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

lib/rpcenc/reader_test.go

@@ -49,7 +49,7 @@ func TestReaderProxy(t *testing.T) {
 	defer testServ.Close()
 
 	re := ReaderParamEncoder("http://" + testServ.Listener.Addr().String() + "/rpc/streams/v0/push")
-	closer, err := jsonrpc.NewMergeClient("ws://"+testServ.Listener.Addr().String()+"/rpc/v0", "ReaderHandler", []interface{}{&client}, nil, re)
+	closer, err := jsonrpc.NewMergeClient(context.Background(), "ws://"+testServ.Listener.Addr().String()+"/rpc/v0", "ReaderHandler", []interface{}{&client}, nil, re)
 	require.NoError(t, err)
 
 	defer closer()
@@ -79,7 +79,7 @@ func TestNullReaderProxy(t *testing.T) {
 	defer testServ.Close()
 
 	re := ReaderParamEncoder("http://" + testServ.Listener.Addr().String() + "/rpc/streams/v0/push")
-	closer, err := jsonrpc.NewMergeClient("ws://"+testServ.Listener.Addr().String()+"/rpc/v0", "ReaderHandler", []interface{}{&client}, nil, re)
+	closer, err := jsonrpc.NewMergeClient(context.Background(), "ws://"+testServ.Listener.Addr().String()+"/rpc/v0", "ReaderHandler", []interface{}{&client}, nil, re)
 	require.NoError(t, err)
 
 	defer closer()

lotuspond/api.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"crypto/rand"
 	"io"
 	"io/ioutil"
@@ -129,9 +130,9 @@ type client struct {
 	Nodes func() []nodeInfo
 }
 
-func apiClient() (*client, error) {
+func apiClient(ctx context.Context) (*client, error) {
 	c := &client{}
-	if _, err := jsonrpc.NewClient("ws://"+listenAddr+"/rpc/v0", "Pond", c, nil); err != nil {
+	if _, err := jsonrpc.NewClient(ctx, "ws://"+listenAddr+"/rpc/v0", "Pond", c, nil); err != nil {
 		return nil, err
 	}
 	return c, nil

lotuspond/main.go

@@ -27,7 +27,7 @@ var onCmd = &cli.Command{
 	Name:  "on",
 	Usage: "run a command on a given node",
 	Action: func(cctx *cli.Context) error {
-		client, err := apiClient()
+		client, err := apiClient(cctx.Context)
 		if err != nil {
 			return err
 		}
@@ -65,7 +65,7 @@ var shCmd = &cli.Command{
 	Name:  "sh",
 	Usage: "spawn shell with node shell variables set",
 	Action: func(cctx *cli.Context) error {
-		client, err := apiClient()
+		client, err := apiClient(cctx.Context)
 		if err != nil {
 			return err
 		}

markets/storageadapter/provider.go

@@ -87,8 +87,13 @@ func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemark
 }
 
 func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagemarket.MinerDeal, pieceSize abi.UnpaddedPieceSize, pieceData io.Reader) (*storagemarket.PackingResult, error) {
+	if deal.PublishCid == nil {
+		return nil, xerrors.Errorf("deal.PublishCid can't be nil")
+	}
+
 	p, offset, err := n.secb.AddPiece(ctx, pieceSize, pieceData, sealing.DealInfo{
-		DealID: deal.DealID,
+		DealID:     deal.DealID,
+		PublishCid: deal.PublishCid,
 		DealSchedule: sealing.DealSchedule{
 			StartEpoch: deal.ClientDealProposal.Proposal.StartEpoch,
 			EndEpoch:   deal.ClientDealProposal.Proposal.EndEpoch,
@@ -351,7 +356,7 @@ func (n *ProviderNodeAdapter) GetChainHead(ctx context.Context) (shared.TipSetTo
 }
 
 func (n *ProviderNodeAdapter) WaitForMessage(ctx context.Context, mcid cid.Cid, cb func(code exitcode.ExitCode, bytes []byte, err error) error) error {
-	receipt, err := n.StateWaitMsg(ctx, mcid, build.MessageConfidence)
+	receipt, err := n.StateWaitMsg(ctx, mcid, 2*build.MessageConfidence)
 	if err != nil {
 		return cb(0, nil, err)
 	}

metrics/metrics.go

@@ -19,6 +19,7 @@ var (
 	Commit, _       = tag.NewKey("commit")
 	PeerID, _       = tag.NewKey("peer_id")
 	FailureType, _  = tag.NewKey("failure_type")
+	Local, _        = tag.NewKey("local")
 	MessageFrom, _  = tag.NewKey("message_from")
 	MessageTo, _    = tag.NewKey("message_to")
 	MessageNonce, _ = tag.NewKey("message_nonce")
@@ -30,7 +31,7 @@ var (
 	LotusInfo                           = stats.Int64("info", "Arbitrary counter to tag lotus info to", stats.UnitDimensionless)
 	ChainNodeHeight                     = stats.Int64("chain/node_height", "Current Height of the node", stats.UnitDimensionless)
 	ChainNodeWorkerHeight               = stats.Int64("chain/node_worker_height", "Current Height of workers on the node", stats.UnitDimensionless)
-	MessagePublished                    = stats.Int64("message/pubished", "Counter for total locally published messages", stats.UnitDimensionless)
+	MessagePublished                    = stats.Int64("message/published", "Counter for total locally published messages", stats.UnitDimensionless)
 	MessageReceived                     = stats.Int64("message/received", "Counter for total received messages", stats.UnitDimensionless)
 	MessageValidationFailure            = stats.Int64("message/failure", "Counter for message validation failures", stats.UnitDimensionless)
 	MessageValidationSuccess            = stats.Int64("message/success", "Counter for message validation successes", stats.UnitDimensionless)
@@ -82,6 +83,10 @@ var (
 		Measure:     BlockValidationDurationMilliseconds,
 		Aggregation: defaultMillisecondsDistribution,
 	}
+	MessagePublishedView = &view.View{
+		Measure:     MessagePublished,
+		Aggregation: view.Count(),
+	}
 	MessageReceivedView = &view.View{
 		Measure:     MessageReceived,
 		Aggregation: view.Count(),
@@ -89,7 +94,7 @@ var (
 	MessageValidationFailureView = &view.View{
 		Measure:     MessageValidationFailure,
 		Aggregation: view.Count(),
-		TagKeys:     []tag.Key{FailureType},
+		TagKeys:     []tag.Key{FailureType, Local},
 	}
 	MessageValidationSuccessView = &view.View{
 		Measure:     MessageValidationSuccess,
@@ -99,6 +104,34 @@ var (
 		Measure:     PeerCount,
 		Aggregation: view.LastValue(),
 	}
+	PubsubPublishMessageView = &view.View{
+		Measure:     PubsubPublishMessage,
+		Aggregation: view.Count(),
+	}
+	PubsubDeliverMessageView = &view.View{
+		Measure:     PubsubDeliverMessage,
+		Aggregation: view.Count(),
+	}
+	PubsubRejectMessageView = &view.View{
+		Measure:     PubsubRejectMessage,
+		Aggregation: view.Count(),
+	}
+	PubsubDuplicateMessageView = &view.View{
+		Measure:     PubsubDuplicateMessage,
+		Aggregation: view.Count(),
+	}
+	PubsubRecvRPCView = &view.View{
+		Measure:     PubsubRecvRPC,
+		Aggregation: view.Count(),
+	}
+	PubsubSendRPCView = &view.View{
+		Measure:     PubsubSendRPC,
+		Aggregation: view.Count(),
+	}
+	PubsubDropRPCView = &view.View{
+		Measure:     PubsubDropRPC,
+		Aggregation: view.Count(),
+	}
 )
 
 // DefaultViews is an array of OpenCensus views for metric gathering purposes
@@ -110,10 +143,18 @@ var DefaultViews = append([]*view.View{
 	BlockValidationFailureView,
 	BlockValidationSuccessView,
 	BlockValidationDurationView,
+	MessagePublishedView,
 	MessageReceivedView,
 	MessageValidationFailureView,
 	MessageValidationSuccessView,
 	PeerCountView,
+	PubsubPublishMessageView,
+	PubsubDeliverMessageView,
+	PubsubRejectMessageView,
+	PubsubDuplicateMessageView,
+	PubsubRecvRPCView,
+	PubsubSendRPCView,
+	PubsubDropRPCView,
 },
 	rpcmetrics.DefaultViews...)
 

miner/miner.go

@@ -32,7 +32,7 @@ import (
 var log = logging.Logger("miner")
 
 // returns a callback reporting whether we mined a blocks in this round
-type waitFunc func(ctx context.Context, baseTime uint64) (func(bool, error), abi.ChainEpoch, error)
+type waitFunc func(ctx context.Context, baseTime uint64) (func(bool, abi.ChainEpoch, error), abi.ChainEpoch, error)
 
 func randTimeOffset(width time.Duration) time.Duration {
 	buf := make([]byte, 8)
@@ -52,7 +52,7 @@ func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address,
 		api:     api,
 		epp:     epp,
 		address: addr,
-		waitFunc: func(ctx context.Context, baseTime uint64) (func(bool, error), abi.ChainEpoch, error) {
+		waitFunc: func(ctx context.Context, baseTime uint64) (func(bool, abi.ChainEpoch, error), abi.ChainEpoch, error) {
 			// Wait around for half the block time in case other parents come in
 			deadline := baseTime + build.PropagationDelaySecs
 			baseT := time.Unix(int64(deadline), 0)
@@ -61,7 +61,7 @@ func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address,
 
 			build.Clock.Sleep(build.Clock.Until(baseT))
 
-			return func(bool, error) {}, 0, nil
+			return func(bool, abi.ChainEpoch, error) {}, 0, nil
 		},
 
 		sf:                sf,
@@ -150,7 +150,7 @@ func (m *Miner) mine(ctx context.Context) {
 		}
 
 		var base *MiningBase
-		var onDone func(bool, error)
+		var onDone func(bool, abi.ChainEpoch, error)
 		var injectNulls abi.ChainEpoch
 
 		for {
@@ -166,7 +166,7 @@ func (m *Miner) mine(ctx context.Context) {
 				break
 			}
 			if base != nil {
-				onDone(false, nil)
+				onDone(false, 0, nil)
 			}
 
 			// TODO: need to change the orchestration here. the problem is that
@@ -205,12 +205,16 @@ func (m *Miner) mine(ctx context.Context) {
 		if err != nil {
 			log.Errorf("mining block failed: %+v", err)
 			m.niceSleep(time.Second)
-			onDone(false, err)
+			onDone(false, 0, err)
 			continue
 		}
 		lastBase = *base
 
-		onDone(b != nil, nil)
+		var h abi.ChainEpoch
+		if b != nil {
+			h = b.Header.Height
+		}
+		onDone(b != nil, h, nil)
 
 		if b != nil {
 			journal.Add("blockMined", map[string]interface{}{

miner/testminer.go

@@ -15,7 +15,7 @@ import (
 
 type MineReq struct {
 	InjectNulls abi.ChainEpoch
-	Done        func(bool, error)
+	Done        func(bool, abi.ChainEpoch, error)
 }
 
 func NewTestMiner(nextCh <-chan MineReq, addr address.Address) func(api.FullNode, gen.WinningPoStProver) *Miner {
@@ -41,8 +41,8 @@ func NewTestMiner(nextCh <-chan MineReq, addr address.Address) func(api.FullNode
 	}
 }
 
-func chanWaiter(next <-chan MineReq) func(ctx context.Context, _ uint64) (func(bool, error), abi.ChainEpoch, error) {
-	return func(ctx context.Context, _ uint64) (func(bool, error), abi.ChainEpoch, error) {
+func chanWaiter(next <-chan MineReq) func(ctx context.Context, _ uint64) (func(bool, abi.ChainEpoch, error), abi.ChainEpoch, error) {
+	return func(ctx context.Context, _ uint64) (func(bool, abi.ChainEpoch, error), abi.ChainEpoch, error) {
 		select {
 		case <-ctx.Done():
 			return nil, 0, ctx.Err()

node/impl/client/client.go

@@ -223,6 +223,21 @@ func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo,
 	}, nil
 }
 
+func (a *API) ClientGetDealUpdates(ctx context.Context) (<-chan api.DealInfo, error) {
+	updates := make(chan api.DealInfo)
+
+	unsub := a.SMDealClient.SubscribeToEvents(func(_ storagemarket.ClientEvent, deal storagemarket.ClientDeal) {
+		updates <- newDealInfo(deal)
+	})
+
+	go func() {
+		defer unsub()
+		<-ctx.Done()
+	}()
+
+	return updates, nil
+}
+
 func (a *API) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) {
 	// TODO: check if we have the ENTIRE dag
 
@@ -432,6 +447,45 @@ func (a *API) ClientRetrieveWithEvents(ctx context.Context, order api.RetrievalO
 	return events, nil
 }
 
+type retrievalSubscribeEvent struct {
+	event rm.ClientEvent
+	state rm.ClientDealState
+}
+
+func readSubscribeEvents(ctx context.Context, subscribeEvents chan retrievalSubscribeEvent, events chan marketevents.RetrievalEvent) error {
+	for {
+		var subscribeEvent retrievalSubscribeEvent
+		select {
+		case <-ctx.Done():
+			return xerrors.New("Retrieval Timed Out")
+		case subscribeEvent = <-subscribeEvents:
+		}
+
+		select {
+		case <-ctx.Done():
+			return xerrors.New("Retrieval Timed Out")
+		case events <- marketevents.RetrievalEvent{
+			Event:         subscribeEvent.event,
+			Status:        subscribeEvent.state.Status,
+			BytesReceived: subscribeEvent.state.TotalReceived,
+			FundsSpent:    subscribeEvent.state.FundsSpent,
+		}:
+		}
+
+		state := subscribeEvent.state
+		switch state.Status {
+		case rm.DealStatusCompleted:
+			return nil
+		case rm.DealStatusRejected:
+			return xerrors.Errorf("Retrieval Proposal Rejected: %s", state.Message)
+		case
+			rm.DealStatusDealNotFound,
+			rm.DealStatusErrored:
+			return xerrors.Errorf("Retrieval Error: %s", state.Message)
+		}
+	}
+}
+
 func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef, events chan marketevents.RetrievalEvent) {
 	defer close(events)
 
@@ -467,33 +521,15 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
 		return err
 	}*/
 
-	retrievalResult := make(chan error, 1)
-
-	var dealId retrievalmarket.DealID
-
+	var dealID retrievalmarket.DealID
+	subscribeEvents := make(chan retrievalSubscribeEvent, 1)
+	subscribeCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	unsubscribe := a.Retrieval.SubscribeToEvents(func(event rm.ClientEvent, state rm.ClientDealState) {
-		if state.PayloadCID.Equals(order.Root) && state.ID == dealId {
-
+		if state.PayloadCID.Equals(order.Root) && state.ID == dealID {
 			select {
-			case <-ctx.Done():
-				return
-			case events <- marketevents.RetrievalEvent{
-				Event:         event,
-				Status:        state.Status,
-				BytesReceived: state.TotalReceived,
-				FundsSpent:    state.FundsSpent,
-			}:
-			}
-
-			switch state.Status {
-			case rm.DealStatusCompleted:
-				retrievalResult <- nil
-			case rm.DealStatusRejected:
-				retrievalResult <- xerrors.Errorf("Retrieval Proposal Rejected: %s", state.Message)
-			case
-				rm.DealStatusDealNotFound,
-				rm.DealStatusErrored:
-				retrievalResult <- xerrors.Errorf("Retrieval Error: %s", state.Message)
+			case <-subscribeCtx.Done():
+			case subscribeEvents <- retrievalSubscribeEvent{event, state}:
 			}
 		}
 	})
@@ -516,7 +552,7 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
 		_ = a.RetrievalStoreMgr.ReleaseStore(store)
 	}()
 
-	dealId, err = a.Retrieval.Retrieve(
+	dealID, err = a.Retrieval.Retrieve(
 		ctx,
 		order.Root,
 		params,
@@ -531,17 +567,12 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
 		return
 	}
 
-	select {
-	case <-ctx.Done():
-		unsubscribe()
-		finish(xerrors.New("Retrieval Timed Out"))
+	err = readSubscribeEvents(ctx, subscribeEvents, events)
+
+	unsubscribe()
+	if err != nil {
+		finish(xerrors.Errorf("Retrieve: %w", err))
 		return
-	case err := <-retrievalResult:
-		unsubscribe()
-		if err != nil {
-			finish(xerrors.Errorf("Retrieve: %w", err))
-			return
-		}
 	}
 
 	// If ref is nil, it only fetches the data into the configured blockstore.
@@ -800,3 +831,19 @@ func (a *API) ClientDataTransferUpdates(ctx context.Context) (<-chan api.DataTra
 
 	return channels, nil
 }
+
+func newDealInfo(v storagemarket.ClientDeal) api.DealInfo {
+	return api.DealInfo{
+		ProposalCid:   v.ProposalCid,
+		DataRef:       v.DataRef,
+		State:         v.State,
+		Message:       v.Message,
+		Provider:      v.Proposal.Provider,
+		PieceCID:      v.Proposal.PieceCID,
+		Size:          uint64(v.Proposal.PieceSize.Unpadded()),
+		PricePerEpoch: v.Proposal.StoragePricePerEpoch,
+		Duration:      uint64(v.Proposal.Duration()),
+		DealID:        v.DealID,
+		CreationTime:  v.CreationTime.Time(),
+	}
+}

node/impl/paych/paych.go

@@ -2,18 +2,15 @@ package paych
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/ipfs/go-cid"
 	"go.uber.org/fx"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/paych"
 
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/types"
 	full "github.com/filecoin-project/lotus/node/impl/full"
 	"github.com/filecoin-project/lotus/paychmgr"
@@ -183,35 +180,6 @@ func (a *PaychAPI) PaychVoucherList(ctx context.Context, pch address.Address) ([
 	return out, nil
 }
 
-func (a *PaychAPI) PaychVoucherSubmit(ctx context.Context, ch address.Address, sv *paych.SignedVoucher) (cid.Cid, error) {
-	ci, err := a.PaychMgr.GetChannelInfo(ch)
-	if err != nil {
-		return cid.Undef, err
-	}
-
-	if sv.Extra != nil || len(sv.SecretPreimage) > 0 {
-		return cid.Undef, fmt.Errorf("cant handle more advanced payment channel stuff yet")
-	}
-
-	enc, err := actors.SerializeParams(&paych.UpdateChannelStateParams{
-		Sv: *sv,
-	})
-	if err != nil {
-		return cid.Undef, err
-	}
-
-	msg := &types.Message{
-		From:   ci.Control,
-		To:     ch,
-		Value:  types.NewInt(0),
-		Method: builtin.MethodsPaych.UpdateChannelState,
-		Params: enc,
-	}
-
-	smsg, err := a.MpoolPushMessage(ctx, msg, nil)
-	if err != nil {
-		return cid.Undef, err
-	}
-
-	return smsg.Cid(), nil
+func (a *PaychAPI) PaychVoucherSubmit(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) {
+	return a.PaychMgr.SubmitVoucher(ctx, ch, sv, secret, proof)
 }

node/impl/remoteworker.go

@@ -33,7 +33,7 @@ func connectRemoteWorker(ctx context.Context, fa api.Common, url string) (*remot
 	headers := http.Header{}
 	headers.Add("Authorization", "Bearer "+string(token))
 
-	wapi, closer, err := client.NewWorkerRPC(url, headers)
+	wapi, closer, err := client.NewWorkerRPC(context.TODO(), url, headers)
 	if err != nil {
 		return nil, xerrors.Errorf("creating jsonrpc client: %w", err)
 	}

node/impl/storminer.go

@@ -153,8 +153,10 @@ func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumb
 			Value: info.SeedValue,
 			Epoch: info.SeedEpoch,
 		},
-		Retries:   info.InvalidProofs,
-		ToUpgrade: sm.Miner.IsMarkedForUpgrade(sid),
+		PreCommitMsg: info.PreCommitMessage,
+		CommitMsg:    info.CommitMessage,
+		Retries:      info.InvalidProofs,
+		ToUpgrade:    sm.Miner.IsMarkedForUpgrade(sid),
 
 		LastErr: info.LastErr,
 		Log:     log,
@@ -175,6 +177,9 @@ func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumb
 
 	onChainInfo, err := sm.Full.StateSectorGetInfo(ctx, sm.Miner.Address(), sid, types.EmptyTSK)
 	if err != nil {
+		return sInfo, err
+	}
+	if onChainInfo == nil {
 		return sInfo, nil
 	}
 	sInfo.SealProof = onChainInfo.SealProof