Merge pull request #3725 from filecoin-project/asr/specs-update-0.9.8

Update to specs-actors v0.9.8

.circleci/config.yml

@@ -113,7 +113,7 @@ jobs:
   test: &test
     description: |
       Run tests with gotestsum.
-    parameters:
+    parameters: &test-params
       executor:
         type: executor
         default: golang
@@ -161,6 +161,7 @@ jobs:
           name: go test
           environment:
             LOTUS_TEST_WINDOW_POST: << parameters.winpost-test >>
+            SKIP_CONFORMANCE: "1"
           command: |
             mkdir -p /tmp/test-reports/<< parameters.test-suite-name >>
             mkdir -p /tmp/test-artifacts
@@ -191,6 +192,68 @@ jobs:
     <<: *test
   test-window-post:
     <<: *test
+  test-conformance:
+    description: |
+      Run tests using a corpus of interoperable test vectors for Filecoin 
+      implementations to test their correctness and compliance with the Filecoin
+      specifications.
+    parameters:
+      <<: *test-params
+      vectors-branch:
+        type: string
+        default: ""
+        description: |
+          Branch on github.com/filecoin-project/test-vectors to checkout and
+          test with. If empty (the default) the commit defined by the git
+          submodule is used.
+    executor: << parameters.executor >>
+    steps:
+      - install-deps
+      - prepare
+      - run:
+          command: make deps lotus
+          no_output_timeout: 30m
+      - download-params
+      - when:
+          condition:
+            not:
+              equal: [ "", << parameters.vectors-branch >> ]
+          steps:
+            - run:
+                name: checkout vectors branch
+                command: |
+                  cd extern/test-vectors
+                  git fetch
+                  git checkout origin/<< parameters.vectors-branch >>
+      - go/install-gotestsum:
+          gobin: $HOME/.local/bin
+          version: 0.5.2
+      - run:
+          name: install statediff globally
+          command: |
+            ## statediff is optional; we succeed even if compilation fails.
+            mkdir -p /tmp/statediff
+            git clone https://github.com/filecoin-project/statediff.git /tmp/statediff
+            cd /tmp/statediff
+            go install ./cmd/statediff || exit 0
+      - run:
+          name: go test
+          environment:
+            SKIP_CONFORMANCE: "0"
+          command: |
+            mkdir -p /tmp/test-reports
+            mkdir -p /tmp/test-artifacts
+            gotestsum \
+              --format pkgname-and-test-fails \
+              --junitfile /tmp/test-reports/junit.xml \
+              -- \
+              -v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/
+            go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html
+          no_output_timeout: 30m
+      - store_test_results:
+          path: /tmp/test-reports
+      - store_artifacts:
+          path: /tmp/test-artifacts/conformance-coverage.html
 
   build-macos:
     description: build darwin lotus binary
@@ -252,6 +315,27 @@ jobs:
       - run:
           command: "! go fmt ./... 2>&1 | read"
 
+  cbor-gen-check:
+    executor: golang
+    steps:
+      - install-deps
+      - prepare
+      - run: make deps
+      - run: go install golang.org/x/tools/cmd/goimports
+      - run: go install github.com/hannahhoward/cbor-gen-for
+      - run: go generate ./...
+      - run: git --no-pager diff
+      - run: git --no-pager diff --quiet
+
+  docs-check:
+    executor: golang
+    steps:
+      - install-deps
+      - prepare
+      - run: make docsgen
+      - run: git --no-pager diff
+      - run: git --no-pager diff --quiet
+
   lint: &lint
     description: |
       Run golangci-lint.
@@ -288,9 +372,6 @@ jobs:
           command: |
             $HOME/.local/bin/golangci-lint run -v --timeout 2m \
               --concurrency << parameters.concurrency >> << parameters.args >>
-  lint-changes:
-    <<: *lint
-
   lint-all:
     <<: *lint
 
@@ -319,10 +400,11 @@ workflows:
   version: 2.1
   ci:
     jobs:
-      - lint-changes:
-          args: "--new-from-rev origin/next"
+      - lint-all
       - mod-tidy-check
       - gofmt
+      - cbor-gen-check
+      - docs-check
       - test:
           codecov-upload: true
           test-suite-name: full
@@ -337,6 +419,14 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+$/
+      - test-conformance:
+          test-suite-name: conformance
+          packages: "./conformance"
+      - test-conformance:
+          name: test-conformance-bleeding-edge
+          test-suite-name: conformance-bleeding-edge
+          packages: "./conformance"
+          vectors-branch: master
       - build-debug
       - build-all:
           requires:

.codecov.yml

@@ -1,3 +1,5 @@
 comment: off
 ignore:
   - "cbor_gen.go"
+github_checks:
+  annotations: false

.github/CODEOWNERS

@@ -0,0 +1,15 @@
+## filecoin-project/lotus CODEOWNERS
+## Refer to https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners.
+##
+## These users or groups will be automatically assigned as reviewers every time
+## a PR is submitted that modifies code in the specified locations.
+##
+## The Lotus repo configuration requires that at least ONE codeowner approves
+## the PR before merging.
+
+### Global owners.
+*                       @magik6k @whyrusleeping @Kubuxu
+
+### Conformance testing.
+conformance/            @raulk
+extern/test-vectors     @raulk

.github/ISSUE_TEMPLATE/sealingfailed.md

@@ -17,18 +17,27 @@ A brief description of the problem you encountered while proving (sealing) a sec
 
 Including what commands you ran, and a description of your setup, is very helpful.
 
-**Sectors list**
-
-The output of `./lotus-miner sectors list`.
-
 **Sectors status**
 
-The output of `./lotus-miner sectors status --log <sectorId>` for the failed sector(s).
+The output of `lotus-miner sectors status --log <sectorId>` for the failed sector(s).
 
 **Lotus miner logs**
 
 Please go through the logs of your miner, and include screenshots of any error-like messages you find.
 
+Alternatively please upload full log files and share a link here
+
+**Lotus miner diagnostic info**
+
+Please collect the following diagnostic information, and share a link here
+
+* lotus-miner diagnostic info `lotus-miner info all > allinfo`
+
+** Code modifications **
+
+If you have modified parts of lotus, please describe which areas were modified,
+and the scope of those modifications
+
 **Version**
 
-The output of `./lotus --version`.
+The output of `lotus --version`.

.github/labels.yml

@@ -26,11 +26,14 @@
   color: 00A4E2
   description: "Area: Chain/VM"
 - name: area/chain/sync
-  color: 00A4E2
+  color: 00A4E4
   description: "Area: Chain/Sync"
 - name: area/chain/misc
-  color: 00A4E2
+  color: 00A4E6
   description: "Area: Chain/Misc"
+- name: area/markets
+  color: 00A4E8
+  description: "Area: Markets"  
 - name: area/sealing/fsm
   color: 0bb1ed
   description: "Area: Sealing/FSM"
@@ -149,6 +152,9 @@
 - name: impact/test-flakiness
   color: DDE1E4
   description: "Impact: Test Flakiness"
+- name: impact/consensus
+  color: b20014
+  description: "Impact: Consensus"
 
 ###
 ### Topics

.gitmodules

@@ -5,3 +5,6 @@
 [submodule "extern/serialization-vectors"]
 	path = extern/serialization-vectors
 	url = https://github.com/filecoin-project/serialization-vectors
+[submodule "extern/test-vectors"]
+	path = extern/test-vectors
+	url = https://github.com/filecoin-project/test-vectors.git

.golangci.yml

@@ -23,6 +23,14 @@ issues:
     - "Potential file inclusion via variable"
     - "should have( a package)? comment"
     - "Error return value of `logging.SetLogLevel` is not checked"
+    - "comment on exported"
+    - "(func|method) \\w+ should be \\w+"
+    - "(type|var|struct field|(method|func) parameter) `\\w+` should be `\\w+`"
+    - "(G306|G301|G307|G108|G302|G204|G104)"
+    - "don't use ALL_CAPS in Go names"
+    - "string .* has .* occurrences, make it a constant"
+    - "a blank import should be only in a main or test package, or have a comment justifying it"
+    - "package comment should be of the form"
 
   exclude-use-default: false
   exclude-rules:
@@ -46,6 +54,19 @@ issues:
       linters:
         - gosec
 
+    - path: chain/vectors/gen/.*
+      linters:
+        - gosec
+
+    - path: cmd/lotus-bench/.*
+      linters:
+        - gosec
+
+    - path: api/test/.*
+      text: "context.Context should be the first parameter"
+      linters:
+        - golint
+
 linters-settings:
   goconst:
     min-occurrences: 6

CHANGELOG.md

@@ -1,7 +1,399 @@
-# lotus changelog
+# Lotus changelog
 
-## 0.1.0 / 2019-12-11
+# 0.6.2 / 2020-09-09
 
-We are very excited to release **lotus** 0.1.0. This is our testnet release. To install lotus and join the testnet, please visit [docs.lotu.sh](docs.lotu.sh). Please file bug reports as [issues](https://github.com/filecoin-project/lotus/issues).
+This release introduces some critical fixes to message selection and gas estimation logic. It also adds the ability for nodes to mark a certain tipset as checkpointed, as well as various minor improvements and bugfixes.
 
-A huge thank you to all contributors for this testnet release!
+## Changes
+
+#### Messagepool 
+
+- Warn when optimal selection fails to pack a block and we fall back to random selection (https://github.com/filecoin-project/lotus/pull/3708)
+- Add basic command for printing gas performance of messages in the mpool (https://github.com/filecoin-project/lotus/pull/3701)
+- Adjust optimal selection to always try to fill blocks (https://github.com/filecoin-project/lotus/pull/3685)
+- Fix very minor bug in repub baseFeeLowerBound (https://github.com/filecoin-project/lotus/pull/3663)
+- Add an auto flag to mpool replace (https://github.com/filecoin-project/lotus/pull/3676)
+- Fix mpool optimal selection packing failure (https://github.com/filecoin-project/lotus/pull/3698)
+
+#### Core Lotus
+
+- Don't use latency as initital estimate for blocksync (https://github.com/filecoin-project/lotus/pull/3648)
+- Add niceSleep 1 second when drand errors (https://github.com/filecoin-project/lotus/pull/3664)
+- Fix isChainNearSync check in block validator (https://github.com/filecoin-project/lotus/pull/3650)
+- Add peer to peer manager before fetching the tipset (https://github.com/filecoin-project/lotus/pull/3667)
+- Add StageFetchingMessages to sync status (https://github.com/filecoin-project/lotus/pull/3668)
+- Pass tipset through upgrade logic (https://github.com/filecoin-project/lotus/pull/3673)
+- Allow nodes to mark tipsets as checkpointed (https://github.com/filecoin-project/lotus/pull/3680)
+- Remove hard-coded late-fee in window PoSt (https://github.com/filecoin-project/lotus/pull/3702)
+- Gas: Fix median calc (https://github.com/filecoin-project/lotus/pull/3686)
+
+#### Storage
+
+- Storage manager: bail out with an error if unsealed cid is undefined (https://github.com/filecoin-project/lotus/pull/3655)
+- Storage: return true from Sealer.ReadPiece() on success (https://github.com/filecoin-project/lotus/pull/3657)
+
+#### Maintenance
+
+- Resolve lotus, test-vectors, statediff dependency cycle (https://github.com/filecoin-project/lotus/pull/3688)
+- Paych: add docs on how to use paych status (https://github.com/filecoin-project/lotus/pull/3690)
+- Initial CODEOWNERS (https://github.com/filecoin-project/lotus/pull/3691)
+
+
+# 0.6.1 / 2020-09-08
+
+This optional release introduces a minor improvement to the sync process, ensuring nodes don't fall behind and then resync.
+
+## Changes
+
+- Update `test-vectors` (https://github.com/filecoin-project/lotus/pull/3645)
+- Revert "only subscribe to pubsub topics once we are synced" (https://github.com/filecoin-project/lotus/pull/3643)
+
+# 0.6.0 / 2020-09-07
+
+This consensus-breaking release of Lotus is designed to test a network upgrade on the space race testnet. The changes that break consensus are:
+
+- Tweaking of some cryptoecon parameters in specs-actors 0.9.7 (https://github.com/filecoin-project/specs-actors/releases/tag/v0.9.7)
+- Rebalancing FIL distribution to make testnet FIL scarce, which prevents base fee spikes and sets better expectations for mainnet
+
+This release also introduces many improvements to Lotus! Among them are a new version of go-fil-markets that supports non-blocking retrieval, various spam reduction measures in the messagepool and p2p logic, and UX improvements to payment channels, dealmaking, and state inspection.
+
+## Changes
+
+#### Core Lotus and dependencies
+
+- Implement faucet funds reallocation logic (https://github.com/filecoin-project/lotus/pull/3632)
+- Network upgrade: Upgrade to correct fork threshold (https://github.com/filecoin-project/lotus/pull/3628)
+- Update to specs 0.9.7 and markets 0.6.0 (https://github.com/filecoin-project/lotus/pull/3627)
+- Network upgrade: Perform base fee tamping (https://github.com/filecoin-project/lotus/pull/3623)
+- Chain events: if cache best() is nil, return chain head (https://github.com/filecoin-project/lotus/pull/3611)
+- Update to specs actors v0.9.6 (https://github.com/filecoin-project/lotus/pull/3603)
+
+#### Messagepool
+
+- Temporarily allow negative chains (https://github.com/filecoin-project/lotus/pull/3625)
+- Improve publish/republish logic (https://github.com/filecoin-project/lotus/pull/3592)
+- Fix selection bug; priority messages were not included if other chains were negative (https://github.com/filecoin-project/lotus/pull/3580)
+- Add defensive check for minimum GasFeeCap for inclusion within the next 20 blocks (https://github.com/filecoin-project/lotus/pull/3579)
+- Add additional info about gas premium (https://github.com/filecoin-project/lotus/pull/3578)
+- Fix GasPremium capping logic  (https://github.com/filecoin-project/lotus/pull/3552)
+
+#### Payment channels 
+
+- Get available funds by address or by from/to (https://github.com/filecoin-project/lotus/pull/3547)
+- Create `lotus paych status` command (https://github.com/filecoin-project/lotus/pull/3523)
+- Rename CLI command from "paych get" to "paych add-funds" (https://github.com/filecoin-project/lotus/pull/3520)
+
+#### Peer-to-peer
+
+- Only subscribe to pubsub topics once we are synced (https://github.com/filecoin-project/lotus/pull/3602)
+- Reduce mpool add failure log spam (https://github.com/filecoin-project/lotus/pull/3562)
+- Republish messages even if the chains have negative performance(https://github.com/filecoin-project/lotus/pull/3557)
+- Adjust gossipsub gossip factor (https://github.com/filecoin-project/lotus/pull/3556)
+- Integrate pubsub Random Early Drop (https://github.com/filecoin-project/lotus/pull/3518)
+
+#### Miscellaneous
+
+- Fix panic in OnDealExpiredSlashed (https://github.com/filecoin-project/lotus/pull/3553)
+- Robustify state manager against holes in actor method numbers (https://github.com/filecoin-project/lotus/pull/3538)
+
+#### UX
+
+- VM: Fix an error message (https://github.com/filecoin-project/lotus/pull/3608)
+- Documentation: Batch replacement,update lotus-storage-miner to lotus-miner (https://github.com/filecoin-project/lotus/pull/3571)
+- CLI: Robust actor lookup (https://github.com/filecoin-project/lotus/pull/3535)
+- Add agent flag to net peers (https://github.com/filecoin-project/lotus/pull/3534)
+- Add watch option to storage-deals list (https://github.com/filecoin-project/lotus/pull/3527)
+
+#### Testing & tooling
+
+- Decommission chain-validation (https://github.com/filecoin-project/lotus/pull/3606)
+- Metrics: add expected height metric (https://github.com/filecoin-project/lotus/pull/3586)
+- PCR: Use current tipset during refund (https://github.com/filecoin-project/lotus/pull/3570)
+- Lotus-shed: Add math command (https://github.com/filecoin-project/lotus/pull/3568)
+- PCR: Add tipset aggergation (https://github.com/filecoin-project/lotus/pull/3565)- Fix broken paych tests (https://github.com/filecoin-project/lotus/pull/3551)
+- Make chain export ~1000x times faster (https://github.com/filecoin-project/lotus/pull/3533)
+- Chainwatch: Stop SyncIncomingBlocks from leaking into chainwatch processing; No panics during processing (https://github.com/filecoin-project/lotus/pull/3526)
+- Conformance: various changes (https://github.com/filecoin-project/lotus/pull/3521)
+
+# 0.5.10 / 2020-09-03
+
+This patch includes a crucial fix to the message pool selection logic, strongly disfavouring messages that might cause a miner penalty.
+
+## Changes
+
+- Fix calculation of GasReward in messagepool (https://github.com/filecoin-project/lotus/pull/3528)
+
+# 0.5.9 / 2020-09-03
+
+This patch includes a hotfix to the `GasEstimateFeeCap` method, capping the estimated fee to a reasonable level by default.
+
+## Changes 
+
+- Added target height to sync wait (https://github.com/filecoin-project/lotus/pull/3502)
+- Disable codecov annotations (https://github.com/filecoin-project/lotus/pull/3514)
+- Cap fees to reasonable level by default (https://github.com/filecoin-project/lotus/pull/3516)
+- Add APIs and command to inspect bandwidth usage (https://github.com/filecoin-project/lotus/pull/3497)
+- Track expected nonce in mpool, ignore messages with large nonce gaps (https://github.com/filecoin-project/lotus/pull/3450)
+
+# 0.5.8 / 2020-09-02
+
+This patch includes some bugfixes to the sector sealing process, and updates go-fil-markets. It also improves the performance of blocksync, adds a method to export chain state trees, and improves chainwatch.
+
+## Changes
+
+- Upgrade markets to v0.5.9 (https://github.com/filecoin-project/lotus/pull/3496)
+- Improve blocksync to load fewer messages: (https://github.com/filecoin-project/lotus/pull/3494)
+- Fix a panic in the ffi-wrapper's `ReadPiece` (https://github.com/filecoin-project/lotus/pull/3492/files)
+- Fix a deadlock in the sealing scheduler (https://github.com/filecoin-project/lotus/pull/3489)
+- Add test vectors for tipset tests (https://github.com/filecoin-project/lotus/pull/3485/files)
+- Improve the advance-block debug command (https://github.com/filecoin-project/lotus/pull/3476)
+- Add toggle for message processing to Lotus PCR (https://github.com/filecoin-project/lotus/pull/3470)
+- Allow exporting recent chain state trees (https://github.com/filecoin-project/lotus/pull/3463)
+- Remove height from chain rand (https://github.com/filecoin-project/lotus/pull/3458)
+- Disable GC on chain badger datastore (https://github.com/filecoin-project/lotus/pull/3457)
+- Account for `GasPremium` in `GasEstimateFeeCap` (https://github.com/filecoin-project/lotus/pull/3456)
+- Update go-libp2p-pubsub to `master` (https://github.com/filecoin-project/lotus/pull/3455)
+- Chainwatch improvements (https://github.com/filecoin-project/lotus/pull/3442)
+
+# 0.5.7 / 2020-08-31
+
+This patch release includes some bugfixes and enhancements to the sector lifecycle and message pool logic. 
+
+## Changes
+
+- Rebuild unsealed infos on miner restart (https://github.com/filecoin-project/lotus/pull/3401)
+- CLI to attach storage paths to workers (https://github.com/filecoin-project/lotus/pull/3405)
+- Do not select negative performing message chains for inclusion (https://github.com/filecoin-project/lotus/pull/3392)
+- Remove a redundant error-check (https://github.com/filecoin-project/lotus/pull/3421)
+- Correctly move unsealed sectors in `FinalizeSectors` (https://github.com/filecoin-project/lotus/pull/3424)
+- Improve worker selection logic (https://github.com/filecoin-project/lotus/pull/3425)
+- Don't use context to close bitswap (https://github.com/filecoin-project/lotus/pull/3430)
+- Correctly estimate gas premium when there is only one message on chain (https://github.com/filecoin-project/lotus/pull/3428)
+
+# 0.5.6 / 2020-08-29
+
+Hotfix release that fixes a panic in the sealing scheduler (https://github.com/filecoin-project/lotus/pull/3389).
+
+# 0.5.5
+
+This patch release introduces a large number of improvements to the sealing process.
+It also updates go-fil-markets to 
+[version 0.5.8](https://github.com/filecoin-project/go-fil-markets/releases/tag/v0.5.8),
+and go-libp2p-pubsub to [v0.3.5](https://github.com/libp2p/go-libp2p-pubsub/releases/tag/v0.3.5).
+
+#### Downstream upgrades
+
+- Upgrades markets to v0.5.8 (https://github.com/filecoin-project/lotus/pull/3384)
+- Upgrades go-libp2p-pubsub to v0.3.5 (https://github.com/filecoin-project/lotus/pull/3305)
+
+#### Sector sealing
+
+- The following improvements were introduced in https://github.com/filecoin-project/lotus/pull/3350.
+
+    - Allow `lotus-miner sectors remove` to remove a sector in any state.
+    - Create a separate state in the storage FSM dedicated to submitting the Commit message.
+    - Recovery for when the Deal IDs of deals in a sector get changed in a reorg.
+    - Auto-retry sending Precommit and Commit messages if they run out of gas
+    - Auto-retry sector remove tasks when they fail
+    - Compact worker windows, and allow their tasks to be executed in any order
+
+- Don't simply skip PoSt for bad sectors (https://github.com/filecoin-project/lotus/pull/3323)
+
+#### Message Pool 
+
+- Spam Protection: Track required funds for pending messages (https://github.com/filecoin-project/lotus/pull/3313)
+
+#### Chainwatch
+
+- Add more power and reward metrics (https://github.com/filecoin-project/lotus/pull/3367)
+- Fix raciness in sector deal table (https://github.com/filecoin-project/lotus/pull/3275)
+- Parallelize miner processing (https://github.com/filecoin-project/lotus/pull/3380)
+- Accept Lotus API and token (https://github.com/filecoin-project/lotus/pull/3337)
+
+# 0.5.4
+
+A patch release, containing a few nice bugfixes and improvements:
+
+- Fix parsing of peer ID in `lotus-miner actor set-peer-id` (@whyrusleeping)
+- Update dependencies, fixing several bugs (@Stebalien)
+- Fix remaining linter warnings (@Stebalien)
+- Use safe string truncation (@Ingar)
+- Allow tweaking of blocksync message window size (@whyrusleeping)
+- Add some additional gas stats to metrics (@Kubuxu)
+- Fix an edge case bug in message selection, add many tests (@vyzo)
+
+# 0.5.3
+
+Yet another hotfix release. 
+A lesson for readers, having people who have been awake for 12+ hours review
+your hotfix PR is not a good idea. Find someone who has enough slept recently
+enough to give you good code review, otherwise you'll end up quickly bumping
+versions again.
+
+- Fixed a bug in the mempool that was introduced in v0.5.2
+
+# 0.5.2 / 2020-08-24
+
+This is a hotfix release.
+
+- Fix message selection to not include messages that are invalid for block
+  inclusion.
+- Improve SelectMessage handling of the case where the message pools tipset
+  differs from our mining base.
+
+# 0.5.1 / 2020-08-24
+
+The Space Race release! 
+This release contains the genesis car file and bootstrap peers for the space
+race network. 
+
+Additionally, we included two small fixes to genesis creation:
+- Randomize ticket value in genesis generation
+- Correctly set t099 (burnt funds actor) to have valid account actor state
+
+# 0.5.0 / 2020-08-20
+
+This version of Lotus will be used for the incentivized testnet Space Race competition,
+and can be considered mainnet-ready code. It includes some protocol
+changes, upgrades of core dependencies, and various bugfixes and UX/performance improvements.
+
+## Highlights
+
+Among the highlights included in this release are:
+
+- Gas changes: We implemented EIP-1559 and introduced real gas values.
+- Deal-making: We now support "Committed Capacity" sectors, "fast-retrieval" deals,
+and the packing of multiple deals into a single sector.
+- Renamed features: We renamed some of the binaries, environment variables, and default
+paths associated with a Lotus node.
+
+### Gas changes
+
+We made some significant changes to the mechanics of gas in this release.
+
+#### Network fee
+
+We implemented something similar to 
+[Ethereum's EIP-1559](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-1559.md).
+The `Message` structure had three changes:
+- The `GasPrice` field has been removed
+- A new `GasFeeCap` field has been added, which controls the maximum cost
+the sender incurs for the message
+- A new `GasPremium` field has been added, which controls the reward a miner
+earns for including the message
+
+A sender will never be charged more than `GasFeeCap * GasLimit`. 
+A miner will typically earn `GasPremium * GasLimit` as a reward.
+
+The `Blockheader` structure has one new field, called `ParentBaseFee`. 
+Informally speaking,the `ParentBaseFee`
+is increased when blocks are densely packed with messages, and decreased otherwise.
+
+The `ParentBaseFee` is used when calculating how much a sender burns when executing a message. _Burning_ simply refers to sending attoFIL to a dedicated, unreachable account.
+A message causes `ParentBaseFee * GasUsed` attoFIL to be burnt.
+
+#### Real gas values
+
+This release also includes our first "real" gas costs for primitive operations.
+The costs were designed to account for both the _time_ that message execution takes,
+as well as the _space_ a message adds to the state tree.
+
+## Deal-making changes
+
+There are three key changes to the deal-making process.
+
+#### Committed Capacity sectors
+
+Miners can now pledge "Committed Capacity" (CC) sectors, which are explicitly
+stated as containing junk data, and must not include any deals. Miners can do this
+to increase their storage power, and win block rewards from this pledged storage.
+
+They can mark these sectors as "upgradable" with `lotus-miner sectors mark-for-upgrade`.
+If the miner receives and accepts one or more storage deals, the sector that includes
+those deals will _replace_ the CC sector. This is intended to maximize the amount of useful
+storage on the Filecoin network.
+
+#### Fast-retrieval deals
+
+Clients can now include a `fast-retrieval` flag when proposing deals with storage miners.
+If set to true, the miner will include an extra copy of the deal data. This
+data can be quickly served in a retrieval deal, since it will not need to be unsealed.
+
+#### Multiple deals per sector
+
+Miners can now pack multiple deals into a single sector, so long as all the deals
+fit into the sector capacity. This should increase the packing efficiency of miners.
+
+### Renamed features
+
+To improve the user experience, we updated several names to mainatin
+standard prefixing, and to better reflect the meaning of the features being referenced.
+
+In particular, the Lotus miner binary is now called `lotus-miner`, the default
+path for miner data is now `~/.lotusminer`, and the environment variable
+that sets the path for miner data is now `$LOTUS_MINER_PATH`. A full list of renamed
+features can be found [here](https://github.com/filecoin-project/lotus/issues/2304).
+
+## Changelog
+
+#### Downstream upgrades
+- Upgrades markets to v0.5.6 (https://github.com/filecoin-project/lotus/pull/3058)
+- Upgrades specs-actors to v0.9.3 (https://github.com/filecoin-project/lotus/pull/3151)
+
+#### Core protocol
+- Introduces gas values, replacing placeholders (https://github.com/filecoin-project/lotus/pull/2343)
+- Implements EIP-1559, introducing a network base fee, message gas fee cap, and message gas fee premium (https://github.com/filecoin-project/lotus/pull/2874)
+- Implements Poisson Sortition for elections (https://github.com/filecoin-project/lotus/pull/2084)
+
+#### Deal-making lifecycle
+- Introduces "Committed Capacity" sectors (https://github.com/filecoin-project/lotus/pull/2220)
+- Introduces "fast-retrieval" flag for deals (https://github.com/filecoin-project/lotus/pull/2323
+- Supports packing multiple deals into one sector (https://github.com/filecoin-project/storage-fsm/pull/38)
+
+#### Enhancements
+
+- Optimized message pool selection logic (https://github.com/filecoin-project/lotus/pull/2838)
+- Window-based scheduling of sealing tasks (https://github.com/filecoin-project/sector-storage/pull/67)
+- Faster window PoSt (https://github.com/filecoin-project/lotus/pull/2209/files)
+- Refactors the payment channel manager (https://github.com/filecoin-project/lotus/pull/2640)
+- Refactors blocksync (https://github.com/filecoin-project/lotus/pull/2715/files)
+
+#### UX
+
+- Provide status updates for data-transfer (https://github.com/filecoin-project/lotus/pull/3162, https://github.com/filecoin-project/lotus/pull/3191)
+- Miners can customise asks (https://github.com/filecoin-project/lotus/pull/2046)
+- Miners can toggle auto-acceptance of deals (https://github.com/filecoin-project/lotus/pull/1994)
+- Miners can maintain a blocklist of piece CIDs (https://github.com/filecoin-project/lotus/pull/2069)
+
+## Contributors
+
+The following contributors had 10 or more commits go into this release.
+We are grateful for every contribution!
+
+| Contributor        | Commits | Lines ±       |
+|--------------------|---------|---------------|
+| magik6k            | 361     | +13197/-6136  |
+| Kubuxu             | 227     | +5670/-2587   |
+| arajasek           | 120     | +2916/-1264   |
+| whyrusleeping      | 112     | +3979/-1089   |
+| vyzo               | 99      | +3343/-1305   |
+| dirkmc             | 68      | +8732/-3621   |
+| laser              | 45      | +1489/-501    |
+| hannahhoward       | 43      | +2654/-990    |
+| frrist             | 37      | +6630/-4338   |
+| schomatis          | 28      | +3016/-1368   |
+| placer14           | 27      | +824/-350     |
+| raulk              | 25      | +28718/-29849 |
+| mrsmkl             | 22      | +560/-368     |
+| travisperson       | 18      | +1354/-314    |
+| nonsense           | 16      | +2956/-2842   |
+| ingar              | 13      | +331/-123     |
+| daviddias          | 11      | +311/-11      |
+| Stebalien          | 11      | +1204/-980    |
+| RobQuistNL         | 10      | +69/-74       |
+
+# 0.1.0 / 2019-12-11
+
+We are very excited to release **lotus** 0.1.0. This is our testnet release. To install lotus and join the testnet, please visit [lotu.sh](lotu.sh). Please file bug reports as [issues](https://github.com/filecoin-project/lotus/issues).
+
+A huge thank you to all contributors for this testnet release!

Makefile

@@ -148,7 +148,7 @@ BINS+=lotus-fountain
 
 lotus-chainwatch:
 	rm -f lotus-chainwatch
-	go build -o lotus-chainwatch ./cmd/lotus-chainwatch
+	go build $(GOFLAGS) -o lotus-chainwatch ./cmd/lotus-chainwatch
 .PHONY: lotus-chainwatch
 BINS+=lotus-chainwatch
 
@@ -272,11 +272,15 @@ dist-clean:
 
 type-gen:
 	go run ./gen/main.go
+	go generate ./...
 
 method-gen:
 	(cd ./lotuspond/front/src/chain && go run ./methodgen.go)
 
 gen: type-gen method-gen
 
+docsgen:
+	go run ./api/docgen > documentation/en/api-methods.md
+
 print-%:
 	@echo $*=$($*)

README.md

@@ -1,16 +1,24 @@
 <p align="center">
-  <a href="https://docs.lotu.sh/" title="Lotus Docs">
+  <a href="https://lotu.sh/" title="Lotus Docs">
     <img src="documentation/images/lotus_logo_h.png" alt="Project Lotus Logo" width="244" />
   </a>
 </p>
 
 <h1 align="center">Project Lotus - 莲</h1>
 
+<p align="center">
+  <a href="https://circleci.com/gh/filecoin-project/lotus"><img src="https://circleci.com/gh/filecoin-project/lotus.svg?style=svg"></a>
+  <a href="https://codecov.io/gh/filecoin-project/lotus"><img src="https://codecov.io/gh/filecoin-project/lotus/branch/master/graph/badge.svg"></a>
+  <a href="https://goreportcard.com/report/github.com/filecoin-project/lotus"><img src="https://goreportcard.com/badge/github.com/filecoin-project/lotus" /></a>  
+  <a href=""><img src="https://img.shields.io/badge/golang-%3E%3D1.14.7-blue.svg" /></a>
+  <br>
+</p>
+
 Lotus is an implementation of the Filecoin Distributed Storage Network. For more details about Filecoin, check out the [Filecoin Spec](https://spec.filecoin.io).
 
 ## Building & Documentation
 
-For instructions on how to build lotus from source, please visit [https://docs.lotu.sh](https://docs.lotu.sh) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation).
+For instructions on how to build lotus from source, please visit [https://lotu.sh](https://lotu.sh) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation).
 
 ## Reporting a Vulnerability
 
@@ -32,9 +40,6 @@ All work is tracked via issues. An attempt at keeping an up-to-date view on rema
 The lotus Filecoin implementation unfolds into the following packages:
 
 - [This repo](https://github.com/filecoin-project/lotus)
-- [storage-fsm](https://github.com/filecoin-project/storage-fsm)
-- [sector-storage](https://github.com/filecoin-project/sector-storage)
-- [specs-storage](https://github.com/filecoin-project/specs-storage)
 - [go-fil-markets](https://github.com/filecoin-project/go-fil-markets) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/markets-shared-components-5daa144a7046a60001c6e253/board)
 - [spec-actors](https://github.com/filecoin-project/specs-actors) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/actors-5ee6f3aa87591f0016c05685/board)
 

api/api_common.go

@@ -4,11 +4,11 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/filecoin-project/go-jsonrpc/auth"
+	metrics "github.com/libp2p/go-libp2p-core/metrics"
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
-	ma "github.com/multiformats/go-multiaddr"
-
-	"github.com/filecoin-project/go-jsonrpc/auth"
+	protocol "github.com/libp2p/go-libp2p-core/protocol"
 
 	"github.com/filecoin-project/lotus/build"
 )
@@ -30,6 +30,19 @@ type Common interface {
 	NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error)
 	NetPubsubScores(context.Context) ([]PubsubScore, error)
 	NetAutoNatStatus(context.Context) (NatInfo, error)
+	NetAgentVersion(ctx context.Context, p peer.ID) (string, error)
+
+	// NetBandwidthStats returns statistics about the nodes total bandwidth
+	// usage and current rate across all peers and protocols.
+	NetBandwidthStats(ctx context.Context) (metrics.Stats, error)
+
+	// NetBandwidthStatsByPeer returns statistics about the nodes bandwidth
+	// usage and current rate per peer
+	NetBandwidthStatsByPeer(ctx context.Context) (map[string]metrics.Stats, error)
+
+	// NetBandwidthStatsByProtocol returns statistics about the nodes bandwidth
+	// usage and current rate per protocol
+	NetBandwidthStatsByProtocol(ctx context.Context) (map[protocol.ID]metrics.Stats, error)
 
 	// MethodGroup: Common
 
@@ -70,5 +83,5 @@ func (v Version) String() string {
 
 type NatInfo struct {
 	Reachability network.Reachability
-	PublicAddr   ma.Multiaddr
+	PublicAddr   string
 }

api/api_full.go

@@ -2,23 +2,29 @@ package api
 
 import (
 	"context"
+	"fmt"
 	"time"
 
+	"github.com/filecoin-project/go-state-types/dline"
+
+	"github.com/filecoin-project/specs-actors/actors/runtime/proof"
+
 	"github.com/ipfs/go-cid"
 	"github.com/libp2p/go-libp2p-core/peer"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-bitfield"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-multistore"
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/specs-actors/actors/builtin/market"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-actors/actors/builtin/paych"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
-	"github.com/filecoin-project/specs-actors/actors/crypto"
 
 	"github.com/filecoin-project/lotus/chain/types"
 	marketevents "github.com/filecoin-project/lotus/markets/loggers"
@@ -109,7 +115,10 @@ type FullNode interface {
 	ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*HeadChange, error)
 
 	// ChainExport returns a stream of bytes with CAR dump of chain data.
-	ChainExport(context.Context, types.TipSetKey) (<-chan []byte, error)
+	// The exported chain data includes the header chain from the given tipset
+	// back to genesis, the entire genesis state, and the most recent 'nroots'
+	// state trees.
+	ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk types.TipSetKey) (<-chan []byte, error)
 
 	// MethodGroup: Beacon
 	// The Beacon method group contains methods for interacting with the random beacon (DRAND)
@@ -132,6 +141,9 @@ type FullNode interface {
 	GasEstimateGasPremium(_ context.Context, nblocksincl uint64,
 		sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error)
 
+	// GasEstimateMessageGas estimates gas values for unset message gas fields
+	GasEstimateMessageGas(context.Context, *types.Message, *MessageSendSpec, types.TipSetKey) (*types.Message, error)
+
 	// MethodGroup: Sync
 	// The Sync method group contains methods for interacting with and
 	// observing the lotus sync service.
@@ -147,10 +159,16 @@ type FullNode interface {
 	// yet synced block headers.
 	SyncIncomingBlocks(ctx context.Context) (<-chan *types.BlockHeader, error)
 
+	// SyncCheckpoint marks a blocks as checkpointed, meaning that it won't ever fork away from it.
+	SyncCheckpoint(ctx context.Context, tsk types.TipSetKey) error
+
 	// SyncMarkBad marks a blocks as bad, meaning that it won't ever by synced.
 	// Use with extreme caution.
 	SyncMarkBad(ctx context.Context, bcid cid.Cid) error
 
+	// SyncUnmarkBad unmarks a blocks as bad, making it possible to be validated and synced again.
+	SyncUnmarkBad(ctx context.Context, bcid cid.Cid) error
+
 	// SyncCheckBad checks if a block was marked as bad, and if it was, returns
 	// the reason.
 	SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error)
@@ -181,6 +199,9 @@ type FullNode interface {
 	MpoolGetNonce(context.Context, address.Address) (uint64, error)
 	MpoolSub(context.Context) (<-chan MpoolUpdate, error)
 
+	// MpoolClear clears pending messages from the mpool
+	MpoolClear(context.Context, bool) error
+
 	// MpoolGetConfig returns (a copy of) the current mpool config
 	MpoolGetConfig(context.Context) (*types.MpoolConfig, error)
 	// MpoolSetConfig sets the mpool config to (a copy of) the supplied config
@@ -237,6 +258,8 @@ type FullNode interface {
 	ClientGetDealInfo(context.Context, cid.Cid) (*DealInfo, error)
 	// ClientListDeals returns information about the deals made by the local client.
 	ClientListDeals(ctx context.Context) ([]DealInfo, error)
+	// ClientGetDealUpdates returns the status of updated deals
+	ClientGetDealUpdates(ctx context.Context) (<-chan DealInfo, error)
 	// ClientHasLocal indicates whether a certain CID is locally stored.
 	ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error)
 	// ClientFindData identifies peers that have a certain file, and returns QueryOffers (one per peer).
@@ -244,7 +267,10 @@ type FullNode interface {
 	// ClientMinerQueryOffer returns a QueryOffer for the specific miner and file.
 	ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (QueryOffer, error)
 	// ClientRetrieve initiates the retrieval of a file, as specified in the order.
-	ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *FileRef) (<-chan marketevents.RetrievalEvent, error)
+	ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *FileRef) error
+	// ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel
+	// of status updates.
+	ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *FileRef) (<-chan marketevents.RetrievalEvent, error)
 	// ClientQueryAsk returns a signed StorageAsk from the specified miner.
 	ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error)
 	// ClientCalcCommP calculates the CommP for a specified file
@@ -253,6 +279,12 @@ type FullNode interface {
 	ClientGenCar(ctx context.Context, ref FileRef, outpath string) error
 	// ClientDealSize calculates real deal data size
 	ClientDealSize(ctx context.Context, root cid.Cid) (DataSize, error)
+	// ClientListTransfers returns the status of all ongoing transfers of data
+	ClientListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)
+	ClientDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)
+	// ClientRetrieveTryRestartInsufficientFunds attempts to restart stalled retrievals on a given payment channel
+	// which are stuck due to insufficient funds
+	ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error
 
 	// ClientUnimport removes references to the specified file from filestore
 	//ClientUnimport(path string)
@@ -283,12 +315,12 @@ type FullNode interface {
 	// StateMinerSectors returns info about the given miner's sectors. If the filter bitfield is nil, all sectors are included.
 	// If the filterOut boolean is set to true, any sectors in the filter are excluded.
 	// If false, only those sectors in the filter are included.
-	StateMinerSectors(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*ChainSectorInfo, error)
+	StateMinerSectors(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*ChainSectorInfo, error)
 	// StateMinerActiveSectors returns info about sectors that a given miner is actively proving.
 	StateMinerActiveSectors(context.Context, address.Address, types.TipSetKey) ([]*ChainSectorInfo, error)
 	// StateMinerProvingDeadline calculates the deadline at some epoch for a proving period
 	// and returns the deadline-related calculations.
-	StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error)
+	StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error)
 	// StateMinerPower returns the power of the indicated miner
 	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error)
 	// StateMinerInfo returns info about the indicated miner
@@ -298,11 +330,11 @@ type FullNode interface {
 	// StateMinerPartitions loads miner partitions for the specified miner/deadline
 	StateMinerPartitions(context.Context, address.Address, uint64, types.TipSetKey) ([]*miner.Partition, error)
 	// StateMinerFaults returns a bitfield indicating the faulty sectors of the given miner
-	StateMinerFaults(context.Context, address.Address, types.TipSetKey) (abi.BitField, error)
+	StateMinerFaults(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error)
 	// StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset
 	StateAllMinerFaults(ctx context.Context, lookback abi.ChainEpoch, ts types.TipSetKey) ([]*Fault, error)
 	// StateMinerRecoveries returns a bitfield indicating the recovering sectors of the given miner
-	StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (abi.BitField, error)
+	StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error)
 	// StateMinerInitialPledgeCollateral returns the precommit deposit for the specified miner's sector
 	StateMinerPreCommitDepositForPower(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error)
 	// StateMinerInitialPledgeCollateral returns the initial pledge collateral for the specified miner's sector
@@ -311,7 +343,7 @@ type FullNode interface {
 	StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)
 	// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector
 	StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error)
-	// StateSectorGetInfo returns the on-chain info for the specified miner's sector
+	// StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found
 	// NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate
 	// expiration epoch
 	StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error)
@@ -359,7 +391,7 @@ type FullNode interface {
 	StateDealProviderCollateralBounds(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (DealCollateralBounds, error)
 
 	// StateCirculatingSupply returns the circulating supply of Filecoin at the given tipset
-	StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error)
+	StateCirculatingSupply(context.Context, types.TipSetKey) (CirculatingSupply, error)
 
 	// MethodGroup: Msig
 	// The Msig methods are used to interact with multisig wallets on the
@@ -367,6 +399,9 @@ type FullNode interface {
 
 	// MsigGetAvailableBalance returns the portion of a multisig's balance that can be withdrawn or spent
 	MsigGetAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)
+	// MsigGetVested returns the amount of FIL that vested in a multisig in a certain period.
+	// It takes the following params: <multisig address>, <start epoch>, <end epoch>
+	MsigGetVested(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error)
 	// MsigCreate creates a multisig wallet
 	// It takes the following params: <required number of senders>, <approving addresses>, <unlock duration>
 	//<initial balance>, <sender address of the create msg>, <gas price>
@@ -383,17 +418,29 @@ type FullNode interface {
 	// It takes the following params: <multisig address>, <proposed message ID>, <recipient address>, <value to transfer>,
 	// <sender address of the cancel msg>, <method to call in the proposed message>, <params to include in the proposed message>
 	MsigCancel(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error)
+	// MsigAddPropose proposes adding a signer in the multisig
+	// It takes the following params: <multisig address>, <sender address of the propose msg>,
+	// <new signer>, <whether the number of required signers should be increased>
+	MsigAddPropose(context.Context, address.Address, address.Address, address.Address, bool) (cid.Cid, error)
+	// MsigAddApprove approves a previously proposed AddSigner message
+	// It takes the following params: <multisig address>, <sender address of the approve msg>, <proposed message ID>,
+	// <proposer address>, <new signer>, <whether the number of required signers should be increased>
+	MsigAddApprove(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, bool) (cid.Cid, error)
+	// MsigAddCancel cancels a previously proposed AddSigner message
+	// It takes the following params: <multisig address>, <sender address of the cancel msg>, <proposed message ID>,
+	// <new signer>, <whether the number of required signers should be increased>
+	MsigAddCancel(context.Context, address.Address, address.Address, uint64, address.Address, bool) (cid.Cid, error)
 	// MsigSwapPropose proposes swapping 2 signers in the multisig
 	// It takes the following params: <multisig address>, <sender address of the propose msg>,
-	// <old signer> <new signer>
+	// <old signer>, <new signer>
 	MsigSwapPropose(context.Context, address.Address, address.Address, address.Address, address.Address) (cid.Cid, error)
 	// MsigSwapApprove approves a previously proposed SwapSigner
 	// It takes the following params: <multisig address>, <sender address of the approve msg>, <proposed message ID>,
-	// <proposer address>, <old signer> <new signer>
+	// <proposer address>, <old signer>, <new signer>
 	MsigSwapApprove(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, address.Address) (cid.Cid, error)
 	// MsigSwapCancel cancels a previously proposed SwapSigner message
 	// It takes the following params: <multisig address>, <sender address of the cancel msg>, <proposed message ID>,
-	// <old signer> <new signer>
+	// <old signer>, <new signer>
 	MsigSwapCancel(context.Context, address.Address, address.Address, uint64, address.Address, address.Address) (cid.Cid, error)
 
 	MarketEnsureAvailable(context.Context, address.Address, address.Address, types.BigInt) (cid.Cid, error)
@@ -404,6 +451,8 @@ type FullNode interface {
 
 	PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*ChannelInfo, error)
 	PaychGetWaitReady(context.Context, cid.Cid) (address.Address, error)
+	PaychAvailableFunds(ch address.Address) (*ChannelAvailableFunds, error)
+	PaychAvailableFundsByFromTo(from, to address.Address) (*ChannelAvailableFunds, error)
 	PaychList(context.Context) ([]address.Address, error)
 	PaychStatus(context.Context, address.Address) (*PaychStatus, error)
 	PaychSettle(context.Context, address.Address) (cid.Cid, error)
@@ -412,10 +461,10 @@ type FullNode interface {
 	PaychNewPayment(ctx context.Context, from, to address.Address, vouchers []VoucherSpec) (*PaymentInfo, error)
 	PaychVoucherCheckValid(context.Context, address.Address, *paych.SignedVoucher) error
 	PaychVoucherCheckSpendable(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (bool, error)
-	PaychVoucherCreate(context.Context, address.Address, types.BigInt, uint64) (*paych.SignedVoucher, error)
+	PaychVoucherCreate(context.Context, address.Address, types.BigInt, uint64) (*VoucherCreateResult, error)
 	PaychVoucherAdd(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error)
 	PaychVoucherList(context.Context, address.Address) ([]*paych.SignedVoucher, error)
-	PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher) (cid.Cid, error)
+	PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error)
 }
 
 type FileRef struct {
@@ -469,6 +518,8 @@ type DealInfo struct {
 	Duration      uint64
 
 	DealID abi.DealID
+
+	CreationTime time.Time
 }
 
 type MsgLookup struct {
@@ -519,6 +570,28 @@ type ChannelInfo struct {
 	WaitSentinel cid.Cid
 }
 
+type ChannelAvailableFunds struct {
+	// Channel is the address of the channel
+	Channel *address.Address
+	// From is the from address of the channel (channel creator)
+	From address.Address
+	// To is the to address of the channel
+	To address.Address
+	// ConfirmedAmt is the amount of funds that have been confirmed on-chain
+	// for the channel
+	ConfirmedAmt types.BigInt
+	// PendingAmt is the amount of funds that are pending confirmation on-chain
+	PendingAmt types.BigInt
+	// PendingWaitSentinel can be used with PaychGetWaitReady to wait for
+	// confirmation of pending funds
+	PendingWaitSentinel *cid.Cid
+	// QueuedAmt is the amount that is queued up behind a pending request
+	QueuedAmt types.BigInt
+	// VoucherRedeemedAmt is the amount that is redeemed by vouchers on-chain
+	// and in the local datastore
+	VoucherReedeemedAmt types.BigInt
+}
+
 type PaymentInfo struct {
 	Channel      address.Address
 	WaitSentinel cid.Cid
@@ -534,6 +607,16 @@ type VoucherSpec struct {
 	Extra *paych.ModVerifyParams
 }
 
+// VoucherCreateResult is the response to calling PaychVoucherCreate
+type VoucherCreateResult struct {
+	// Voucher that was created, or nil if there was an error or if there
+	// were insufficient funds in the channel
+	Voucher *paych.SignedVoucher
+	// Shortfall is the additional amount that would be needed in the channel
+	// in order to be able to create the voucher
+	Shortfall types.BigInt
+}
+
 type MinerPower struct {
 	MinerPower power.Claim
 	TotalPower power.Claim
@@ -609,14 +692,15 @@ type MethodCall struct {
 }
 
 type StartDealParams struct {
-	Data              *storagemarket.DataRef
-	Wallet            address.Address
-	Miner             address.Address
-	EpochPrice        types.BigInt
-	MinBlocksDuration uint64
-	DealStartEpoch    abi.ChainEpoch
-	FastRetrieval     bool
-	VerifiedDeal      bool
+	Data               *storagemarket.DataRef
+	Wallet             address.Address
+	Miner              address.Address
+	EpochPrice         types.BigInt
+	MinBlocksDuration  uint64
+	ProviderCollateral big.Int
+	DealStartEpoch     abi.ChainEpoch
+	FastRetrieval      bool
+	VerifiedDeal       bool
 }
 
 type IpldObject struct {
@@ -649,8 +733,28 @@ const (
 	StageMessages
 	StageSyncComplete
 	StageSyncErrored
+	StageFetchingMessages
 )
 
+func (v SyncStateStage) String() string {
+	switch v {
+	case StageHeaders:
+		return "header sync"
+	case StagePersistHeaders:
+		return "persisting headers"
+	case StageMessages:
+		return "message sync"
+	case StageSyncComplete:
+		return "complete"
+	case StageSyncErrored:
+		return "error"
+	case StageFetchingMessages:
+		return "fetching messages"
+	default:
+		return fmt.Sprintf("<unknown: %d>", v)
+	}
+}
+
 type MpoolChange int
 
 const (
@@ -673,10 +777,18 @@ type DealCollateralBounds struct {
 	Max abi.TokenAmount
 }
 
+type CirculatingSupply struct {
+	FilVested      abi.TokenAmount
+	FilMined       abi.TokenAmount
+	FilBurnt       abi.TokenAmount
+	FilLocked      abi.TokenAmount
+	FilCirculating abi.TokenAmount
+}
+
 type MiningBaseInfo struct {
 	MinerPower      types.BigInt
 	NetworkPower    types.BigInt
-	Sectors         []abi.SectorInfo
+	Sectors         []proof.SectorInfo
 	WorkerKey       address.Address
 	SectorSize      abi.SectorSize
 	PrevBeaconEntry types.BeaconEntry
@@ -693,7 +805,7 @@ type BlockTemplate struct {
 	Messages         []*types.SignedMessage
 	Epoch            abi.ChainEpoch
 	Timestamp        uint64
-	WinningPoStProof []abi.PoStProof
+	WinningPoStProof []proof.PoStProof
 }
 
 type DataSize struct {

api/api_storage.go

@@ -11,11 +11,11 @@ import (
 	"github.com/filecoin-project/go-fil-markets/piecestore"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/sector-storage/fsutil"
-	"github.com/filecoin-project/sector-storage/stores"
-	"github.com/filecoin-project/sector-storage/storiface"
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
+	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
+	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 )
 
 // StorageMiner is a low-level interface to the Filecoin network storage miner node
@@ -73,12 +73,14 @@ type StorageMiner interface {
 	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error
 	MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error)
 	MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error)
-	MarketGetDealUpdates(ctx context.Context, d cid.Cid) (<-chan storagemarket.MinerDeal, error)
+	MarketGetDealUpdates(ctx context.Context) (<-chan storagemarket.MinerDeal, error)
 	MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error)
 	MarketSetAsk(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error
 	MarketGetAsk(ctx context.Context) (*storagemarket.SignedStorageAsk, error)
 	MarketSetRetrievalAsk(ctx context.Context, rask *retrievalmarket.Ask) error
 	MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error)
+	MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)
+	MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)
 
 	DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error
 	DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error)
@@ -118,15 +120,18 @@ type SectorLog struct {
 }
 
 type SectorInfo struct {
-	SectorID abi.SectorNumber
-	State    SectorState
-	CommD    *cid.Cid
-	CommR    *cid.Cid
-	Proof    []byte
-	Deals    []abi.DealID
-	Ticket   SealTicket
-	Seed     SealSeed
-	Retries  uint64
+	SectorID     abi.SectorNumber
+	State        SectorState
+	CommD        *cid.Cid
+	CommR        *cid.Cid
+	Proof        []byte
+	Deals        []abi.DealID
+	Ticket       SealTicket
+	Seed         SealSeed
+	PreCommitMsg *cid.Cid
+	CommitMsg    *cid.Cid
+	Retries      uint64
+	ToUpgrade    bool
 
 	LastErr string
 

api/api_worker.go

@@ -6,10 +6,10 @@ import (
 
 	"github.com/ipfs/go-cid"
 
-	"github.com/filecoin-project/sector-storage/sealtasks"
-	"github.com/filecoin-project/sector-storage/stores"
-	"github.com/filecoin-project/sector-storage/storiface"
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
+	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
+	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 	"github.com/filecoin-project/specs-storage/storage"
 
 	"github.com/filecoin-project/lotus/build"
@@ -23,13 +23,17 @@ type WorkerAPI interface {
 	Paths(context.Context) ([]stores.StoragePath, error)
 	Info(context.Context) (storiface.WorkerInfo, error)
 
+	AddPiece(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error)
+
 	storage.Sealer
 
-	MoveStorage(ctx context.Context, sector abi.SectorID) error
+	MoveStorage(ctx context.Context, sector abi.SectorID, types stores.SectorFileType) error
 
 	UnsealPiece(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error
 	ReadPiece(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) (bool, error)
 
+	StorageAddLocal(ctx context.Context, path string) error
+
 	Fetch(context.Context, abi.SectorID, stores.SectorFileType, stores.PathType, stores.AcquireMode) error
 
 	Closing(context.Context) (<-chan struct{}, error)

api/apistruct/struct.go

@@ -5,27 +5,32 @@ import (
 	"io"
 	"time"
 
+	"github.com/filecoin-project/go-state-types/dline"
+
 	"github.com/ipfs/go-cid"
+	metrics "github.com/libp2p/go-libp2p-core/metrics"
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
+	protocol "github.com/libp2p/go-libp2p-core/protocol"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-bitfield"
 	"github.com/filecoin-project/go-fil-markets/piecestore"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-jsonrpc/auth"
 	"github.com/filecoin-project/go-multistore"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
+	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
+	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
+	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 	marketevents "github.com/filecoin-project/lotus/markets/loggers"
-	"github.com/filecoin-project/sector-storage/fsutil"
-	"github.com/filecoin-project/sector-storage/sealtasks"
-	"github.com/filecoin-project/sector-storage/stores"
-	"github.com/filecoin-project/sector-storage/storiface"
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-actors/actors/builtin/paych"
 	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
-	"github.com/filecoin-project/specs-actors/actors/crypto"
 	"github.com/filecoin-project/specs-storage/storage"
 
 	"github.com/filecoin-project/lotus/api"
@@ -42,14 +47,18 @@ type CommonStruct struct {
 		AuthVerify func(ctx context.Context, token string) ([]auth.Permission, error) `perm:"read"`
 		AuthNew    func(ctx context.Context, perms []auth.Permission) ([]byte, error) `perm:"admin"`
 
-		NetConnectedness func(context.Context, peer.ID) (network.Connectedness, error) `perm:"read"`
-		NetPeers         func(context.Context) ([]peer.AddrInfo, error)                `perm:"read"`
-		NetConnect       func(context.Context, peer.AddrInfo) error                    `perm:"write"`
-		NetAddrsListen   func(context.Context) (peer.AddrInfo, error)                  `perm:"read"`
-		NetDisconnect    func(context.Context, peer.ID) error                          `perm:"write"`
-		NetFindPeer      func(context.Context, peer.ID) (peer.AddrInfo, error)         `perm:"read"`
-		NetPubsubScores  func(context.Context) ([]api.PubsubScore, error)              `perm:"read"`
-		NetAutoNatStatus func(context.Context) (api.NatInfo, error)                    `perm:"read"`
+		NetConnectedness            func(context.Context, peer.ID) (network.Connectedness, error)    `perm:"read"`
+		NetPeers                    func(context.Context) ([]peer.AddrInfo, error)                   `perm:"read"`
+		NetConnect                  func(context.Context, peer.AddrInfo) error                       `perm:"write"`
+		NetAddrsListen              func(context.Context) (peer.AddrInfo, error)                     `perm:"read"`
+		NetDisconnect               func(context.Context, peer.ID) error                             `perm:"write"`
+		NetFindPeer                 func(context.Context, peer.ID) (peer.AddrInfo, error)            `perm:"read"`
+		NetPubsubScores             func(context.Context) ([]api.PubsubScore, error)                 `perm:"read"`
+		NetAutoNatStatus            func(context.Context) (api.NatInfo, error)                       `perm:"read"`
+		NetBandwidthStats           func(ctx context.Context) (metrics.Stats, error)                 `perm:"read"`
+		NetBandwidthStatsByPeer     func(ctx context.Context) (map[string]metrics.Stats, error)      `perm:"read"`
+		NetBandwidthStatsByProtocol func(ctx context.Context) (map[protocol.ID]metrics.Stats, error) `perm:"read"`
+		NetAgentVersion             func(ctx context.Context, p peer.ID) (string, error)             `perm:"read"`
 
 		ID      func(context.Context) (peer.ID, error)     `perm:"read"`
 		Version func(context.Context) (api.Version, error) `perm:"read"`
@@ -86,18 +95,21 @@ type FullNodeStruct struct {
 		ChainGetNode                  func(ctx context.Context, p string) (*api.IpldObject, error)                                                       `perm:"read"`
 		ChainGetMessage               func(context.Context, cid.Cid) (*types.Message, error)                                                             `perm:"read"`
 		ChainGetPath                  func(context.Context, types.TipSetKey, types.TipSetKey) ([]*api.HeadChange, error)                                 `perm:"read"`
-		ChainExport                   func(context.Context, types.TipSetKey) (<-chan []byte, error)                                                      `perm:"read"`
+		ChainExport                   func(context.Context, abi.ChainEpoch, types.TipSetKey) (<-chan []byte, error)                                      `perm:"read"`
 
 		BeaconGetEntry func(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) `perm:"read"`
 
-		GasEstimateGasPremium func(context.Context, uint64, address.Address, int64, types.TipSetKey) (types.BigInt, error) `perm:"read"`
-		GasEstimateGasLimit   func(context.Context, *types.Message, types.TipSetKey) (int64, error)                        `perm:"read"`
-		GasEstimateFeeCap     func(context.Context, *types.Message, int64, types.TipSetKey) (types.BigInt, error)          `perm:"read"`
+		GasEstimateGasPremium func(context.Context, uint64, address.Address, int64, types.TipSetKey) (types.BigInt, error)         `perm:"read"`
+		GasEstimateGasLimit   func(context.Context, *types.Message, types.TipSetKey) (int64, error)                                `perm:"read"`
+		GasEstimateFeeCap     func(context.Context, *types.Message, int64, types.TipSetKey) (types.BigInt, error)                  `perm:"read"`
+		GasEstimateMessageGas func(context.Context, *types.Message, *api.MessageSendSpec, types.TipSetKey) (*types.Message, error) `perm:"read"`
 
 		SyncState          func(context.Context) (*api.SyncState, error)                `perm:"read"`
 		SyncSubmitBlock    func(ctx context.Context, blk *types.BlockMsg) error         `perm:"write"`
 		SyncIncomingBlocks func(ctx context.Context) (<-chan *types.BlockHeader, error) `perm:"read"`
+		SyncCheckpoint     func(ctx context.Context, key types.TipSetKey) error         `perm:"admin"`
 		SyncMarkBad        func(ctx context.Context, bcid cid.Cid) error                `perm:"admin"`
+		SyncUnmarkBad      func(ctx context.Context, bcid cid.Cid) error                `perm:"admin"`
 		SyncCheckBad       func(ctx context.Context, bcid cid.Cid) (string, error)      `perm:"read"`
 
 		MpoolGetConfig func(context.Context) (*types.MpoolConfig, error) `perm:"read"`
@@ -105,7 +117,9 @@ type FullNodeStruct struct {
 
 		MpoolSelect func(context.Context, types.TipSetKey, float64) ([]*types.SignedMessage, error) `perm:"read"`
 
-		MpoolPending     func(context.Context, types.TipSetKey) ([]*types.SignedMessage, error)                    `perm:"read"`
+		MpoolPending func(context.Context, types.TipSetKey) ([]*types.SignedMessage, error) `perm:"read"`
+		MpoolClear   func(context.Context, bool) error                                      `perm:"write"`
+
 		MpoolPush        func(context.Context, *types.SignedMessage) (cid.Cid, error)                              `perm:"write"`
 		MpoolPushMessage func(context.Context, *types.Message, *api.MessageSendSpec) (*types.SignedMessage, error) `perm:"sign"`
 		MpoolGetNonce    func(context.Context, address.Address) (uint64, error)                                    `perm:"read"`
@@ -127,32 +141,37 @@ type FullNodeStruct struct {
 		WalletImport         func(context.Context, *types.KeyInfo) (address.Address, error)                       `perm:"admin"`
 		WalletDelete         func(context.Context, address.Address) error                                         `perm:"write"`
 
-		ClientImport          func(ctx context.Context, ref api.FileRef) (*api.ImportRes, error)                                                `perm:"admin"`
-		ClientListImports     func(ctx context.Context) ([]api.Import, error)                                                                   `perm:"write"`
-		ClientRemoveImport    func(ctx context.Context, importID multistore.StoreID) error                                                      `perm:"admin"`
-		ClientHasLocal        func(ctx context.Context, root cid.Cid) (bool, error)                                                             `perm:"write"`
-		ClientFindData        func(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]api.QueryOffer, error)                                 `perm:"read"`
-		ClientMinerQueryOffer func(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error)            `perm:"read"`
-		ClientStartDeal       func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error)                                          `perm:"admin"`
-		ClientGetDealInfo     func(context.Context, cid.Cid) (*api.DealInfo, error)                                                             `perm:"read"`
-		ClientListDeals       func(ctx context.Context) ([]api.DealInfo, error)                                                                 `perm:"write"`
-		ClientRetrieve        func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) `perm:"admin"`
-		ClientQueryAsk        func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error)              `perm:"read"`
-		ClientCalcCommP       func(ctx context.Context, inpath string) (*api.CommPRet, error)                                                   `perm:"read"`
-		ClientGenCar          func(ctx context.Context, ref api.FileRef, outpath string) error                                                  `perm:"write"`
-		ClientDealSize        func(ctx context.Context, root cid.Cid) (api.DataSize, error)                                                     `perm:"read"`
+		ClientImport                              func(ctx context.Context, ref api.FileRef) (*api.ImportRes, error)                                                `perm:"admin"`
+		ClientListImports                         func(ctx context.Context) ([]api.Import, error)                                                                   `perm:"write"`
+		ClientRemoveImport                        func(ctx context.Context, importID multistore.StoreID) error                                                      `perm:"admin"`
+		ClientHasLocal                            func(ctx context.Context, root cid.Cid) (bool, error)                                                             `perm:"write"`
+		ClientFindData                            func(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]api.QueryOffer, error)                                 `perm:"read"`
+		ClientMinerQueryOffer                     func(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error)            `perm:"read"`
+		ClientStartDeal                           func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error)                                          `perm:"admin"`
+		ClientGetDealInfo                         func(context.Context, cid.Cid) (*api.DealInfo, error)                                                             `perm:"read"`
+		ClientListDeals                           func(ctx context.Context) ([]api.DealInfo, error)                                                                 `perm:"write"`
+		ClientGetDealUpdates                      func(ctx context.Context) (<-chan api.DealInfo, error)                                                            `perm:"read"`
+		ClientRetrieve                            func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error                                       `perm:"admin"`
+		ClientRetrieveWithEvents                  func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) `perm:"admin"`
+		ClientQueryAsk                            func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error)              `perm:"read"`
+		ClientCalcCommP                           func(ctx context.Context, inpath string) (*api.CommPRet, error)                                                   `perm:"read"`
+		ClientGenCar                              func(ctx context.Context, ref api.FileRef, outpath string) error                                                  `perm:"write"`
+		ClientDealSize                            func(ctx context.Context, root cid.Cid) (api.DataSize, error)                                                     `perm:"read"`
+		ClientListDataTransfers                   func(ctx context.Context) ([]api.DataTransferChannel, error)                                                      `perm:"write"`
+		ClientDataTransferUpdates                 func(ctx context.Context) (<-chan api.DataTransferChannel, error)                                                 `perm:"write"`
+		ClientRetrieveTryRestartInsufficientFunds func(ctx context.Context, paymentChannel address.Address) error                                                   `perm:"write"`
 
 		StateNetworkName                   func(context.Context) (dtypes.NetworkName, error)                                                                   `perm:"read"`
-		StateMinerSectors                  func(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error)        `perm:"read"`
+		StateMinerSectors                  func(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error)   `perm:"read"`
 		StateMinerActiveSectors            func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error)                             `perm:"read"`
-		StateMinerProvingDeadline          func(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error)                                `perm:"read"`
+		StateMinerProvingDeadline          func(context.Context, address.Address, types.TipSetKey) (*dline.Info, error)                                        `perm:"read"`
 		StateMinerPower                    func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)                                    `perm:"read"`
 		StateMinerInfo                     func(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error)                                      `perm:"read"`
 		StateMinerDeadlines                func(context.Context, address.Address, types.TipSetKey) ([]*miner.Deadline, error)                                  `perm:"read"`
 		StateMinerPartitions               func(context.Context, address.Address, uint64, types.TipSetKey) ([]*miner.Partition, error)                         `perm:"read"`
-		StateMinerFaults                   func(context.Context, address.Address, types.TipSetKey) (abi.BitField, error)                                       `perm:"read"`
+		StateMinerFaults                   func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error)                                  `perm:"read"`
 		StateAllMinerFaults                func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error)                                        `perm:"read"`
-		StateMinerRecoveries               func(context.Context, address.Address, types.TipSetKey) (abi.BitField, error)                                       `perm:"read"`
+		StateMinerRecoveries               func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error)                                  `perm:"read"`
 		StateMinerPreCommitDepositForPower func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error)            `perm:"read"`
 		StateMinerInitialPledgeCollateral  func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error)            `perm:"read"`
 		StateMinerAvailableBalance         func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)                                       `perm:"read"`
@@ -181,34 +200,40 @@ type FullNodeStruct struct {
 		StateCompute                       func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error)           `perm:"read"`
 		StateVerifiedClientStatus          func(context.Context, address.Address, types.TipSetKey) (*verifreg.DataCap, error)                                  `perm:"read"`
 		StateDealProviderCollateralBounds  func(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error)                 `perm:"read"`
-		StateCirculatingSupply             func(context.Context, types.TipSetKey) (abi.TokenAmount, error)                                                     `perm:"read"`
+		StateCirculatingSupply             func(context.Context, types.TipSetKey) (api.CirculatingSupply, error)                                               `perm:"read"`
 
 		MsigGetAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)                                                                    `perm:"read"`
+		MsigGetVested           func(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error)                                                   `perm:"read"`
 		MsigCreate              func(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error)                   `perm:"sign"`
 		MsigPropose             func(context.Context, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error)                          `perm:"sign"`
 		MsigApprove             func(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"`
 		MsigCancel              func(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error)                  `perm:"sign"`
+		MsigAddPropose          func(context.Context, address.Address, address.Address, address.Address, bool) (cid.Cid, error)                                                  `perm:"sign"`
+		MsigAddApprove          func(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, bool) (cid.Cid, error)                         `perm:"sign"`
+		MsigAddCancel           func(context.Context, address.Address, address.Address, uint64, address.Address, bool) (cid.Cid, error)                                          `perm:"sign"`
 		MsigSwapPropose         func(context.Context, address.Address, address.Address, address.Address, address.Address) (cid.Cid, error)                                       `perm:"sign"`
 		MsigSwapApprove         func(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, address.Address) (cid.Cid, error)              `perm:"sign"`
 		MsigSwapCancel          func(context.Context, address.Address, address.Address, uint64, address.Address, address.Address) (cid.Cid, error)                               `perm:"sign"`
 
 		MarketEnsureAvailable func(context.Context, address.Address, address.Address, types.BigInt) (cid.Cid, error) `perm:"sign"`
 
-		PaychGet                   func(ctx context.Context, from, to address.Address, amt types.BigInt) (*api.ChannelInfo, error)           `perm:"sign"`
-		PaychGetWaitReady          func(context.Context, cid.Cid) (address.Address, error)                                                   `perm:"sign"`
-		PaychList                  func(context.Context) ([]address.Address, error)                                                          `perm:"read"`
-		PaychStatus                func(context.Context, address.Address) (*api.PaychStatus, error)                                          `perm:"read"`
-		PaychSettle                func(context.Context, address.Address) (cid.Cid, error)                                                   `perm:"sign"`
-		PaychCollect               func(context.Context, address.Address) (cid.Cid, error)                                                   `perm:"sign"`
-		PaychAllocateLane          func(context.Context, address.Address) (uint64, error)                                                    `perm:"sign"`
-		PaychNewPayment            func(ctx context.Context, from, to address.Address, vouchers []api.VoucherSpec) (*api.PaymentInfo, error) `perm:"sign"`
-		PaychVoucherCheck          func(context.Context, *paych.SignedVoucher) error                                                         `perm:"read"`
-		PaychVoucherCheckValid     func(context.Context, address.Address, *paych.SignedVoucher) error                                        `perm:"read"`
-		PaychVoucherCheckSpendable func(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (bool, error)                `perm:"read"`
-		PaychVoucherAdd            func(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error)  `perm:"write"`
-		PaychVoucherCreate         func(context.Context, address.Address, big.Int, uint64) (*paych.SignedVoucher, error)                     `perm:"sign"`
-		PaychVoucherList           func(context.Context, address.Address) ([]*paych.SignedVoucher, error)                                    `perm:"write"`
-		PaychVoucherSubmit         func(context.Context, address.Address, *paych.SignedVoucher) (cid.Cid, error)                             `perm:"sign"`
+		PaychGet                    func(ctx context.Context, from, to address.Address, amt types.BigInt) (*api.ChannelInfo, error)           `perm:"sign"`
+		PaychGetWaitReady           func(context.Context, cid.Cid) (address.Address, error)                                                   `perm:"sign"`
+		PaychAvailableFunds         func(address.Address) (*api.ChannelAvailableFunds, error)                                                 `perm:"sign"`
+		PaychAvailableFundsByFromTo func(address.Address, address.Address) (*api.ChannelAvailableFunds, error)                                `perm:"sign"`
+		PaychList                   func(context.Context) ([]address.Address, error)                                                          `perm:"read"`
+		PaychStatus                 func(context.Context, address.Address) (*api.PaychStatus, error)                                          `perm:"read"`
+		PaychSettle                 func(context.Context, address.Address) (cid.Cid, error)                                                   `perm:"sign"`
+		PaychCollect                func(context.Context, address.Address) (cid.Cid, error)                                                   `perm:"sign"`
+		PaychAllocateLane           func(context.Context, address.Address) (uint64, error)                                                    `perm:"sign"`
+		PaychNewPayment             func(ctx context.Context, from, to address.Address, vouchers []api.VoucherSpec) (*api.PaymentInfo, error) `perm:"sign"`
+		PaychVoucherCheck           func(context.Context, *paych.SignedVoucher) error                                                         `perm:"read"`
+		PaychVoucherCheckValid      func(context.Context, address.Address, *paych.SignedVoucher) error                                        `perm:"read"`
+		PaychVoucherCheckSpendable  func(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (bool, error)                `perm:"read"`
+		PaychVoucherAdd             func(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error)  `perm:"write"`
+		PaychVoucherCreate          func(context.Context, address.Address, big.Int, uint64) (*api.VoucherCreateResult, error)                 `perm:"sign"`
+		PaychVoucherList            func(context.Context, address.Address) ([]*paych.SignedVoucher, error)                                    `perm:"write"`
+		PaychVoucherSubmit          func(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error)             `perm:"sign"`
 	}
 }
 
@@ -228,12 +253,14 @@ type StorageMinerStruct struct {
 		MarketImportDealData      func(context.Context, cid.Cid, string) error                                                                                                                                 `perm:"write"`
 		MarketListDeals           func(ctx context.Context) ([]storagemarket.StorageDeal, error)                                                                                                               `perm:"read"`
 		MarketListRetrievalDeals  func(ctx context.Context) ([]retrievalmarket.ProviderDealState, error)                                                                                                       `perm:"read"`
-		MarketGetDealUpdates      func(ctx context.Context, d cid.Cid) (<-chan storagemarket.MinerDeal, error)                                                                                                 `perm:"read"`
+		MarketGetDealUpdates      func(ctx context.Context) (<-chan storagemarket.MinerDeal, error)                                                                                                            `perm:"read"`
 		MarketListIncompleteDeals func(ctx context.Context) ([]storagemarket.MinerDeal, error)                                                                                                                 `perm:"read"`
 		MarketSetAsk              func(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error `perm:"admin"`
 		MarketGetAsk              func(ctx context.Context) (*storagemarket.SignedStorageAsk, error)                                                                                                           `perm:"read"`
 		MarketSetRetrievalAsk     func(ctx context.Context, rask *retrievalmarket.Ask) error                                                                                                                   `perm:"admin"`
 		MarketGetRetrievalAsk     func(ctx context.Context) (*retrievalmarket.Ask, error)                                                                                                                      `perm:"read"`
+		MarketListDataTransfers   func(ctx context.Context) ([]api.DataTransferChannel, error)                                                                                                                 `perm:"write"`
+		MarketDataTransferUpdates func(ctx context.Context) (<-chan api.DataTransferChannel, error)                                                                                                            `perm:"write"`
 
 		PledgeSector func(context.Context) error `perm:"write"`
 
@@ -300,6 +327,7 @@ type WorkerStruct struct {
 		Paths     func(context.Context) ([]stores.StoragePath, error)            `perm:"admin"`
 		Info      func(context.Context) (storiface.WorkerInfo, error)            `perm:"admin"`
 
+		AddPiece        func(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error)                      `perm:"admin"`
 		SealPreCommit1  func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error)                                                           `perm:"admin"`
 		SealPreCommit2  func(context.Context, abi.SectorID, storage.PreCommit1Out) (cids storage.SectorCids, err error)                                                                                            `perm:"admin"`
 		SealCommit1     func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) `perm:"admin"`
@@ -307,7 +335,8 @@ type WorkerStruct struct {
 		FinalizeSector  func(context.Context, abi.SectorID, []storage.Range) error                                                                                                                                 `perm:"admin"`
 		ReleaseUnsealed func(ctx context.Context, sector abi.SectorID, safeToFree []storage.Range) error                                                                                                           `perm:"admin"`
 		Remove          func(ctx context.Context, sector abi.SectorID) error                                                                                                                                       `perm:"admin"`
-		MoveStorage     func(ctx context.Context, sector abi.SectorID) error                                                                                                                                       `perm:"admin"`
+		MoveStorage     func(ctx context.Context, sector abi.SectorID, types stores.SectorFileType) error                                                                                                          `perm:"admin"`
+		StorageAddLocal func(ctx context.Context, path string) error                                                                                                                                               `perm:"admin"`
 
 		UnsealPiece func(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error `perm:"admin"`
 		ReadPiece   func(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) (bool, error)           `perm:"admin"`
@@ -360,6 +389,22 @@ func (c *CommonStruct) NetAutoNatStatus(ctx context.Context) (api.NatInfo, error
 	return c.Internal.NetAutoNatStatus(ctx)
 }
 
+func (c *CommonStruct) NetBandwidthStats(ctx context.Context) (metrics.Stats, error) {
+	return c.Internal.NetBandwidthStats(ctx)
+}
+
+func (c *CommonStruct) NetBandwidthStatsByPeer(ctx context.Context) (map[string]metrics.Stats, error) {
+	return c.Internal.NetBandwidthStatsByPeer(ctx)
+}
+
+func (c *CommonStruct) NetBandwidthStatsByProtocol(ctx context.Context) (map[protocol.ID]metrics.Stats, error) {
+	return c.Internal.NetBandwidthStatsByProtocol(ctx)
+}
+
+func (c *CommonStruct) NetAgentVersion(ctx context.Context, p peer.ID) (string, error) {
+	return c.Internal.NetAgentVersion(ctx, p)
+}
+
 // ID implements API.ID
 func (c *CommonStruct) ID(ctx context.Context) (peer.ID, error) {
 	return c.Internal.ID(ctx)
@@ -424,10 +469,18 @@ func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]api.DealInfo, e
 	return c.Internal.ClientListDeals(ctx)
 }
 
-func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
+func (c *FullNodeStruct) ClientGetDealUpdates(ctx context.Context) (<-chan api.DealInfo, error) {
+	return c.Internal.ClientGetDealUpdates(ctx)
+}
+
+func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error {
 	return c.Internal.ClientRetrieve(ctx, order, ref)
 }
 
+func (c *FullNodeStruct) ClientRetrieveWithEvents(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
+	return c.Internal.ClientRetrieveWithEvents(ctx, order, ref)
+}
+
 func (c *FullNodeStruct) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) {
 	return c.Internal.ClientQueryAsk(ctx, p, miner)
 }
@@ -443,17 +496,31 @@ func (c *FullNodeStruct) ClientDealSize(ctx context.Context, root cid.Cid) (api.
 	return c.Internal.ClientDealSize(ctx, root)
 }
 
-func (c *FullNodeStruct) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64,
-	sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) {
+func (c *FullNodeStruct) ClientListDataTransfers(ctx context.Context) ([]api.DataTransferChannel, error) {
+	return c.Internal.ClientListDataTransfers(ctx)
+}
+
+func (c *FullNodeStruct) ClientDataTransferUpdates(ctx context.Context) (<-chan api.DataTransferChannel, error) {
+	return c.Internal.ClientDataTransferUpdates(ctx)
+}
+
+func (c *FullNodeStruct) ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error {
+	return c.Internal.ClientRetrieveTryRestartInsufficientFunds(ctx, paymentChannel)
+}
+
+func (c *FullNodeStruct) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) {
 	return c.Internal.GasEstimateGasPremium(ctx, nblocksincl, sender, gaslimit, tsk)
 }
-func (c *FullNodeStruct) GasEstimateFeeCap(ctx context.Context, msg *types.Message,
-	maxqueueblks int64, tsk types.TipSetKey) (types.BigInt, error) {
+
+func (c *FullNodeStruct) GasEstimateFeeCap(ctx context.Context, msg *types.Message, maxqueueblks int64, tsk types.TipSetKey) (types.BigInt, error) {
 	return c.Internal.GasEstimateFeeCap(ctx, msg, maxqueueblks, tsk)
 }
 
-func (c *FullNodeStruct) GasEstimateGasLimit(ctx context.Context, msg *types.Message,
-	tsk types.TipSetKey) (int64, error) {
+func (c *FullNodeStruct) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) {
+	return c.Internal.GasEstimateMessageGas(ctx, msg, spec, tsk)
+}
+
+func (c *FullNodeStruct) GasEstimateGasLimit(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (int64, error) {
 	return c.Internal.GasEstimateGasLimit(ctx, msg, tsk)
 }
 
@@ -473,6 +540,10 @@ func (c *FullNodeStruct) MpoolPending(ctx context.Context, tsk types.TipSetKey)
 	return c.Internal.MpoolPending(ctx, tsk)
 }
 
+func (c *FullNodeStruct) MpoolClear(ctx context.Context, local bool) error {
+	return c.Internal.MpoolClear(ctx, local)
+}
+
 func (c *FullNodeStruct) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) {
 	return c.Internal.MpoolPush(ctx, smsg)
 }
@@ -621,8 +692,8 @@ func (c *FullNodeStruct) ChainGetPath(ctx context.Context, from types.TipSetKey,
 	return c.Internal.ChainGetPath(ctx, from, to)
 }
 
-func (c *FullNodeStruct) ChainExport(ctx context.Context, tsk types.TipSetKey) (<-chan []byte, error) {
-	return c.Internal.ChainExport(ctx, tsk)
+func (c *FullNodeStruct) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk types.TipSetKey) (<-chan []byte, error) {
+	return c.Internal.ChainExport(ctx, nroots, tsk)
 }
 
 func (c *FullNodeStruct) BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) {
@@ -641,10 +712,18 @@ func (c *FullNodeStruct) SyncIncomingBlocks(ctx context.Context) (<-chan *types.
 	return c.Internal.SyncIncomingBlocks(ctx)
 }
 
+func (c *FullNodeStruct) SyncCheckpoint(ctx context.Context, tsk types.TipSetKey) error {
+	return c.Internal.SyncCheckpoint(ctx, tsk)
+}
+
 func (c *FullNodeStruct) SyncMarkBad(ctx context.Context, bcid cid.Cid) error {
 	return c.Internal.SyncMarkBad(ctx, bcid)
 }
 
+func (c *FullNodeStruct) SyncUnmarkBad(ctx context.Context, bcid cid.Cid) error {
+	return c.Internal.SyncUnmarkBad(ctx, bcid)
+}
+
 func (c *FullNodeStruct) SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error) {
 	return c.Internal.SyncCheckBad(ctx, bcid)
 }
@@ -653,7 +732,7 @@ func (c *FullNodeStruct) StateNetworkName(ctx context.Context) (dtypes.NetworkNa
 	return c.Internal.StateNetworkName(ctx)
 }
 
-func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, filter *abi.BitField, filterOut bool, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) {
+func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, filterOut bool, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) {
 	return c.Internal.StateMinerSectors(ctx, addr, filter, filterOut, tsk)
 }
 
@@ -661,7 +740,7 @@ func (c *FullNodeStruct) StateMinerActiveSectors(ctx context.Context, addr addre
 	return c.Internal.StateMinerActiveSectors(ctx, addr, tsk)
 }
 
-func (c *FullNodeStruct) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*miner.DeadlineInfo, error) {
+func (c *FullNodeStruct) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error) {
 	return c.Internal.StateMinerProvingDeadline(ctx, addr, tsk)
 }
 
@@ -681,7 +760,7 @@ func (c *FullNodeStruct) StateMinerPartitions(ctx context.Context, m address.Add
 	return c.Internal.StateMinerPartitions(ctx, m, dlIdx, tsk)
 }
 
-func (c *FullNodeStruct) StateMinerFaults(ctx context.Context, actor address.Address, tsk types.TipSetKey) (abi.BitField, error) {
+func (c *FullNodeStruct) StateMinerFaults(ctx context.Context, actor address.Address, tsk types.TipSetKey) (bitfield.BitField, error) {
 	return c.Internal.StateMinerFaults(ctx, actor, tsk)
 }
 
@@ -689,7 +768,7 @@ func (c *FullNodeStruct) StateAllMinerFaults(ctx context.Context, cutoff abi.Cha
 	return c.Internal.StateAllMinerFaults(ctx, cutoff, endTsk)
 }
 
-func (c *FullNodeStruct) StateMinerRecoveries(ctx context.Context, actor address.Address, tsk types.TipSetKey) (abi.BitField, error) {
+func (c *FullNodeStruct) StateMinerRecoveries(ctx context.Context, actor address.Address, tsk types.TipSetKey) (bitfield.BitField, error) {
 	return c.Internal.StateMinerRecoveries(ctx, actor, tsk)
 }
 
@@ -801,7 +880,7 @@ func (c *FullNodeStruct) StateDealProviderCollateralBounds(ctx context.Context,
 	return c.Internal.StateDealProviderCollateralBounds(ctx, size, verified, tsk)
 }
 
-func (c *FullNodeStruct) StateCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (abi.TokenAmount, error) {
+func (c *FullNodeStruct) StateCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) {
 	return c.Internal.StateCirculatingSupply(ctx, tsk)
 }
 
@@ -809,6 +888,10 @@ func (c *FullNodeStruct) MsigGetAvailableBalance(ctx context.Context, a address.
 	return c.Internal.MsigGetAvailableBalance(ctx, a, tsk)
 }
 
+func (c *FullNodeStruct) MsigGetVested(ctx context.Context, a address.Address, sTsk types.TipSetKey, eTsk types.TipSetKey) (types.BigInt, error) {
+	return c.Internal.MsigGetVested(ctx, a, sTsk, eTsk)
+}
+
 func (c *FullNodeStruct) MsigCreate(ctx context.Context, req uint64, addrs []address.Address, duration abi.ChainEpoch, val types.BigInt, src address.Address, gp types.BigInt) (cid.Cid, error) {
 	return c.Internal.MsigCreate(ctx, req, addrs, duration, val, src, gp)
 }
@@ -825,6 +908,18 @@ func (c *FullNodeStruct) MsigCancel(ctx context.Context, msig address.Address, t
 	return c.Internal.MsigCancel(ctx, msig, txID, to, amt, src, method, params)
 }
 
+func (c *FullNodeStruct) MsigAddPropose(ctx context.Context, msig address.Address, src address.Address, newAdd address.Address, inc bool) (cid.Cid, error) {
+	return c.Internal.MsigAddPropose(ctx, msig, src, newAdd, inc)
+}
+
+func (c *FullNodeStruct) MsigAddApprove(ctx context.Context, msig address.Address, src address.Address, txID uint64, proposer address.Address, newAdd address.Address, inc bool) (cid.Cid, error) {
+	return c.Internal.MsigAddApprove(ctx, msig, src, txID, proposer, newAdd, inc)
+}
+
+func (c *FullNodeStruct) MsigAddCancel(ctx context.Context, msig address.Address, src address.Address, txID uint64, newAdd address.Address, inc bool) (cid.Cid, error) {
+	return c.Internal.MsigAddCancel(ctx, msig, src, txID, newAdd, inc)
+}
+
 func (c *FullNodeStruct) MsigSwapPropose(ctx context.Context, msig address.Address, src address.Address, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) {
 	return c.Internal.MsigSwapPropose(ctx, msig, src, oldAdd, newAdd)
 }
@@ -849,6 +944,14 @@ func (c *FullNodeStruct) PaychGetWaitReady(ctx context.Context, sentinel cid.Cid
 	return c.Internal.PaychGetWaitReady(ctx, sentinel)
 }
 
+func (c *FullNodeStruct) PaychAvailableFunds(ch address.Address) (*api.ChannelAvailableFunds, error) {
+	return c.Internal.PaychAvailableFunds(ch)
+}
+
+func (c *FullNodeStruct) PaychAvailableFundsByFromTo(from, to address.Address) (*api.ChannelAvailableFunds, error) {
+	return c.Internal.PaychAvailableFundsByFromTo(from, to)
+}
+
 func (c *FullNodeStruct) PaychList(ctx context.Context) ([]address.Address, error) {
 	return c.Internal.PaychList(ctx)
 }
@@ -869,7 +972,7 @@ func (c *FullNodeStruct) PaychVoucherAdd(ctx context.Context, addr address.Addre
 	return c.Internal.PaychVoucherAdd(ctx, addr, sv, proof, minDelta)
 }
 
-func (c *FullNodeStruct) PaychVoucherCreate(ctx context.Context, pch address.Address, amt types.BigInt, lane uint64) (*paych.SignedVoucher, error) {
+func (c *FullNodeStruct) PaychVoucherCreate(ctx context.Context, pch address.Address, amt types.BigInt, lane uint64) (*api.VoucherCreateResult, error) {
 	return c.Internal.PaychVoucherCreate(ctx, pch, amt, lane)
 }
 
@@ -893,8 +996,8 @@ func (c *FullNodeStruct) PaychNewPayment(ctx context.Context, from, to address.A
 	return c.Internal.PaychNewPayment(ctx, from, to, vouchers)
 }
 
-func (c *FullNodeStruct) PaychVoucherSubmit(ctx context.Context, ch address.Address, sv *paych.SignedVoucher) (cid.Cid, error) {
-	return c.Internal.PaychVoucherSubmit(ctx, ch, sv)
+func (c *FullNodeStruct) PaychVoucherSubmit(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) {
+	return c.Internal.PaychVoucherSubmit(ctx, ch, sv, secret, proof)
 }
 
 // StorageMinerStruct
@@ -1037,8 +1140,8 @@ func (c *StorageMinerStruct) MarketListRetrievalDeals(ctx context.Context) ([]re
 	return c.Internal.MarketListRetrievalDeals(ctx)
 }
 
-func (c *StorageMinerStruct) MarketGetDealUpdates(ctx context.Context, d cid.Cid) (<-chan storagemarket.MinerDeal, error) {
-	return c.Internal.MarketGetDealUpdates(ctx, d)
+func (c *StorageMinerStruct) MarketGetDealUpdates(ctx context.Context) (<-chan storagemarket.MinerDeal, error) {
+	return c.Internal.MarketGetDealUpdates(ctx)
 }
 
 func (c *StorageMinerStruct) MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error) {
@@ -1061,6 +1164,14 @@ func (c *StorageMinerStruct) MarketGetRetrievalAsk(ctx context.Context) (*retrie
 	return c.Internal.MarketGetRetrievalAsk(ctx)
 }
 
+func (c *StorageMinerStruct) MarketListDataTransfers(ctx context.Context) ([]api.DataTransferChannel, error) {
+	return c.Internal.MarketListDataTransfers(ctx)
+}
+
+func (c *StorageMinerStruct) MarketDataTransferUpdates(ctx context.Context) (<-chan api.DataTransferChannel, error) {
+	return c.Internal.MarketDataTransferUpdates(ctx)
+}
+
 func (c *StorageMinerStruct) DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error {
 	return c.Internal.DealsImportData(ctx, dealPropCid, file)
 }
@@ -1147,6 +1258,10 @@ func (w *WorkerStruct) Info(ctx context.Context) (storiface.WorkerInfo, error) {
 	return w.Internal.Info(ctx)
 }
 
+func (w *WorkerStruct) AddPiece(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) {
+	return w.Internal.AddPiece(ctx, sector, pieceSizes, newPieceSize, pieceData)
+}
+
 func (w *WorkerStruct) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) {
 	return w.Internal.SealPreCommit1(ctx, sector, ticket, pieces)
 }
@@ -1175,8 +1290,12 @@ func (w *WorkerStruct) Remove(ctx context.Context, sector abi.SectorID) error {
 	return w.Internal.Remove(ctx, sector)
 }
 
-func (w *WorkerStruct) MoveStorage(ctx context.Context, sector abi.SectorID) error {
-	return w.Internal.MoveStorage(ctx, sector)
+func (w *WorkerStruct) MoveStorage(ctx context.Context, sector abi.SectorID, types stores.SectorFileType) error {
+	return w.Internal.MoveStorage(ctx, sector, types)
+}
+
+func (w *WorkerStruct) StorageAddLocal(ctx context.Context, path string) error {
+	return w.Internal.StorageAddLocal(ctx, path)
 }
 
 func (w *WorkerStruct) UnsealPiece(ctx context.Context, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, c cid.Cid) error {

api/cbor_gen.go

@@ -6,8 +6,8 @@ import (
 	"fmt"
 	"io"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/builtin/paych"
+	abi "github.com/filecoin-project/go-state-types/abi"
+	paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	xerrors "golang.org/x/xerrors"
 )

api/client/client.go

@@ -1,18 +1,23 @@
 package client
 
 import (
+	"context"
 	"net/http"
+	"net/url"
+	"path"
+	"time"
 
 	"github.com/filecoin-project/go-jsonrpc"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/apistruct"
+	"github.com/filecoin-project/lotus/lib/rpcenc"
 )
 
 // NewCommonRPC creates a new http jsonrpc client.
-func NewCommonRPC(addr string, requestHeader http.Header) (api.Common, jsonrpc.ClientCloser, error) {
+func NewCommonRPC(ctx context.Context, addr string, requestHeader http.Header) (api.Common, jsonrpc.ClientCloser, error) {
 	var res apistruct.CommonStruct
-	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
 		[]interface{}{
 			&res.Internal,
 		},
@@ -23,9 +28,9 @@ func NewCommonRPC(addr string, requestHeader http.Header) (api.Common, jsonrpc.C
 }
 
 // NewFullNodeRPC creates a new http jsonrpc client.
-func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
+func NewFullNodeRPC(ctx context.Context, addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
 	var res apistruct.FullNodeStruct
-	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
 		[]interface{}{
 			&res.CommonStruct.Internal,
 			&res.Internal,
@@ -35,26 +40,44 @@ func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonr
 }
 
 // NewStorageMinerRPC creates a new http jsonrpc client for miner
-func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMiner, jsonrpc.ClientCloser, error) {
+func NewStorageMinerRPC(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.StorageMiner, jsonrpc.ClientCloser, error) {
 	var res apistruct.StorageMinerStruct
-	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
 		[]interface{}{
 			&res.CommonStruct.Internal,
 			&res.Internal,
 		},
 		requestHeader,
+		opts...,
 	)
 
 	return &res, closer, err
 }
 
-func NewWorkerRPC(addr string, requestHeader http.Header) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
+func NewWorkerRPC(ctx context.Context, addr string, requestHeader http.Header) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
+	u, err := url.Parse(addr)
+	if err != nil {
+		return nil, nil, err
+	}
+	switch u.Scheme {
+	case "ws":
+		u.Scheme = "http"
+	case "wss":
+		u.Scheme = "https"
+	}
+	///rpc/v0 -> /rpc/streams/v0/push
+
+	u.Path = path.Join(u.Path, "../streams/v0/push")
+
 	var res apistruct.WorkerStruct
-	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
 		[]interface{}{
 			&res.Internal,
 		},
 		requestHeader,
+		rpcenc.ReaderParamEncoder(u.String()),
+		jsonrpc.WithNoReconnect(),
+		jsonrpc.WithTimeout(30*time.Second),
 	)
 
 	return &res, closer, err

api/docgen/docgen.go

@@ -12,22 +12,31 @@ import (
 	"time"
 	"unicode"
 
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-filestore"
+	metrics "github.com/libp2p/go-libp2p-core/metrics"
+	"github.com/libp2p/go-libp2p-core/network"
+	"github.com/libp2p/go-libp2p-core/peer"
+	protocol "github.com/libp2p/go-libp2p-core/protocol"
+	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	"github.com/multiformats/go-multiaddr"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
+	datatransfer "github.com/filecoin-project/go-data-transfer"
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-jsonrpc/auth"
+	"github.com/filecoin-project/go-multistore"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/go-state-types/exitcode"
+
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/apistruct"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/crypto"
-	"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
-	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-filestore"
-	"github.com/libp2p/go-libp2p-core/network"
-	peer "github.com/libp2p/go-libp2p-peer"
-	"github.com/multiformats/go-multiaddr"
 )
 
 var ExampleValues = map[reflect.Type]interface{}{
@@ -66,11 +75,12 @@ func init() {
 
 	ExampleValues[reflect.TypeOf(addr)] = addr
 
-	pid, err := peer.IDB58Decode("12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf")
+	pid, err := peer.Decode("12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf")
 	if err != nil {
 		panic(err)
 	}
 	addExample(pid)
+	addExample(&pid)
 
 	addExample(bitfield.NewFromSet([]uint64{5}))
 	addExample(abi.RegisteredSealProof_StackedDrg32GiBV1)
@@ -95,9 +105,15 @@ func init() {
 	addExample(network.Connected)
 	addExample(dtypes.NetworkName("lotus"))
 	addExample(api.SyncStateStage(1))
-	addExample(build.APIVersion)
+	addExample(build.FullAPIVersion)
 	addExample(api.PCHInbound)
 	addExample(time.Minute)
+	addExample(datatransfer.TransferID(3))
+	addExample(datatransfer.Ongoing)
+	addExample(multistore.StoreID(50))
+	addExample(retrievalmarket.ClientEventDealAccepted)
+	addExample(retrievalmarket.DealStatusNew)
+	addExample(network.ReachabilityPublic)
 	addExample(&types.ExecutionTrace{
 		Msg:    exampleValue(reflect.TypeOf(&types.Message{}), nil).(*types.Message),
 		MsgRct: exampleValue(reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt),
@@ -111,6 +127,30 @@ func init() {
 	addExample(map[string]api.MarketBalance{
 		"t026363": exampleValue(reflect.TypeOf(api.MarketBalance{}), nil).(api.MarketBalance),
 	})
+	addExample(map[string]*pubsub.TopicScoreSnapshot{
+		"/blocks": {
+			TimeInMesh:               time.Minute,
+			FirstMessageDeliveries:   122,
+			MeshMessageDeliveries:    1234,
+			InvalidMessageDeliveries: 3,
+		},
+	})
+	addExample(map[string]metrics.Stats{
+		"12D3KooWSXmXLJmBR1M7i9RW9GQPNUhZSzXKzxDHWtAgNuJAbyEJ": {
+			RateIn:   100,
+			RateOut:  50,
+			TotalIn:  174000,
+			TotalOut: 12500,
+		},
+	})
+	addExample(map[protocol.ID]metrics.Stats{
+		"/fil/hello/1.0.0": {
+			RateIn:   100,
+			RateOut:  50,
+			TotalIn:  174000,
+			TotalOut: 12500,
+		},
+	})
 
 	maddr, err := multiaddr.NewMultiaddr("/ip4/52.36.61.156/tcp/1347/p2p/12D3KooWFETiESTf1v4PGUvtnxMAcEFMzLZbJGg4tjWfGEimYior")
 	if err != nil {

api/test/blockminer.go

@@ -0,0 +1,56 @@
+package test
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/miner"
+)
+
+type BlockMiner struct {
+	ctx       context.Context
+	t         *testing.T
+	miner     TestStorageNode
+	blocktime time.Duration
+	mine      int64
+	nulls     int64
+	done      chan struct{}
+}
+
+func NewBlockMiner(ctx context.Context, t *testing.T, miner TestStorageNode, blocktime time.Duration) *BlockMiner {
+	return &BlockMiner{
+		ctx:       ctx,
+		t:         t,
+		miner:     miner,
+		blocktime: blocktime,
+		mine:      int64(1),
+		done:      make(chan struct{}),
+	}
+}
+
+func (bm *BlockMiner) MineBlocks() {
+	time.Sleep(time.Second)
+	go func() {
+		defer close(bm.done)
+		for atomic.LoadInt64(&bm.mine) == 1 {
+			time.Sleep(bm.blocktime)
+			nulls := atomic.SwapInt64(&bm.nulls, 0)
+			if err := bm.miner.MineOne(bm.ctx, miner.MineReq{
+				InjectNulls: abi.ChainEpoch(nulls),
+				Done:        func(bool, abi.ChainEpoch, error) {},
+			}); err != nil {
+				bm.t.Error(err)
+			}
+		}
+	}()
+}
+
+func (bm *BlockMiner) Stop() {
+	atomic.AddInt64(&bm.mine, -1)
+	fmt.Println("shutting down mining")
+	<-bm.done
+}

api/test/ccupgrade.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
 
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/node/impl"
@@ -20,7 +20,7 @@ func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) {
 	_ = os.Setenv("BELLMAN_NO_GPU", "1")
 
 	ctx := context.Background()
-	n, sn := b(t, 1, oneMiner)
+	n, sn := b(t, 1, OneMiner)
 	client := n[0].FullNode.(*impl.FullNodeAPI)
 	miner := sn[0]
 

api/test/deals.go

@@ -20,10 +20,11 @@ import (
 	"github.com/ipld/go-car"
 
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
+	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
 	"github.com/filecoin-project/lotus/miner"
-	sealing "github.com/filecoin-project/storage-fsm"
 	dag "github.com/ipfs/go-merkledag"
 	dstest "github.com/ipfs/go-merkledag/test"
 	unixfile "github.com/ipfs/go-unixfs/file"
@@ -35,7 +36,7 @@ import (
 
 var MineNext = miner.MineReq{
 	InjectNulls: 0,
-	Done:        func(bool, error) {},
+	Done:        func(bool, abi.ChainEpoch, error) {},
 }
 
 func init() {
@@ -47,7 +48,7 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport
 	_ = os.Setenv("BELLMAN_NO_GPU", "1")
 
 	ctx := context.Background()
-	n, sn := b(t, 1, oneMiner)
+	n, sn := b(t, 1, OneMiner)
 	client := n[0].FullNode.(*impl.FullNodeAPI)
 	miner := sn[0]
 
@@ -84,7 +85,7 @@ func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
 	_ = os.Setenv("BELLMAN_NO_GPU", "1")
 
 	ctx := context.Background()
-	n, sn := b(t, 1, oneMiner)
+	n, sn := b(t, 1, OneMiner)
 	client := n[0].FullNode.(*impl.FullNodeAPI)
 	miner := sn[0]
 
@@ -141,14 +142,14 @@ func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNod
 	info, err := client.ClientGetDealInfo(ctx, *deal)
 	require.NoError(t, err)
 
-	testRetrieval(t, ctx, err, client, fcid, &info.PieceCID, carExport, data)
+	testRetrieval(t, ctx, client, fcid, &info.PieceCID, carExport, data)
 }
 
 func TestFastRetrievalDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
 	_ = os.Setenv("BELLMAN_NO_GPU", "1")
 
 	ctx := context.Background()
-	n, sn := b(t, 1, oneMiner)
+	n, sn := b(t, 1, OneMiner)
 	client := n[0].FullNode.(*impl.FullNodeAPI)
 	miner := sn[0]
 
@@ -193,7 +194,7 @@ func TestFastRetrievalDealFlow(t *testing.T, b APIBuilder, blocktime time.Durati
 	info, err := client.ClientGetDealInfo(ctx, *deal)
 	require.NoError(t, err)
 
-	testRetrieval(t, ctx, err, client, fcid, &info.PieceCID, false, data)
+	testRetrieval(t, ctx, client, fcid, &info.PieceCID, false, data)
 	atomic.AddInt64(&mine, -1)
 	fmt.Println("shutting down mining")
 	<-done
@@ -203,7 +204,7 @@ func TestSenondDealRetrieval(t *testing.T, b APIBuilder, blocktime time.Duration
 	_ = os.Setenv("BELLMAN_NO_GPU", "1")
 
 	ctx := context.Background()
-	n, sn := b(t, 1, oneMiner)
+	n, sn := b(t, 1, OneMiner)
 	client := n[0].FullNode.(*impl.FullNodeAPI)
 	miner := sn[0]
 
@@ -267,7 +268,7 @@ func TestSenondDealRetrieval(t *testing.T, b APIBuilder, blocktime time.Duration
 		rf, _ := miner.SectorsRefs(ctx)
 		fmt.Printf("refs: %+v\n", rf)
 
-		testRetrieval(t, ctx, err, client, fcid2, &info.PieceCID, false, data2)
+		testRetrieval(t, ctx, client, fcid2, &info.PieceCID, false, data2)
 	}
 
 	atomic.AddInt64(&mine, -1)
@@ -333,7 +334,7 @@ loop:
 func waitDealPublished(t *testing.T, ctx context.Context, miner TestStorageNode, deal *cid.Cid) {
 	subCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
-	updates, err := miner.MarketGetDealUpdates(subCtx, *deal)
+	updates, err := miner.MarketGetDealUpdates(subCtx)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -342,18 +343,20 @@ func waitDealPublished(t *testing.T, ctx context.Context, miner TestStorageNode,
 		case <-ctx.Done():
 			t.Fatal("context timeout")
 		case di := <-updates:
-			switch di.State {
-			case storagemarket.StorageDealProposalRejected:
-				t.Fatal("deal rejected")
-			case storagemarket.StorageDealFailing:
-				t.Fatal("deal failed")
-			case storagemarket.StorageDealError:
-				t.Fatal("deal errored", di.Message)
-			case storagemarket.StorageDealFinalizing, storagemarket.StorageDealSealing, storagemarket.StorageDealActive:
-				fmt.Println("COMPLETE", di)
-				return
+			if deal.Equals(di.ProposalCid) {
+				switch di.State {
+				case storagemarket.StorageDealProposalRejected:
+					t.Fatal("deal rejected")
+				case storagemarket.StorageDealFailing:
+					t.Fatal("deal failed")
+				case storagemarket.StorageDealError:
+					t.Fatal("deal errored", di.Message)
+				case storagemarket.StorageDealFinalizing, storagemarket.StorageDealSealing, storagemarket.StorageDealActive:
+					fmt.Println("COMPLETE", di)
+					return
+				}
+				fmt.Println("Deal state: ", storagemarket.DealStates[di.State])
 			}
-			fmt.Println("Deal state: ", storagemarket.DealStates[di.State])
 		}
 	}
 }
@@ -373,7 +376,7 @@ func startSealingWaiting(t *testing.T, ctx context.Context, miner TestStorageNod
 	}
 }
 
-func testRetrieval(t *testing.T, ctx context.Context, err error, client *impl.FullNodeAPI, fcid cid.Cid, piece *cid.Cid, carExport bool, data []byte) {
+func testRetrieval(t *testing.T, ctx context.Context, client *impl.FullNodeAPI, fcid cid.Cid, piece *cid.Cid, carExport bool, data []byte) {
 	offers, err := client.ClientFindData(ctx, fcid, piece)
 	if err != nil {
 		t.Fatal(err)
@@ -398,7 +401,7 @@ func testRetrieval(t *testing.T, ctx context.Context, err error, client *impl.Fu
 		Path:  filepath.Join(rpath, "ret"),
 		IsCAR: carExport,
 	}
-	updates, err := client.ClientRetrieve(ctx, offers[0].Order(caddr), ref)
+	updates, err := client.ClientRetrieveWithEvents(ctx, offers[0].Order(caddr), ref)
 	for update := range updates {
 		if update.Err != "" {
 			t.Fatalf("%v", err)

api/test/mining.go

@@ -12,7 +12,7 @@ import (
 
 	logging "github.com/ipfs/go-log/v2"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/lotus/build"
@@ -20,32 +20,31 @@ import (
 	"github.com/filecoin-project/lotus/node/impl"
 )
 
+//nolint:deadcode,varcheck
 var log = logging.Logger("apitest")
 
 func (ts *testSuite) testMining(t *testing.T) {
 	ctx := context.Background()
-	apis, sn := ts.makeNodes(t, 1, oneMiner)
+	apis, sn := ts.makeNodes(t, 1, OneMiner)
 	api := apis[0]
 
 	newHeads, err := api.ChainNotify(ctx)
 	require.NoError(t, err)
 	initHead := (<-newHeads)[0]
-	if initHead.Val.Height() != 2 {
-		<-newHeads
-	}
+	baseHeight := initHead.Val.Height()
 
 	h1, err := api.ChainHead(ctx)
 	require.NoError(t, err)
-	require.Equal(t, abi.ChainEpoch(2), h1.Height())
+	require.Equal(t, int64(h1.Height()), int64(baseHeight))
 
-	err = sn[0].MineOne(ctx, MineNext)
+	MineUntilBlock(ctx, t, apis[0], sn[0], nil)
 	require.NoError(t, err)
 
 	<-newHeads
 
 	h2, err := api.ChainHead(ctx)
 	require.NoError(t, err)
-	require.Equal(t, abi.ChainEpoch(3), h2.Height())
+	require.Greater(t, int64(h2.Height()), int64(h1.Height()))
 }
 
 func (ts *testSuite) testMiningReal(t *testing.T) {
@@ -55,7 +54,7 @@ func (ts *testSuite) testMiningReal(t *testing.T) {
 	}()
 
 	ctx := context.Background()
-	apis, sn := ts.makeNodes(t, 1, oneMiner)
+	apis, sn := ts.makeNodes(t, 1, OneMiner)
 	api := apis[0]
 
 	newHeads, err := api.ChainNotify(ctx)
@@ -69,7 +68,7 @@ func (ts *testSuite) testMiningReal(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, abi.ChainEpoch(2), h1.Height())
 
-	err = sn[0].MineOne(ctx, MineNext)
+	MineUntilBlock(ctx, t, apis[0], sn[0], nil)
 	require.NoError(t, err)
 
 	<-newHeads
@@ -78,7 +77,7 @@ func (ts *testSuite) testMiningReal(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, abi.ChainEpoch(3), h2.Height())
 
-	err = sn[0].MineOne(ctx, MineNext)
+	MineUntilBlock(ctx, t, apis[0], sn[0], nil)
 	require.NoError(t, err)
 
 	<-newHeads
@@ -143,7 +142,7 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo
 		complChan := minedTwo
 		for atomic.LoadInt32(&mine) != 0 {
 			wait := make(chan int)
-			mdone := func(mined bool, err error) {
+			mdone := func(mined bool, _ abi.ChainEpoch, err error) {
 				n := 0
 				if mined {
 					n = 1

api/test/paych.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin/paych"
 	"github.com/ipfs/go-cid"
 
@@ -23,14 +23,13 @@ import (
 	"github.com/filecoin-project/lotus/chain/events/state"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/wallet"
-	"github.com/filecoin-project/lotus/miner"
 )
 
 func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) {
 	_ = os.Setenv("BELLMAN_NO_GPU", "1")
 
 	ctx := context.Background()
-	n, sn := b(t, 2, oneMiner)
+	n, sn := b(t, 2, OneMiner)
 
 	paymentCreator := n[0]
 	paymentReceiver := n[1]
@@ -51,8 +50,8 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) {
 	}
 
 	// start mining blocks
-	bm := newBlockMiner(ctx, t, miner, blocktime)
-	bm.mineBlocks()
+	bm := NewBlockMiner(ctx, t, miner, blocktime)
+	bm.MineBlocks()
 
 	// send some funds to register the receiver
 	receiverAddr, err := paymentReceiver.WalletNew(ctx, wallet.ActSigType("secp256k1"))
@@ -60,7 +59,7 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) {
 		t.Fatal(err)
 	}
 
-	sendFunds(ctx, t, paymentCreator, receiverAddr, abi.NewTokenAmount(1e18))
+	SendFunds(ctx, t, paymentCreator, receiverAddr, abi.NewTokenAmount(1e18))
 
 	// setup the payment channel
 	createrAddr, err := paymentCreator.WalletDefaultAddress(ctx)
@@ -97,18 +96,24 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) {
 		if err != nil {
 			t.Fatal(err)
 		}
+		if vouch1.Voucher == nil {
+			t.Fatal(fmt.Errorf("Not enough funds to create voucher: missing %d", vouch1.Shortfall))
+		}
 		vouch2, err := paymentCreator.PaychVoucherCreate(ctx, channel, abi.NewTokenAmount(2000), lane)
 		if err != nil {
 			t.Fatal(err)
 		}
-		delta1, err := paymentReceiver.PaychVoucherAdd(ctx, channel, vouch1, nil, abi.NewTokenAmount(1000))
+		if vouch2.Voucher == nil {
+			t.Fatal(fmt.Errorf("Not enough funds to create voucher: missing %d", vouch2.Shortfall))
+		}
+		delta1, err := paymentReceiver.PaychVoucherAdd(ctx, channel, vouch1.Voucher, nil, abi.NewTokenAmount(1000))
 		if err != nil {
 			t.Fatal(err)
 		}
 		if !delta1.Equals(abi.NewTokenAmount(1000)) {
 			t.Fatal("voucher didn't have the right amount")
 		}
-		delta2, err := paymentReceiver.PaychVoucherAdd(ctx, channel, vouch2, nil, abi.NewTokenAmount(1000))
+		delta2, err := paymentReceiver.PaychVoucherAdd(ctx, channel, vouch2.Voucher, nil, abi.NewTokenAmount(1000))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -154,6 +159,9 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) {
 	}, int(build.MessageConfidence)+1, build.SealRandomnessLookbackLimit, func(oldTs, newTs *types.TipSet) (bool, events.StateChange, error) {
 		return preds.OnPaymentChannelActorChanged(channel, preds.OnToSendAmountChanges())(ctx, oldTs.Key(), newTs.Key())
 	})
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	select {
 	case <-finished:
@@ -201,10 +209,10 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) {
 	}
 
 	// shut down mining
-	bm.stop()
+	bm.Stop()
 }
 
-func waitForBlocks(ctx context.Context, t *testing.T, bm *blockMiner, paymentReceiver TestNode, receiverAddr address.Address, count int) {
+func waitForBlocks(ctx context.Context, t *testing.T, bm *BlockMiner, paymentReceiver TestNode, receiverAddr address.Address, count int) {
 	// We need to add null blocks in batches, if we add too many the chain can't sync
 	batchSize := 60
 	for i := 0; i < count; i += batchSize {
@@ -249,73 +257,3 @@ func waitForMessage(ctx context.Context, t *testing.T, paymentCreator TestNode,
 	fmt.Println("Confirmed", desc)
 	return res
 }
-
-type blockMiner struct {
-	ctx       context.Context
-	t         *testing.T
-	miner     TestStorageNode
-	blocktime time.Duration
-	mine      int64
-	nulls     int64
-	done      chan struct{}
-}
-
-func newBlockMiner(ctx context.Context, t *testing.T, miner TestStorageNode, blocktime time.Duration) *blockMiner {
-	return &blockMiner{
-		ctx:       ctx,
-		t:         t,
-		miner:     miner,
-		blocktime: blocktime,
-		mine:      int64(1),
-		done:      make(chan struct{}),
-	}
-}
-
-func (bm *blockMiner) mineBlocks() {
-	time.Sleep(time.Second)
-	go func() {
-		defer close(bm.done)
-		for atomic.LoadInt64(&bm.mine) == 1 {
-			time.Sleep(bm.blocktime)
-			nulls := atomic.SwapInt64(&bm.nulls, 0)
-			if err := bm.miner.MineOne(bm.ctx, miner.MineReq{
-				InjectNulls: abi.ChainEpoch(nulls),
-				Done:        func(bool, error) {},
-			}); err != nil {
-				bm.t.Error(err)
-			}
-		}
-	}()
-}
-
-func (bm *blockMiner) stop() {
-	atomic.AddInt64(&bm.mine, -1)
-	fmt.Println("shutting down mining")
-	<-bm.done
-}
-
-func sendFunds(ctx context.Context, t *testing.T, sender TestNode, addr address.Address, amount abi.TokenAmount) {
-
-	senderAddr, err := sender.WalletDefaultAddress(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	msg := &types.Message{
-		From:  senderAddr,
-		To:    addr,
-		Value: amount,
-	}
-
-	sm, err := sender.MpoolPushMessage(ctx, msg, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	res, err := sender.StateWaitMsg(ctx, sm.Cid(), 1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if res.Receipt.ExitCode != 0 {
-		t.Fatal("did not successfully send money")
-	}
-}

api/test/test.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"testing"
 
+	"github.com/multiformats/go-multiaddr"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -14,10 +16,16 @@ import (
 
 type TestNode struct {
 	api.FullNode
+	// ListenAddr is the address on which an API server is listening, if an
+	// API server is created for this Node
+	ListenAddr multiaddr.Multiaddr
 }
 
 type TestStorageNode struct {
 	api.StorageMiner
+	// ListenAddr is the address on which an API server is listening, if an
+	// API server is created for this Node
+	ListenAddr multiaddr.Multiaddr
 
 	MineOne func(context.Context, miner.MineReq) error
 }
@@ -54,11 +62,13 @@ func TestApis(t *testing.T, b APIBuilder) {
 	t.Run("testMiningReal", ts.testMiningReal)
 }
 
-var oneMiner = []StorageMiner{{Full: 0, Preseal: PresealGenesis}}
+var OneMiner = []StorageMiner{{Full: 0, Preseal: PresealGenesis}}
 
 func (ts *testSuite) testVersion(t *testing.T) {
+	build.RunningNodeType = build.NodeFull
+
 	ctx := context.Background()
-	apis, _ := ts.makeNodes(t, 1, oneMiner)
+	apis, _ := ts.makeNodes(t, 1, OneMiner)
 	api := apis[0]
 
 	v, err := api.Version(ctx)
@@ -70,7 +80,7 @@ func (ts *testSuite) testVersion(t *testing.T) {
 
 func (ts *testSuite) testID(t *testing.T) {
 	ctx := context.Background()
-	apis, _ := ts.makeNodes(t, 1, oneMiner)
+	apis, _ := ts.makeNodes(t, 1, OneMiner)
 	api := apis[0]
 
 	id, err := api.ID(ctx)
@@ -82,7 +92,7 @@ func (ts *testSuite) testID(t *testing.T) {
 
 func (ts *testSuite) testConnectTwo(t *testing.T) {
 	ctx := context.Background()
-	apis, _ := ts.makeNodes(t, 2, oneMiner)
+	apis, _ := ts.makeNodes(t, 2, OneMiner)
 
 	p, err := apis[0].NetPeers(ctx)
 	if err != nil {

api/test/util.go

@@ -0,0 +1,86 @@
+package test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/filecoin-project/go-state-types/abi"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/miner"
+)
+
+func SendFunds(ctx context.Context, t *testing.T, sender TestNode, addr address.Address, amount abi.TokenAmount) {
+	senderAddr, err := sender.WalletDefaultAddress(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	msg := &types.Message{
+		From:  senderAddr,
+		To:    addr,
+		Value: amount,
+	}
+
+	sm, err := sender.MpoolPushMessage(ctx, msg, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	res, err := sender.StateWaitMsg(ctx, sm.Cid(), 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.Receipt.ExitCode != 0 {
+		t.Fatal("did not successfully send money")
+	}
+}
+
+func MineUntilBlock(ctx context.Context, t *testing.T, fn TestNode, sn TestStorageNode, cb func(abi.ChainEpoch)) {
+	for i := 0; i < 1000; i++ {
+		var success bool
+		var err error
+		var epoch abi.ChainEpoch
+		wait := make(chan struct{})
+		mineErr := sn.MineOne(ctx, miner.MineReq{
+			Done: func(win bool, ep abi.ChainEpoch, e error) {
+				success = win
+				err = e
+				epoch = ep
+				wait <- struct{}{}
+			},
+		})
+		if mineErr != nil {
+			t.Fatal(mineErr)
+		}
+		<-wait
+		if err != nil {
+			t.Fatal(err)
+		}
+		if success {
+			// Wait until it shows up on the given full nodes ChainHead
+			nloops := 50
+			for i := 0; i < nloops; i++ {
+				ts, err := fn.ChainHead(ctx)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if ts.Height() == epoch {
+					break
+				}
+				if i == nloops-1 {
+					t.Fatal("block never managed to sync to node")
+				}
+				time.Sleep(time.Millisecond * 10)
+			}
+
+			if cb != nil {
+				cb(epoch)
+			}
+			return
+		}
+		t.Log("did not mine block, trying again", i)
+	}
+	t.Fatal("failed to mine 1000 times in a row...")
+}

api/test/window_post.go

@@ -12,10 +12,10 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/sector-storage/mock"
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/extern/sector-storage/mock"
+	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
 	miner2 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
-	sealing "github.com/filecoin-project/storage-fsm"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
@@ -24,11 +24,16 @@ import (
 	"github.com/filecoin-project/lotus/node/impl"
 )
 
-func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) {
-	os.Setenv("BELLMAN_NO_GPU", "1")
+func init() {
+	err := os.Setenv("BELLMAN_NO_GPU", "1")
+	if err != nil {
+		panic(fmt.Sprintf("failed to set BELLMAN_NO_GPU env variable: %s", err))
+	}
+}
 
+func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) {
 	ctx := context.Background()
-	n, sn := b(t, 1, oneMiner)
+	n, sn := b(t, 1, OneMiner)
 	client := n[0].FullNode.(*impl.FullNodeAPI)
 	miner := sn[0]
 
@@ -48,7 +53,7 @@ func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSect
 		defer close(done)
 		for mine {
 			build.Clock.Sleep(blocktime)
-			if err := sn[0].MineOne(ctx, bminer.MineReq{Done: func(bool, error) {
+			if err := sn[0].MineOne(ctx, bminer.MineReq{Done: func(bool, abi.ChainEpoch, error) {
 
 			}}); err != nil {
 				t.Error(err)
@@ -110,10 +115,8 @@ func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n,
 }
 
 func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) {
-	os.Setenv("BELLMAN_NO_GPU", "1")
-
 	ctx := context.Background()
-	n, sn := b(t, 1, oneMiner)
+	n, sn := b(t, 1, OneMiner)
 	client := n[0].FullNode.(*impl.FullNodeAPI)
 	miner := sn[0]
 
@@ -213,6 +216,10 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 		sn, err := parts[0].Sectors.First()
 		require.NoError(t, err)
 
+		all, err := parts[0].Sectors.All(2)
+		require.NoError(t, err)
+		fmt.Println("the sectors", all)
+
 		s = abi.SectorID{
 			Miner:  abi.ActorID(mid),
 			Number: abi.SectorNumber(sn),

api/types.go

@@ -2,10 +2,14 @@ package api
 
 import (
 	"encoding/json"
+	"fmt"
+
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	datatransfer "github.com/filecoin-project/go-data-transfer"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
+	"github.com/ipfs/go-cid"
 
 	"github.com/libp2p/go-libp2p-core/peer"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
@@ -46,11 +50,12 @@ type PubsubScore struct {
 }
 
 type MinerInfo struct {
-	Owner                      address.Address // Must be an ID-address.
-	Worker                     address.Address // Must be an ID-address.
-	NewWorker                  address.Address // Must be an ID-address.
+	Owner                      address.Address   // Must be an ID-address.
+	Worker                     address.Address   // Must be an ID-address.
+	NewWorker                  address.Address   // Must be an ID-address.
+	ControlAddresses           []address.Address // Must be an ID-addresses.
 	WorkerChangeEpoch          abi.ChainEpoch
-	PeerId                     peer.ID
+	PeerId                     *peer.ID
 	Multiaddrs                 []abi.Multiaddrs
 	SealProofType              abi.RegisteredSealProof
 	SectorSize                 abi.SectorSize
@@ -58,12 +63,20 @@ type MinerInfo struct {
 }
 
 func NewApiMinerInfo(info *miner.MinerInfo) MinerInfo {
+	var pid *peer.ID
+	if peerID, err := peer.IDFromBytes(info.PeerId); err == nil {
+		pid = &peerID
+	}
+
 	mi := MinerInfo{
-		Owner:                      info.Owner,
-		Worker:                     info.Worker,
-		NewWorker:                  address.Undef,
-		WorkerChangeEpoch:          -1,
-		PeerId:                     peer.ID(info.PeerId),
+		Owner:            info.Owner,
+		Worker:           info.Worker,
+		ControlAddresses: info.ControlAddresses,
+
+		NewWorker:         address.Undef,
+		WorkerChangeEpoch: -1,
+
+		PeerId:                     pid,
 		Multiaddrs:                 info.Multiaddrs,
 		SealProofType:              info.SealProofType,
 		SectorSize:                 info.SectorSize,
@@ -83,7 +96,8 @@ type MessageSendSpec struct {
 }
 
 var DefaultMessageSendSpec = MessageSendSpec{
-	MaxFee: big.Zero(),
+	// MaxFee of 0.1FIL
+	MaxFee: abi.NewTokenAmount(int64(build.FilecoinPrecision) / 10),
 }
 
 func (ms *MessageSendSpec) Get() MessageSendSpec {
@@ -93,3 +107,47 @@ func (ms *MessageSendSpec) Get() MessageSendSpec {
 
 	return *ms
 }
+
+type DataTransferChannel struct {
+	TransferID  datatransfer.TransferID
+	Status      datatransfer.Status
+	BaseCID     cid.Cid
+	IsInitiator bool
+	IsSender    bool
+	Voucher     string
+	Message     string
+	OtherPeer   peer.ID
+	Transferred uint64
+}
+
+// NewDataTransferChannel constructs an API DataTransferChannel type from full channel state snapshot and a host id
+func NewDataTransferChannel(hostID peer.ID, channelState datatransfer.ChannelState) DataTransferChannel {
+	channel := DataTransferChannel{
+		TransferID: channelState.TransferID(),
+		Status:     channelState.Status(),
+		BaseCID:    channelState.BaseCID(),
+		IsSender:   channelState.Sender() == hostID,
+		Message:    channelState.Message(),
+	}
+	stringer, ok := channelState.Voucher().(fmt.Stringer)
+	if ok {
+		channel.Voucher = stringer.String()
+	} else {
+		voucherJSON, err := json.Marshal(channelState.Voucher())
+		if err != nil {
+			channel.Voucher = fmt.Errorf("Voucher Serialization: %w", err).Error()
+		} else {
+			channel.Voucher = string(voucherJSON)
+		}
+	}
+	if channel.IsSender {
+		channel.IsInitiator = !channelState.IsPull()
+		channel.Transferred = channelState.Sent()
+		channel.OtherPeer = channelState.Recipient()
+	} else {
+		channel.IsInitiator = channelState.IsPull()
+		channel.Transferred = channelState.Received()
+		channel.OtherPeer = channelState.Sender()
+	}
+	return channel
+}

api/utils.go

@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/specs-actors/actors/crypto"
+	"github.com/filecoin-project/go-state-types/crypto"
 )
 
 type SignFunc = func(context.Context, []byte) (*crypto.Signature, error)

build/bootstrap/bootstrappers.pi

@@ -1,4 +1,6 @@
-/dns4/bootstrap-0.calibration.fildev.network/tcp/1347/p2p/12D3KooWDHzHtmsH9BMf9is7hCzJnd1jtVb5rdeAmfm9CCoZFqeY
-/dns4/bootstrap-1.calibration.fildev.network/tcp/1347/p2p/12D3KooWHyTNdCjTviTCMpFiyYVb3TamxuLmM3wEcrhH5T4M9Q3J
-/dns4/bootstrap-2.calibration.fildev.network/tcp/1347/p2p/12D3KooWBM8Zdq288tyYF5yT1j2ym9ynApddGJFjYnXNiFiCbdXp
-/dns4/bootstrap-3.calibration.fildev.network/tcp/1347/p2p/12D3KooWKoegjZRfgY8Zp6QMqr4UtuJHzTKeWVmrniFNXeikzvGT
+/dns4/bootstrap-0.testnet.fildev.network/tcp/1347/p2p/12D3KooWJTUBUjtzWJGWU1XSiY21CwmHaCNLNYn2E7jqHEHyZaP7
+/dns4/bootstrap-1.testnet.fildev.network/tcp/1347/p2p/12D3KooW9yeKXha4hdrJKq74zEo99T8DhriQdWNoojWnnQbsgB3v
+/dns4/bootstrap-2.testnet.fildev.network/tcp/1347/p2p/12D3KooWCrx8yVG9U9Kf7w8KLN3Edkj5ZKDhgCaeMqQbcQUoB6CT
+/dns4/bootstrap-4.testnet.fildev.network/tcp/1347/p2p/12D3KooWPkL9LrKRQgHtq7kn9ecNhGU9QaziG8R5tX8v9v7t3h34
+/dns4/bootstrap-3.testnet.fildev.network/tcp/1347/p2p/12D3KooWKYSsbpgZ3HAjax5M1BXCwXLa6gVkUARciz7uN3FNtr7T
+/dns4/bootstrap-5.testnet.fildev.network/tcp/1347/p2p/12D3KooWQYzqnLASJAabyMpPb1GcWZvNSe7JDcRuhdRqonFoiK9W

build/drand.go

@@ -1,20 +1,32 @@
 package build
 
-import "github.com/filecoin-project/lotus/node/modules/dtypes"
+import (
+	"sort"
 
-var DrandNetwork = DrandMainnet
-
-func DrandConfig() dtypes.DrandConfig {
-	return DrandConfigs[DrandNetwork]
-}
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+)
 
 type DrandEnum int
 
+func DrandConfigSchedule() dtypes.DrandSchedule {
+	out := dtypes.DrandSchedule{}
+	for start, config := range DrandSchedule {
+		out = append(out, dtypes.DrandPoint{Start: start, Config: DrandConfigs[config]})
+	}
+
+	sort.Slice(out, func(i, j int) bool {
+		return out[i].Start < out[j].Start
+	})
+
+	return out
+}
+
 const (
 	DrandMainnet DrandEnum = iota + 1
 	DrandTestnet
 	DrandDevnet
 	DrandLocalnet
+	DrandIncentinet
 )
 
 var DrandConfigs = map[DrandEnum]dtypes.DrandConfig{
@@ -55,4 +67,17 @@ var DrandConfigs = map[DrandEnum]dtypes.DrandConfig{
 		},
 		ChainInfoJSON: `{"public_key":"8cda589f88914aa728fd183f383980b35789ce81b274e5daee1f338b77d02566ef4d3fb0098af1f844f10f9c803c1827","period":25,"genesis_time":1595348225,"hash":"e73b7dc3c4f6a236378220c0dd6aa110eb16eed26c11259606e07ee122838d4f","groupHash":"567d4785122a5a3e75a9bc9911d7ea807dd85ff76b78dc4ff06b075712898607"}`,
 	},
+	DrandIncentinet: {
+		Servers: []string{
+			"https://pl-eu.incentinet.drand.sh",
+			"https://pl-us.incentinet.drand.sh",
+			"https://pl-sin.incentinet.drand.sh",
+		},
+		Relays: []string{
+			"/dnsaddr/pl-eu.incentinet.drand.sh/",
+			"/dnsaddr/pl-us.incentinet.drand.sh/",
+			"/dnsaddr/pl-sin.incentinet.drand.sh/",
+		},
+		ChainInfoJSON: `{"public_key":"8cad0c72c606ab27d36ee06de1d5b2db1faf92e447025ca37575ab3a8aac2eaae83192f846fc9e158bc738423753d000","period":30,"genesis_time":1595873820,"hash":"80c8b872c714f4c00fdd3daa465d5514049f457f01f85a4caf68cdcd394ba039","groupHash":"d9406aaed487f7af71851b4399448e311f2328923d454e971536c05398ce2d9b"}`,
+	},
 }

build/params_2k.go

@@ -3,13 +3,22 @@
 package build
 
 import (
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
 )
 
+const UpgradeBreezeHeight = -1
+const BreezeGasTampingDuration = 0
+
+const UpgradeSmokeHeight = -1
+
+var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
+	0: DrandMainnet,
+}
+
 func init() {
 	power.ConsensusMinerMinPower = big.NewInt(2048)
 	miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{
@@ -20,7 +29,7 @@ func init() {
 	BuildType |= Build2k
 }
 
-const BlockDelaySecs = uint64(4)
+const BlockDelaySecs = uint64(30)
 
 const PropagationDelaySecs = uint64(1)
 

build/params_shared_funcs.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/libp2p/go-libp2p-core/protocol"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
@@ -36,3 +36,11 @@ func MessagesTopic(netName dtypes.NetworkName) string { return "/fil/msgs/" + st
 func DhtProtocolName(netName dtypes.NetworkName) protocol.ID {
 	return protocol.ID("/fil/kad/" + string(netName))
 }
+
+func UseNewestNetwork() bool {
+	// TODO: Put these in a container we can iterate over
+	if UpgradeBreezeHeight <= 0 && UpgradeSmokeHeight <= 0 {
+		return true
+	}
+	return false
+}

build/params_shared_vals.go

@@ -5,7 +5,9 @@ package build
 import (
 	"math/big"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/network"
+
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 )
@@ -20,6 +22,7 @@ const UnixfsLinksPerLevel = 1024
 // Consensus / Network
 
 const AllowableClockDriftSecs = uint64(1)
+const NewestNetworkVersion = network.Version2
 
 // Epochs
 const ForkLengthThreshold = Finality
@@ -89,11 +92,14 @@ const VerifSigCacheSize = 32000
 
 // TODO: If this is gonna stay, it should move to specs-actors
 const BlockMessageLimit = 10000
+
 const BlockGasLimit = 10_000_000_000
 const BlockGasTarget = BlockGasLimit / 2
 const BaseFeeMaxChangeDenom = 8 // 12.5%
 const InitialBaseFee = 100e6
 const MinimumBaseFee = 100
+const PackingEfficiencyNum = 4
+const PackingEfficiencyDenom = 5
 
 // Actor consts
 // TODO: Pull from actors when its made not private

build/params_testground.go

@@ -10,7 +10,8 @@ package build
 import (
 	"math/big"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/network"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 )
@@ -66,4 +67,18 @@ var (
 	// Actor consts
 	// TODO: Pull from actors when its made not private
 	MinDealDuration = abi.ChainEpoch(180 * builtin.EpochsInDay)
+
+	PackingEfficiencyNum   int64 = 4
+	PackingEfficiencyDenom int64 = 5
+
+	UpgradeBreezeHeight      abi.ChainEpoch = -1
+	BreezeGasTampingDuration abi.ChainEpoch = 0
+
+	UpgradeSmokeHeight abi.ChainEpoch = -1
+
+	DrandSchedule = map[abi.ChainEpoch]DrandEnum{
+		0: DrandMainnet,
+	}
+
+	NewestNetworkVersion = network.Version2
 )

build/params_testnet.go

@@ -5,19 +5,28 @@
 package build
 
 import (
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 )
 
+var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
+	0:                  DrandIncentinet,
+	UpgradeSmokeHeight: DrandMainnet,
+}
+
+const UpgradeBreezeHeight = 41280
+const BreezeGasTampingDuration = 120
+
+const UpgradeSmokeHeight = 51000
+
 func init() {
-	power.ConsensusMinerMinPower = big.NewInt(512 << 30)
+	power.ConsensusMinerMinPower = big.NewInt(10 << 40)
 	miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{
-		abi.RegisteredSealProof_StackedDrg512MiBV1: {},
-		abi.RegisteredSealProof_StackedDrg32GiBV1:  {},
-		abi.RegisteredSealProof_StackedDrg64GiBV1:  {},
+		abi.RegisteredSealProof_StackedDrg32GiBV1: {},
+		abi.RegisteredSealProof_StackedDrg64GiBV1: {},
 	}
 }
 

build/version.go

@@ -1,6 +1,10 @@
 package build
 
-import "fmt"
+import (
+	"fmt"
+
+	"golang.org/x/xerrors"
+)
 
 var CurrentCommit string
 var BuildType int
@@ -25,7 +29,7 @@ func buildType() string {
 }
 
 // BuildVersion is the local build version, set by build system
-const BuildVersion = "0.4.4"
+const BuildVersion = "0.6.2"
 
 func UserVersion() string {
 	return BuildVersion + buildType() + CurrentCommit
@@ -52,8 +56,37 @@ func (ve Version) EqMajorMinor(v2 Version) bool {
 	return ve&minorMask == v2&minorMask
 }
 
-// APIVersion is a semver version of the rpc api exposed
-var APIVersion Version = newVer(0, 10, 0)
+type NodeType int
+
+const (
+	NodeUnknown NodeType = iota
+
+	NodeFull
+	NodeMiner
+	NodeWorker
+)
+
+var RunningNodeType NodeType
+
+func VersionForType(nodeType NodeType) (Version, error) {
+	switch nodeType {
+	case NodeFull:
+		return FullAPIVersion, nil
+	case NodeMiner:
+		return MinerAPIVersion, nil
+	case NodeWorker:
+		return WorkerAPIVersion, nil
+	default:
+		return Version(0), xerrors.Errorf("unknown node type %d", nodeType)
+	}
+}
+
+// semver versions of the rpc api exposed
+var (
+	FullAPIVersion   = newVer(0, 14, 0)
+	MinerAPIVersion  = newVer(0, 14, 0)
+	WorkerAPIVersion = newVer(0, 14, 0)
+)
 
 //nolint:varcheck,deadcode
 const (

chain/actors/aerrors/error.go

@@ -3,7 +3,7 @@ package aerrors
 import (
 	"fmt"
 
-	"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
+	"github.com/filecoin-project/go-state-types/exitcode"
 	"golang.org/x/xerrors"
 )
 

chain/actors/aerrors/error_test.go

@@ -3,8 +3,8 @@ package aerrors_test
 import (
 	"testing"
 
+	"github.com/filecoin-project/go-state-types/exitcode"
 	. "github.com/filecoin-project/lotus/chain/actors/aerrors"
-	"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
 
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/xerrors"

chain/actors/aerrors/wrap.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
+	"github.com/filecoin-project/go-state-types/exitcode"
 	cbor "github.com/ipfs/go-ipld-cbor"
 	"golang.org/x/xerrors"
 )

chain/badtscache.go

@@ -56,6 +56,10 @@ func (bts *BadBlockCache) Add(c cid.Cid, bbr BadBlockReason) {
 	bts.badBlocks.Add(c, bbr)
 }
 
+func (bts *BadBlockCache) Remove(c cid.Cid) {
+	bts.badBlocks.Remove(c)
+}
+
 func (bts *BadBlockCache) Has(c cid.Cid) (BadBlockReason, bool) {
 	rval, ok := bts.badBlocks.Get(c)
 	if !ok {

chain/beacon/beacon.go

@@ -3,7 +3,7 @@ package beacon
 import (
 	"context"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
 	logging "github.com/ipfs/go-log"
 	"golang.org/x/xerrors"
 
@@ -18,6 +18,23 @@ type Response struct {
 	Err   error
 }
 
+type Schedule []BeaconPoint
+
+func (bs Schedule) BeaconForEpoch(e abi.ChainEpoch) RandomBeacon {
+	for i := len(bs) - 1; i >= 0; i-- {
+		bp := bs[i]
+		if e >= bp.Start {
+			return bp.Beacon
+		}
+	}
+	return bs[0].Beacon
+}
+
+type BeaconPoint struct {
+	Start  abi.ChainEpoch
+	Beacon RandomBeacon
+}
+
 // RandomBeacon represents a system that provides randomness to Lotus.
 // Other components interrogate the RandomBeacon to acquire randomness that's
 // valid for a specific chain epoch. Also to verify beacon entries that have
@@ -25,11 +42,30 @@ type Response struct {
 type RandomBeacon interface {
 	Entry(context.Context, uint64) <-chan Response
 	VerifyEntry(types.BeaconEntry, types.BeaconEntry) error
-	MaxBeaconRoundForEpoch(abi.ChainEpoch, types.BeaconEntry) uint64
+	MaxBeaconRoundForEpoch(abi.ChainEpoch) uint64
 }
 
-func ValidateBlockValues(b RandomBeacon, h *types.BlockHeader, prevEntry types.BeaconEntry) error {
-	maxRound := b.MaxBeaconRoundForEpoch(h.Height, prevEntry)
+func ValidateBlockValues(bSchedule Schedule, h *types.BlockHeader, parentEpoch abi.ChainEpoch,
+	prevEntry types.BeaconEntry) error {
+	{
+		parentBeacon := bSchedule.BeaconForEpoch(parentEpoch)
+		currBeacon := bSchedule.BeaconForEpoch(h.Height)
+		if parentBeacon != currBeacon {
+			if len(h.BeaconEntries) != 2 {
+				return xerrors.Errorf("expected two beacon entries at beacon fork, got %d", len(h.BeaconEntries))
+			}
+			err := currBeacon.VerifyEntry(h.BeaconEntries[1], h.BeaconEntries[0])
+			if err != nil {
+				return xerrors.Errorf("beacon at fork point invalid: (%v, %v): %w",
+					h.BeaconEntries[1], h.BeaconEntries[0], err)
+			}
+			return nil
+		}
+	}
+
+	// TODO: fork logic
+	b := bSchedule.BeaconForEpoch(h.Height)
+	maxRound := b.MaxBeaconRoundForEpoch(h.Height)
 	if maxRound == prevEntry.Round {
 		if len(h.BeaconEntries) != 0 {
 			return xerrors.Errorf("expected not to have any beacon entries in this block, got %d", len(h.BeaconEntries))
@@ -37,6 +73,10 @@ func ValidateBlockValues(b RandomBeacon, h *types.BlockHeader, prevEntry types.B
 		return nil
 	}
 
+	if len(h.BeaconEntries) == 0 {
+		return xerrors.Errorf("expected to have beacon entries in this block, but didn't find any")
+	}
+
 	last := h.BeaconEntries[len(h.BeaconEntries)-1]
 	if last.Round != maxRound {
 		return xerrors.Errorf("expected final beacon entry in block to be at round %d, got %d", maxRound, last.Round)
@@ -52,10 +92,35 @@ func ValidateBlockValues(b RandomBeacon, h *types.BlockHeader, prevEntry types.B
 	return nil
 }
 
-func BeaconEntriesForBlock(ctx context.Context, beacon RandomBeacon, round abi.ChainEpoch, prev types.BeaconEntry) ([]types.BeaconEntry, error) {
+func BeaconEntriesForBlock(ctx context.Context, bSchedule Schedule, epoch abi.ChainEpoch, parentEpoch abi.ChainEpoch, prev types.BeaconEntry) ([]types.BeaconEntry, error) {
+	{
+		parentBeacon := bSchedule.BeaconForEpoch(parentEpoch)
+		currBeacon := bSchedule.BeaconForEpoch(epoch)
+		if parentBeacon != currBeacon {
+			// Fork logic
+			round := currBeacon.MaxBeaconRoundForEpoch(epoch)
+			out := make([]types.BeaconEntry, 2)
+			rch := currBeacon.Entry(ctx, round-1)
+			res := <-rch
+			if res.Err != nil {
+				return nil, xerrors.Errorf("getting entry %d returned error: %w", round-1, res.Err)
+			}
+			out[0] = res.Entry
+			rch = currBeacon.Entry(ctx, round)
+			res = <-rch
+			if res.Err != nil {
+				return nil, xerrors.Errorf("getting entry %d returned error: %w", round, res.Err)
+			}
+			out[1] = res.Entry
+			return out, nil
+		}
+	}
+
+	beacon := bSchedule.BeaconForEpoch(epoch)
+
 	start := build.Clock.Now()
 
-	maxRound := beacon.MaxBeaconRoundForEpoch(round, prev)
+	maxRound := beacon.MaxBeaconRoundForEpoch(epoch)
 	if maxRound == prev.Round {
 		return nil, nil
 	}
@@ -78,7 +143,7 @@ func BeaconEntriesForBlock(ctx context.Context, beacon RandomBeacon, round abi.C
 			out = append(out, resp.Entry)
 			cur = resp.Entry.Round - 1
 		case <-ctx.Done():
-			return nil, xerrors.Errorf("context timed out waiting on beacon entry to come back for round %d: %w", round, ctx.Err())
+			return nil, xerrors.Errorf("context timed out waiting on beacon entry to come back for epoch %d: %w", epoch, ctx.Err())
 		}
 	}
 

chain/beacon/drand/drand.go

@@ -19,7 +19,7 @@ import (
 	logging "github.com/ipfs/go-log"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
 
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/beacon"
@@ -187,7 +187,7 @@ func (db *DrandBeacon) VerifyEntry(curr types.BeaconEntry, prev types.BeaconEntr
 	return err
 }
 
-func (db *DrandBeacon) MaxBeaconRoundForEpoch(filEpoch abi.ChainEpoch, prevEntry types.BeaconEntry) uint64 {
+func (db *DrandBeacon) MaxBeaconRoundForEpoch(filEpoch abi.ChainEpoch) uint64 {
 	// TODO: sometimes the genesis time for filecoin is zero and this goes negative
 	latestTs := ((uint64(filEpoch) * db.filRoundTime) + db.filGenTime) - db.filRoundTime
 	dround := (latestTs - db.drandGenTime) / uint64(db.interval.Seconds())

chain/beacon/drand/drand_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestPrintGroupInfo(t *testing.T) {
-	server := build.DrandConfig().Servers[0]
+	server := build.DrandConfigs[build.DrandIncentinet].Servers[0]
 	c, err := hclient.New(server, nil, nil)
 	assert.NoError(t, err)
 	cg := c.(interface {

chain/beacon/mock.go

@@ -6,8 +6,8 @@ import (
 	"encoding/binary"
 	"time"
 
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/minio/blake2b-simd"
 	"golang.org/x/xerrors"
 )
@@ -53,11 +53,7 @@ func (mb *mockBeacon) VerifyEntry(from types.BeaconEntry, to types.BeaconEntry)
 	return nil
 }
 
-func (mb *mockBeacon) IsEntryForEpoch(e types.BeaconEntry, epoch abi.ChainEpoch, nulls int) (bool, error) {
-	return int64(e.Round) <= int64(epoch) && int64(epoch)-int64(nulls) >= int64(e.Round), nil
-}
-
-func (mb *mockBeacon) MaxBeaconRoundForEpoch(epoch abi.ChainEpoch, prevEntry types.BeaconEntry) uint64 {
+func (mb *mockBeacon) MaxBeaconRoundForEpoch(epoch abi.ChainEpoch) uint64 {
 	return uint64(epoch)
 }
 

chain/block_receipt_tracker.go

@@ -7,8 +7,8 @@ import (
 
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/hashicorp/golang-lru"
-	peer "github.com/libp2p/go-libp2p-core/peer"
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/libp2p/go-libp2p-core/peer"
 )
 
 type blockReceiptTracker struct {

chain/checkpoint.go

@@ -0,0 +1,81 @@
+package chain
+
+import (
+	"encoding/json"
+
+	"github.com/filecoin-project/lotus/chain/types"
+
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+	"github.com/ipfs/go-datastore"
+	"golang.org/x/xerrors"
+)
+
+var CheckpointKey = datastore.NewKey("/chain/checks")
+
+func loadCheckpoint(ds dtypes.MetadataDS) (types.TipSetKey, error) {
+	haveChks, err := ds.Has(CheckpointKey)
+	if err != nil {
+		return types.EmptyTSK, err
+	}
+
+	if !haveChks {
+		return types.EmptyTSK, nil
+	}
+
+	tskBytes, err := ds.Get(CheckpointKey)
+	if err != nil {
+		return types.EmptyTSK, err
+	}
+
+	var tsk types.TipSetKey
+	err = json.Unmarshal(tskBytes, &tsk)
+	if err != nil {
+		return types.EmptyTSK, err
+	}
+
+	return tsk, err
+}
+
+func (syncer *Syncer) SetCheckpoint(tsk types.TipSetKey) error {
+	if tsk == types.EmptyTSK {
+		return xerrors.Errorf("called with empty tsk")
+	}
+
+	syncer.checkptLk.Lock()
+	defer syncer.checkptLk.Unlock()
+
+	ts, err := syncer.ChainStore().LoadTipSet(tsk)
+	if err != nil {
+		return xerrors.Errorf("cannot find tipset: %w", err)
+	}
+
+	hts := syncer.ChainStore().GetHeaviestTipSet()
+	anc, err := syncer.ChainStore().IsAncestorOf(ts, hts)
+	if err != nil {
+		return xerrors.Errorf("cannot determine whether checkpoint tipset is in main-chain: %w", err)
+	}
+
+	if !hts.Equals(ts) && !anc {
+		return xerrors.Errorf("cannot mark tipset as checkpoint, since it isn't in the main-chain: %w", err)
+	}
+
+	tskBytes, err := json.Marshal(tsk)
+	if err != nil {
+		return err
+	}
+
+	err = syncer.ds.Put(CheckpointKey, tskBytes)
+	if err != nil {
+		return err
+	}
+
+	syncer.checkpt = tsk
+
+	return nil
+}
+
+func (syncer *Syncer) GetCheckpoint() types.TipSetKey {
+	syncer.checkptLk.Lock()
+	defer syncer.checkptLk.Unlock()
+	return syncer.checkpt
+}

chain/events/events.go

@@ -5,7 +5,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log/v2"
 	"golang.org/x/xerrors"
@@ -35,6 +35,7 @@ type eventAPI interface {
 	ChainNotify(context.Context) (<-chan []*api.HeadChange, error)
 	ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error)
 	ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)
+	ChainHead(context.Context) (*types.TipSet, error)
 	StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)
 	ChainGetTipSet(context.Context, types.TipSetKey) (*types.TipSet, error)
 
@@ -57,7 +58,7 @@ type Events struct {
 func NewEvents(ctx context.Context, api eventAPI) *Events {
 	gcConfidence := 2 * build.ForkLengthThreshold
 
-	tsc := newTSCache(gcConfidence, api.ChainGetTipSetByHeight)
+	tsc := newTSCache(gcConfidence, api)
 
 	e := &Events{
 		api: api,

chain/events/events_called.go

@@ -5,7 +5,7 @@ import (
 	"math"
 	"sync"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
 
@@ -307,7 +307,10 @@ func (e *hcEvents) onHeadChanged(check CheckFunc, hnd EventHandler, rev RevertHa
 	defer e.lk.Unlock()
 
 	// Check if the event has already occurred
-	ts := e.tsc.best()
+	ts, err := e.tsc.best()
+	if err != nil {
+		return 0, xerrors.Errorf("error getting best tipset: %w", err)
+	}
 	done, more, err := check(ts)
 	if err != nil {
 		return 0, xerrors.Errorf("called check error (h: %d): %w", ts.Height(), err)

chain/events/events_height.go

@@ -4,8 +4,9 @@ import (
 	"context"
 	"sync"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
 	"go.opencensus.io/trace"
+	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/chain/types"
 )
@@ -152,8 +153,12 @@ func (e *heightEvents) ChainAt(hnd HeightHandler, rev RevertHandler, confidence
 
 	e.lk.Lock() // Tricky locking, check your locks if you modify this function!
 
-	bestH := e.tsc.best().Height()
+	best, err := e.tsc.best()
+	if err != nil {
+		return xerrors.Errorf("error getting best tipset: %w", err)
+	}
 
+	bestH := best.Height()
 	if bestH >= h+abi.ChainEpoch(confidence) {
 		ts, err := e.tsc.getNonNull(h)
 		if err != nil {
@@ -172,7 +177,11 @@ func (e *heightEvents) ChainAt(hnd HeightHandler, rev RevertHandler, confidence
 		}
 
 		e.lk.Lock()
-		bestH = e.tsc.best().Height()
+		best, err = e.tsc.best()
+		if err != nil {
+			return xerrors.Errorf("error getting best tipset: %w", err)
+		}
+		bestH = best.Height()
 	}
 
 	defer e.lk.Unlock()

chain/events/events_test.go

@@ -11,8 +11,8 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/crypto"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/crypto"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
@@ -46,6 +46,10 @@ type fakeCS struct {
 	sub func(rev, app []*types.TipSet)
 }
 
+func (fcs *fakeCS) ChainHead(ctx context.Context) (*types.TipSet, error) {
+	panic("implement me")
+}
+
 func (fcs *fakeCS) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) {
 	return fcs.tipsets[key], nil
 }
@@ -110,7 +114,11 @@ func (fcs *fakeCS) makeTs(t *testing.T, parents []cid.Cid, h abi.ChainEpoch, msg
 
 func (fcs *fakeCS) ChainNotify(context.Context) (<-chan []*api.HeadChange, error) {
 	out := make(chan []*api.HeadChange, 1)
-	out <- []*api.HeadChange{{Type: store.HCCurrent, Val: fcs.tsc.best()}}
+	best, err := fcs.tsc.best()
+	if err != nil {
+		return nil, err
+	}
+	out <- []*api.HeadChange{{Type: store.HCCurrent, Val: best}}
 
 	fcs.sub = func(rev, app []*types.TipSet) {
 		notif := make([]*api.HeadChange, len(rev)+len(app))
@@ -174,7 +182,8 @@ func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { /
 
 	var revs []*types.TipSet
 	for i := 0; i < rev; i++ {
-		ts := fcs.tsc.best()
+		ts, err := fcs.tsc.best()
+		require.NoError(fcs.t, err)
 
 		if _, ok := nullm[int(ts.Height())]; !ok {
 			revs = append(revs, ts)
@@ -196,7 +205,9 @@ func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { /
 			continue
 		}
 
-		ts := fcs.makeTs(fcs.t, fcs.tsc.best().Key().Cids(), fcs.h, mc)
+		best, err := fcs.tsc.best()
+		require.NoError(fcs.t, err)
+		ts := fcs.makeTs(fcs.t, best.Key().Cids(), fcs.h, mc)
 		require.NoError(fcs.t, fcs.tsc.add(ts))
 
 		if hasMsgs {

chain/events/state/diff_adt.go

@@ -2,6 +2,7 @@ package state
 
 import (
 	"bytes"
+
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
 	typegen "github.com/whyrusleeping/cbor-gen"
 )

chain/events/state/predicates.go

@@ -3,9 +3,10 @@ package state
 import (
 	"bytes"
 	"context"
+
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	init_ "github.com/filecoin-project/specs-actors/actors/builtin/init"
 	"github.com/filecoin-project/specs-actors/actors/builtin/market"

chain/events/state/predicates_test.go

@@ -13,11 +13,11 @@ import (
 	cbornode "github.com/ipfs/go-ipld-cbor"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/specs-actors/actors/builtin/market"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
-	"github.com/filecoin-project/specs-actors/actors/crypto"
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
 	tutils "github.com/filecoin-project/specs-actors/support/testing"
 
@@ -114,10 +114,10 @@ func TestMarketPredicates(t *testing.T) {
 	}
 
 	oldBalances := map[address.Address]balance{
-		tutils.NewIDAddr(t, 1): balance{abi.NewTokenAmount(1000), abi.NewTokenAmount(1000)},
-		tutils.NewIDAddr(t, 2): balance{abi.NewTokenAmount(2000), abi.NewTokenAmount(500)},
-		tutils.NewIDAddr(t, 3): balance{abi.NewTokenAmount(3000), abi.NewTokenAmount(2000)},
-		tutils.NewIDAddr(t, 5): balance{abi.NewTokenAmount(3000), abi.NewTokenAmount(1000)},
+		tutils.NewIDAddr(t, 1): {abi.NewTokenAmount(1000), abi.NewTokenAmount(1000)},
+		tutils.NewIDAddr(t, 2): {abi.NewTokenAmount(2000), abi.NewTokenAmount(500)},
+		tutils.NewIDAddr(t, 3): {abi.NewTokenAmount(3000), abi.NewTokenAmount(2000)},
+		tutils.NewIDAddr(t, 5): {abi.NewTokenAmount(3000), abi.NewTokenAmount(1000)},
 	}
 
 	oldStateC := createMarketState(ctx, t, store, oldDeals, oldProps, oldBalances)
@@ -162,10 +162,10 @@ func TestMarketPredicates(t *testing.T) {
 		// NB: DealProposals cannot be modified, so don't test that case.
 	}
 	newBalances := map[address.Address]balance{
-		tutils.NewIDAddr(t, 1): balance{abi.NewTokenAmount(3000), abi.NewTokenAmount(0)},
-		tutils.NewIDAddr(t, 2): balance{abi.NewTokenAmount(2000), abi.NewTokenAmount(500)},
-		tutils.NewIDAddr(t, 4): balance{abi.NewTokenAmount(5000), abi.NewTokenAmount(0)},
-		tutils.NewIDAddr(t, 5): balance{abi.NewTokenAmount(1000), abi.NewTokenAmount(3000)},
+		tutils.NewIDAddr(t, 1): {abi.NewTokenAmount(3000), abi.NewTokenAmount(0)},
+		tutils.NewIDAddr(t, 2): {abi.NewTokenAmount(2000), abi.NewTokenAmount(500)},
+		tutils.NewIDAddr(t, 4): {abi.NewTokenAmount(5000), abi.NewTokenAmount(0)},
+		tutils.NewIDAddr(t, 5): {abi.NewTokenAmount(1000), abi.NewTokenAmount(3000)},
 	}
 
 	newStateC := createMarketState(ctx, t, store, newDeals, newProps, newBalances)
@@ -505,6 +505,7 @@ func createBalanceTable(ctx context.Context, t *testing.T, store adt.Store, bala
 	lockedMapRootCid, err := lockedMapRoot.Root()
 	require.NoError(t, err)
 	lockedRoot, err := adt.AsBalanceTable(store, lockedMapRootCid)
+	require.NoError(t, err)
 
 	for addr, balance := range balances {
 		err := escrowRoot.Add(addr, big.Add(balance.available, balance.locked))
@@ -542,6 +543,7 @@ func createEmptyMinerState(ctx context.Context, t *testing.T, store adt.Store, o
 
 	emptyVestingFunds := miner.ConstructVestingFunds()
 	emptyVestingFundsCid, err := store.Put(store.Context(), emptyVestingFunds)
+	require.NoError(t, err)
 
 	emptyDeadlines := miner.ConstructDeadlines(emptyDeadline)
 	emptyDeadlinesCid, err := store.Put(store.Context(), emptyDeadlines)

chain/events/tscache.go

@@ -3,13 +3,16 @@ package events
 import (
 	"context"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
-type tsByHFunc func(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)
+type tsCacheAPI interface {
+	ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)
+	ChainHead(context.Context) (*types.TipSet, error)
+}
 
 // tipSetCache implements a simple ring-buffer cache to keep track of recent
 // tipsets
@@ -18,10 +21,10 @@ type tipSetCache struct {
 	start int
 	len   int
 
-	storage tsByHFunc
+	storage tsCacheAPI
 }
 
-func newTSCache(cap abi.ChainEpoch, storage tsByHFunc) *tipSetCache {
+func newTSCache(cap abi.ChainEpoch, storage tsCacheAPI) *tipSetCache {
 	return &tipSetCache{
 		cache: make([]*types.TipSet, cap),
 		start: 0,
@@ -94,7 +97,7 @@ func (tsc *tipSetCache) getNonNull(height abi.ChainEpoch) (*types.TipSet, error)
 func (tsc *tipSetCache) get(height abi.ChainEpoch) (*types.TipSet, error) {
 	if tsc.len == 0 {
 		log.Warnf("tipSetCache.get: cache is empty, requesting from storage (h=%d)", height)
-		return tsc.storage(context.TODO(), height, types.EmptyTSK)
+		return tsc.storage.ChainGetTipSetByHeight(context.TODO(), height, types.EmptyTSK)
 	}
 
 	headH := tsc.cache[tsc.start].Height()
@@ -114,14 +117,18 @@ func (tsc *tipSetCache) get(height abi.ChainEpoch) (*types.TipSet, error) {
 
 	if height < tail.Height() {
 		log.Warnf("tipSetCache.get: requested tipset not in cache, requesting from storage (h=%d; tail=%d)", height, tail.Height())
-		return tsc.storage(context.TODO(), height, tail.Key())
+		return tsc.storage.ChainGetTipSetByHeight(context.TODO(), height, tail.Key())
 	}
 
 	return tsc.cache[normalModulo(tsc.start-int(headH-height), clen)], nil
 }
 
-func (tsc *tipSetCache) best() *types.TipSet {
-	return tsc.cache[tsc.start]
+func (tsc *tipSetCache) best() (*types.TipSet, error) {
+	best := tsc.cache[tsc.start]
+	if best == nil {
+		return tsc.storage.ChainHead(context.TODO())
+	}
+	return best, nil
 }
 
 func normalModulo(n, m int) int {

chain/events/tscache_test.go

@@ -4,8 +4,8 @@ import (
 	"context"
 	"testing"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/crypto"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/go-address"
@@ -13,10 +13,7 @@ import (
 )
 
 func TestTsCache(t *testing.T) {
-	tsc := newTSCache(50, func(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) {
-		t.Fatal("storage call")
-		return &types.TipSet{}, nil
-	})
+	tsc := newTSCache(50, &tsCacheAPIFailOnStorageCall{t: t})
 
 	h := abi.ChainEpoch(75)
 
@@ -43,7 +40,12 @@ func TestTsCache(t *testing.T) {
 
 	for i := 0; i < 9000; i++ {
 		if i%90 > 60 {
-			if err := tsc.revert(tsc.best()); err != nil {
+			best, err := tsc.best()
+			if err != nil {
+				t.Fatal(err, "; i:", i)
+				return
+			}
+			if err := tsc.revert(best); err != nil {
 				t.Fatal(err, "; i:", i)
 				return
 			}
@@ -55,11 +57,21 @@ func TestTsCache(t *testing.T) {
 
 }
 
+type tsCacheAPIFailOnStorageCall struct {
+	t *testing.T
+}
+
+func (tc *tsCacheAPIFailOnStorageCall) ChainGetTipSetByHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) {
+	tc.t.Fatal("storage call")
+	return &types.TipSet{}, nil
+}
+func (tc *tsCacheAPIFailOnStorageCall) ChainHead(ctx context.Context) (*types.TipSet, error) {
+	tc.t.Fatal("storage call")
+	return &types.TipSet{}, nil
+}
+
 func TestTsCacheNulls(t *testing.T) {
-	tsc := newTSCache(50, func(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) {
-		t.Fatal("storage call")
-		return &types.TipSet{}, nil
-	})
+	tsc := newTSCache(50, &tsCacheAPIFailOnStorageCall{t: t})
 
 	h := abi.ChainEpoch(75)
 
@@ -91,7 +103,9 @@ func TestTsCacheNulls(t *testing.T) {
 	add()
 	add()
 
-	require.Equal(t, h-1, tsc.best().Height())
+	best, err := tsc.best()
+	require.NoError(t, err)
+	require.Equal(t, h-1, best.Height())
 
 	ts, err := tsc.get(h - 1)
 	require.NoError(t, err)
@@ -109,9 +123,17 @@ func TestTsCacheNulls(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, h-8, ts.Height())
 
-	require.NoError(t, tsc.revert(tsc.best()))
-	require.NoError(t, tsc.revert(tsc.best()))
-	require.Equal(t, h-8, tsc.best().Height())
+	best, err = tsc.best()
+	require.NoError(t, err)
+	require.NoError(t, tsc.revert(best))
+
+	best, err = tsc.best()
+	require.NoError(t, err)
+	require.NoError(t, tsc.revert(best))
+
+	best, err = tsc.best()
+	require.NoError(t, err)
+	require.Equal(t, h-8, best.Height())
 
 	h += 50
 	add()
@@ -120,3 +142,27 @@ func TestTsCacheNulls(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, h-1, ts.Height())
 }
+
+type tsCacheAPIStorageCallCounter struct {
+	t                      *testing.T
+	chainGetTipSetByHeight int
+	chainHead              int
+}
+
+func (tc *tsCacheAPIStorageCallCounter) ChainGetTipSetByHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) {
+	tc.chainGetTipSetByHeight++
+	return &types.TipSet{}, nil
+}
+func (tc *tsCacheAPIStorageCallCounter) ChainHead(ctx context.Context) (*types.TipSet, error) {
+	tc.chainHead++
+	return &types.TipSet{}, nil
+}
+
+func TestTsCacheEmpty(t *testing.T) {
+	// Calling best on an empty cache should just call out to the chain API
+	callCounter := &tsCacheAPIStorageCallCounter{t: t}
+	tsc := newTSCache(50, callCounter)
+	_, err := tsc.best()
+	require.NoError(t, err)
+	require.Equal(t, 1, callCounter.chainHead)
+}

chain/exchange/cbor_gen.go

@@ -1,13 +1,13 @@
 // Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
 
-package blocksync
+package exchange
 
 import (
 	"fmt"
 	"io"
 
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/ipfs/go-cid"
+	types "github.com/filecoin-project/lotus/chain/types"
+	cid "github.com/ipfs/go-cid"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	xerrors "golang.org/x/xerrors"
 )

chain/exchange/client.go

@@ -1,4 +1,4 @@
-package blocksync
+package exchange
 
 import (
 	"bufio"
@@ -7,13 +7,16 @@ import (
 	"math/rand"
 	"time"
 
-	host "github.com/libp2p/go-libp2p-core/host"
-	inet "github.com/libp2p/go-libp2p-core/network"
+	"github.com/libp2p/go-libp2p-core/host"
+	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
+
 	"go.opencensus.io/trace"
+	"go.uber.org/fx"
 	"golang.org/x/xerrors"
 
 	cborutil "github.com/filecoin-project/go-cbor-util"
+
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -21,11 +24,9 @@ import (
 	"github.com/filecoin-project/lotus/lib/peermgr"
 )
 
-// Protocol client.
-// FIXME: Rename to just `Client`. Not done at the moment to avoid
-//  disrupting too much of the consumer code, should be done along
-//  https://github.com/filecoin-project/lotus/issues/2612.
-type BlockSync struct {
+// client implements exchange.Client, using the libp2p ChainExchange protocol
+// as the fetching mechanism.
+type client struct {
 	// Connection manager used to contact the server.
 	// FIXME: We should have a reduced interface here, initialized
 	//  just with our protocol ID, we shouldn't be able to open *any*
@@ -35,13 +36,14 @@ type BlockSync struct {
 	peerTracker *bsPeerTracker
 }
 
-func NewClient(
-	host host.Host,
-	pmgr peermgr.MaybePeerMgr,
-) *BlockSync {
-	return &BlockSync{
+var _ Client = (*client)(nil)
+
+// NewClient creates a new libp2p-based exchange.Client that uses the libp2p
+// ChainExhange protocol as the fetching mechanism.
+func NewClient(lc fx.Lifecycle, host host.Host, pmgr peermgr.MaybePeerMgr) Client {
+	return &client{
 		host:        host,
-		peerTracker: newPeerTracker(pmgr.Mgr),
+		peerTracker: newPeerTracker(lc, host, pmgr.Mgr),
 	}
 }
 
@@ -62,11 +64,7 @@ func NewClient(
 // request options without disrupting external calls. In the future the
 // consumers should be forced to use a more standardized service and
 // adhere to a single API derived from this function.
-func (client *BlockSync) doRequest(
-	ctx context.Context,
-	req *Request,
-	singlePeer *peer.ID,
-) (*validatedResponse, error) {
+func (c *client) doRequest(ctx context.Context, req *Request, singlePeer *peer.ID) (*validatedResponse, error) {
 	// Validate request.
 	if req.Length == 0 {
 		return nil, xerrors.Errorf("invalid request of length 0")
@@ -86,7 +84,7 @@ func (client *BlockSync) doRequest(
 	if singlePeer != nil {
 		peers = []peer.ID{*singlePeer}
 	} else {
-		peers = client.getShuffledPeers()
+		peers = c.getShuffledPeers()
 		if len(peers) == 0 {
 			return nil, xerrors.Errorf("no peers available")
 		}
@@ -107,9 +105,9 @@ func (client *BlockSync) doRequest(
 		}
 
 		// Send request, read response.
-		res, err := client.sendRequestToPeer(ctx, peer, req)
+		res, err := c.sendRequestToPeer(ctx, peer, req)
 		if err != nil {
-			if !xerrors.Is(err, inet.ErrNoConn) {
+			if !xerrors.Is(err, network.ErrNoConn) {
 				log.Warnf("could not connect to peer %s: %s",
 					peer.String(), err)
 			}
@@ -117,15 +115,15 @@ func (client *BlockSync) doRequest(
 		}
 
 		// Process and validate response.
-		validRes, err := client.processResponse(req, res)
+		validRes, err := c.processResponse(req, res)
 		if err != nil {
 			log.Warnf("processing peer %s response failed: %s",
 				peer.String(), err)
 			continue
 		}
 
-		client.peerTracker.logGlobalSuccess(build.Clock.Since(globalTime))
-		client.host.ConnManager().TagPeer(peer, "bsync", SUCCESS_PEER_TAG_VALUE)
+		c.peerTracker.logGlobalSuccess(build.Clock.Since(globalTime))
+		c.host.ConnManager().TagPeer(peer, "bsync", SuccessPeerTagValue)
 		return validRes, nil
 	}
 
@@ -144,11 +142,8 @@ func (client *BlockSync) doRequest(
 // We are conflating in the single error returned both status and validation
 // errors. Peer penalization should happen here then, before returning, so
 // we can apply the correct penalties depending on the cause of the error.
-func (client *BlockSync) processResponse(
-	req *Request,
-	res *Response,
-	// FIXME: Add the `peer` as argument once we implement penalties.
-) (*validatedResponse, error) {
+// FIXME: Add the `peer` as argument once we implement penalties.
+func (c *client) processResponse(req *Request, res *Response) (*validatedResponse, error) {
 	err := res.statusToError()
 	if err != nil {
 		return nil, xerrors.Errorf("status error: %s", err)
@@ -172,7 +167,7 @@ func (client *BlockSync) processResponse(
 			resLength, req.Length)
 	}
 	if resLength < int(req.Length) && res.Status != Partial {
-		return nil, xerrors.Errorf("got less than requested without a proper status: %s", res.Status)
+		return nil, xerrors.Errorf("got less than requested without a proper status: %d", res.Status)
 	}
 
 	validRes := &validatedResponse{}
@@ -205,7 +200,7 @@ func (client *BlockSync) processResponse(
 		validRes.messages = make([]*CompactedMessages, resLength)
 		for i := 0; i < resLength; i++ {
 			if res.Chain[i].Messages == nil {
-				return nil, xerrors.Errorf("no messages included for tipset at height (head - %d): %w", i)
+				return nil, xerrors.Errorf("no messages included for tipset at height (head - %d)", i)
 			}
 			validRes.messages[i] = res.Chain[i].Messages
 		}
@@ -246,16 +241,8 @@ func (client *BlockSync) processResponse(
 	return validRes, nil
 }
 
-// GetBlocks fetches count blocks from the network, from the provided tipset
-// *backwards*, returning as many tipsets as count.
-//
-// {hint/usage}: This is used by the Syncer during normal chain syncing and when
-// resolving forks.
-func (client *BlockSync) GetBlocks(
-	ctx context.Context,
-	tsk types.TipSetKey,
-	count int,
-) ([]*types.TipSet, error) {
+// GetBlocks implements Client.GetBlocks(). Refer to the godocs there.
+func (c *client) GetBlocks(ctx context.Context, tsk types.TipSetKey, count int) ([]*types.TipSet, error) {
 	ctx, span := trace.StartSpan(ctx, "bsync.GetBlocks")
 	defer span.End()
 	if span.IsRecordingEvents() {
@@ -271,7 +258,7 @@ func (client *BlockSync) GetBlocks(
 		Options: Headers,
 	}
 
-	validRes, err := client.doRequest(ctx, req, nil)
+	validRes, err := c.doRequest(ctx, req, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -279,11 +266,8 @@ func (client *BlockSync) GetBlocks(
 	return validRes.tipsets, nil
 }
 
-func (client *BlockSync) GetFullTipSet(
-	ctx context.Context,
-	peer peer.ID,
-	tsk types.TipSetKey,
-) (*store.FullTipSet, error) {
+// GetFullTipSet implements Client.GetFullTipSet(). Refer to the godocs there.
+func (c *client) GetFullTipSet(ctx context.Context, peer peer.ID, tsk types.TipSetKey) (*store.FullTipSet, error) {
 	// TODO: round robin through these peers on error
 
 	req := &Request{
@@ -292,7 +276,7 @@ func (client *BlockSync) GetFullTipSet(
 		Options: Headers | Messages,
 	}
 
-	validRes, err := client.doRequest(ctx, req, &peer)
+	validRes, err := c.doRequest(ctx, req, &peer)
 	if err != nil {
 		return nil, err
 	}
@@ -302,12 +286,15 @@ func (client *BlockSync) GetFullTipSet(
 	//  *one* tipset here, so it's safe to index directly.
 }
 
-func (client *BlockSync) GetChainMessages(
-	ctx context.Context,
-	head *types.TipSet,
-	length uint64,
-) ([]*CompactedMessages, error) {
+// GetChainMessages implements Client.GetChainMessages(). Refer to the godocs there.
+func (c *client) GetChainMessages(ctx context.Context, head *types.TipSet, length uint64) ([]*CompactedMessages, error) {
 	ctx, span := trace.StartSpan(ctx, "GetChainMessages")
+	if span.IsRecordingEvents() {
+		span.AddAttributes(
+			trace.StringAttribute("tipset", fmt.Sprint(head.Cids())),
+			trace.Int64Attribute("count", int64(length)),
+		)
+	}
 	defer span.End()
 
 	req := &Request{
@@ -316,7 +303,7 @@ func (client *BlockSync) GetChainMessages(
 		Options: Messages,
 	}
 
-	validRes, err := client.doRequest(ctx, req, nil)
+	validRes, err := c.doRequest(ctx, req, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -327,11 +314,7 @@ func (client *BlockSync) GetChainMessages(
 // Send a request to a peer. Write request in the stream and read the
 // response back. We do not do any processing of the request/response
 // here.
-func (client *BlockSync) sendRequestToPeer(
-	ctx context.Context,
-	peer peer.ID,
-	req *Request,
-) (_ *Response, err error) {
+func (c *client) sendRequestToPeer(ctx context.Context, peer peer.ID, req *Request) (_ *Response, err error) {
 	// Trace code.
 	ctx, span := trace.StartSpan(ctx, "sendRequestToPeer")
 	defer span.End()
@@ -352,34 +335,33 @@ func (client *BlockSync) sendRequestToPeer(
 	}()
 	// -- TRACE --
 
-	supported, err := client.host.Peerstore().SupportsProtocols(peer, BlockSyncProtocolID)
+	supported, err := c.host.Peerstore().SupportsProtocols(peer, BlockSyncProtocolID, ChainExchangeProtocolID)
 	if err != nil {
+		c.RemovePeer(peer)
 		return nil, xerrors.Errorf("failed to get protocols for peer: %w", err)
 	}
-	if len(supported) == 0 || supported[0] != BlockSyncProtocolID {
-		return nil, xerrors.Errorf("peer %s does not support protocol %s",
-			peer, BlockSyncProtocolID)
-		// FIXME: `ProtoBook` should support a *single* protocol check that returns
-		//  a bool instead of a list.
+	if len(supported) == 0 || (supported[0] != BlockSyncProtocolID && supported[0] != ChainExchangeProtocolID) {
+		return nil, xerrors.Errorf("peer %s does not support protocols %s",
+			peer, []string{BlockSyncProtocolID, ChainExchangeProtocolID})
 	}
 
 	connectionStart := build.Clock.Now()
 
 	// Open stream to peer.
-	stream, err := client.host.NewStream(
-		inet.WithNoDial(ctx, "should already have connection"),
+	stream, err := c.host.NewStream(
+		network.WithNoDial(ctx, "should already have connection"),
 		peer,
-		BlockSyncProtocolID)
+		ChainExchangeProtocolID, BlockSyncProtocolID)
 	if err != nil {
-		client.RemovePeer(peer)
+		c.RemovePeer(peer)
 		return nil, xerrors.Errorf("failed to open stream to peer: %w", err)
 	}
 
 	// Write request.
-	_ = stream.SetWriteDeadline(time.Now().Add(WRITE_REQ_DEADLINE))
+	_ = stream.SetWriteDeadline(time.Now().Add(WriteReqDeadline))
 	if err := cborutil.WriteCborRPC(stream, req); err != nil {
 		_ = stream.SetWriteDeadline(time.Time{})
-		client.peerTracker.logFailure(peer, build.Clock.Since(connectionStart))
+		c.peerTracker.logFailure(peer, build.Clock.Since(connectionStart), req.Length)
 		// FIXME: Should we also remove peer here?
 		return nil, err
 	}
@@ -389,11 +371,11 @@ func (client *BlockSync) sendRequestToPeer(
 	// Read response.
 	var res Response
 	err = cborutil.ReadCborRPC(
-		bufio.NewReader(incrt.New(stream, READ_RES_MIN_SPEED, READ_RES_DEADLINE)),
+		bufio.NewReader(incrt.New(stream, ReadResMinSpeed, ReadResDeadline)),
 		&res)
 	if err != nil {
-		client.peerTracker.logFailure(peer, build.Clock.Since(connectionStart))
-		return nil, xerrors.Errorf("failed to read blocksync response: %w", err)
+		c.peerTracker.logFailure(peer, build.Clock.Since(connectionStart), req.Length)
+		return nil, xerrors.Errorf("failed to read chainxchg response: %w", err)
 	}
 
 	// FIXME: Move all this together at the top using a defer as done elsewhere.
@@ -406,32 +388,34 @@ func (client *BlockSync) sendRequestToPeer(
 		)
 	}
 
-	client.peerTracker.logSuccess(peer, build.Clock.Since(connectionStart))
+	c.peerTracker.logSuccess(peer, build.Clock.Since(connectionStart), uint64(len(res.Chain)))
 	// FIXME: We should really log a success only after we validate the response.
 	//  It might be a bit hard to do.
 	return &res, nil
 }
 
-func (client *BlockSync) AddPeer(p peer.ID) {
-	client.peerTracker.addPeer(p)
+// AddPeer implements Client.AddPeer(). Refer to the godocs there.
+func (c *client) AddPeer(p peer.ID) {
+	c.peerTracker.addPeer(p)
 }
 
-func (client *BlockSync) RemovePeer(p peer.ID) {
-	client.peerTracker.removePeer(p)
+// RemovePeer implements Client.RemovePeer(). Refer to the godocs there.
+func (c *client) RemovePeer(p peer.ID) {
+	c.peerTracker.removePeer(p)
 }
 
 // getShuffledPeers returns a preference-sorted set of peers (by latency
 // and failure counting), shuffling the first few peers so we don't always
 // pick the same peer.
 // FIXME: Consider merging with `shufflePrefix()s`.
-func (client *BlockSync) getShuffledPeers() []peer.ID {
-	peers := client.peerTracker.prefSortedPeers()
+func (c *client) getShuffledPeers() []peer.ID {
+	peers := c.peerTracker.prefSortedPeers()
 	shufflePrefix(peers)
 	return peers
 }
 
 func shufflePrefix(peers []peer.ID) {
-	prefix := SHUFFLE_PEERS_PREFIX
+	prefix := ShufflePeersPrefix
 	if len(peers) < prefix {
 		prefix = len(peers)
 	}

chain/exchange/doc.go

@@ -0,0 +1,19 @@
+// Package exchange contains the ChainExchange server and client components.
+//
+// ChainExchange is the basic chain synchronization protocol of Filecoin.
+// ChainExchange is an RPC-oriented protocol, with a single operation to
+// request blocks for now.
+//
+// A request contains a start anchor block (referred to with a CID), and a
+// amount of blocks requested beyond the anchor (including the anchor itself).
+//
+// A client can also pass options, encoded as a 64-bit bitfield. Lotus supports
+// two options at the moment:
+//
+//  - include block contents
+//  - include block messages
+//
+// The response will include a status code, an optional message, and the
+// response payload in case of success. The payload is a slice of serialized
+// tipsets.
+package exchange

chain/exchange/interfaces.go

@@ -0,0 +1,51 @@
+package exchange
+
+import (
+	"context"
+
+	"github.com/libp2p/go-libp2p-core/network"
+	"github.com/libp2p/go-libp2p-core/peer"
+
+	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/types"
+)
+
+// Server is the responder side of the ChainExchange protocol. It accepts
+// requests from clients and services them by returning the requested
+// chain data.
+type Server interface {
+	// HandleStream is the protocol handler to be registered on a libp2p
+	// protocol router.
+	//
+	// In the current version of the protocol, streams are single-use. The
+	// server will read a single Request, and will respond with a single
+	// Response. It will dispose of the stream straight after.
+	HandleStream(stream network.Stream)
+}
+
+// Client is the requesting side of the ChainExchange protocol. It acts as
+// a proxy for other components to request chain data from peers. It is chiefly
+// used by the Syncer.
+type Client interface {
+	// GetBlocks fetches block headers from the network, from the provided
+	// tipset *backwards*, returning as many tipsets as the count parameter,
+	// or less.
+	GetBlocks(ctx context.Context, tsk types.TipSetKey, count int) ([]*types.TipSet, error)
+
+	// GetChainMessages fetches messages from the network, from the provided
+	// tipset *backwards*, returning the messages from as many tipsets as the
+	// count parameter, or less.
+	GetChainMessages(ctx context.Context, head *types.TipSet, length uint64) ([]*CompactedMessages, error)
+
+	// GetFullTipSet fetches a full tipset from a given peer. If successful,
+	// the fetched object contains block headers and all messages in full form.
+	GetFullTipSet(ctx context.Context, peer peer.ID, tsk types.TipSetKey) (*store.FullTipSet, error)
+
+	// AddPeer adds a peer to the pool of peers that the Client requests
+	// data from.
+	AddPeer(peer peer.ID)
+
+	// RemovePeer removes a peer from the pool of peers that the Client
+	// requests data from.
+	RemovePeer(peer peer.ID)
+}

chain/exchange/peer_tracker.go

@@ -1,13 +1,16 @@
-package blocksync
+package exchange
 
 // FIXME: This needs to be reviewed.
 
 import (
+	"context"
 	"sort"
 	"sync"
 	"time"
 
+	host "github.com/libp2p/go-libp2p-core/host"
 	"github.com/libp2p/go-libp2p-core/peer"
+	"go.uber.org/fx"
 
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/lib/peermgr"
@@ -29,11 +32,30 @@ type bsPeerTracker struct {
 	pmgr *peermgr.PeerMgr
 }
 
-func newPeerTracker(pmgr *peermgr.PeerMgr) *bsPeerTracker {
-	return &bsPeerTracker{
+func newPeerTracker(lc fx.Lifecycle, h host.Host, pmgr *peermgr.PeerMgr) *bsPeerTracker {
+	bsPt := &bsPeerTracker{
 		peers: make(map[peer.ID]*peerStats),
 		pmgr:  pmgr,
 	}
+
+	sub, err := h.EventBus().Subscribe(new(peermgr.NewFilPeer))
+	if err != nil {
+		panic(err)
+	}
+
+	go func() {
+		for newPeer := range sub.Out() {
+			bsPt.addPeer(newPeer.(peermgr.NewFilPeer).Id)
+		}
+	}()
+
+	lc.Append(fx.Hook{
+		OnStop: func(ctx context.Context) error {
+			return sub.Close()
+		},
+	})
+
+	return bsPt
 }
 
 func (bpt *bsPeerTracker) addPeer(p peer.ID) {
@@ -72,16 +94,7 @@ func (bpt *bsPeerTracker) prefSortedPeers() []peer.ID {
 		var costI, costJ float64
 
 		getPeerInitLat := func(p peer.ID) float64 {
-			var res float64
-			if bpt.pmgr != nil {
-				if lat, ok := bpt.pmgr.GetPeerLatency(p); ok {
-					res = float64(lat)
-				}
-			}
-			if res == 0 {
-				res = float64(bpt.avgGlobalTime)
-			}
-			return res * newPeerMul
+			return float64(bpt.avgGlobalTime) * newPeerMul
 		}
 
 		if pi.successes+pi.failures > 0 {
@@ -107,8 +120,8 @@ func (bpt *bsPeerTracker) prefSortedPeers() []peer.ID {
 const (
 	// xInvAlpha = (N+1)/2
 
-	localInvAlpha  = 5  // 86% of the value is the last 9
-	globalInvAlpha = 20 // 86% of the value is the last 39
+	localInvAlpha  = 10 // 86% of the value is the last 19
+	globalInvAlpha = 25 // 86% of the value is the last 49
 )
 
 func (bpt *bsPeerTracker) logGlobalSuccess(dur time.Duration) {
@@ -133,7 +146,7 @@ func logTime(pi *peerStats, dur time.Duration) {
 
 }
 
-func (bpt *bsPeerTracker) logSuccess(p peer.ID, dur time.Duration) {
+func (bpt *bsPeerTracker) logSuccess(p peer.ID, dur time.Duration, reqSize uint64) {
 	bpt.lk.Lock()
 	defer bpt.lk.Unlock()
 
@@ -145,10 +158,13 @@ func (bpt *bsPeerTracker) logSuccess(p peer.ID, dur time.Duration) {
 	}
 
 	pi.successes++
-	logTime(pi, dur)
+	if reqSize == 0 {
+		reqSize = 1
+	}
+	logTime(pi, dur/time.Duration(reqSize))
 }
 
-func (bpt *bsPeerTracker) logFailure(p peer.ID, dur time.Duration) {
+func (bpt *bsPeerTracker) logFailure(p peer.ID, dur time.Duration, reqSize uint64) {
 	bpt.lk.Lock()
 	defer bpt.lk.Unlock()
 
@@ -160,7 +176,10 @@ func (bpt *bsPeerTracker) logFailure(p peer.ID, dur time.Duration) {
 	}
 
 	pi.failures++
-	logTime(pi, dur)
+	if reqSize == 0 {
+		reqSize = 1
+	}
+	logTime(pi, dur/time.Duration(reqSize))
 }
 
 func (bpt *bsPeerTracker) removePeer(p peer.ID) {

chain/exchange/protocol.go

@@ -1,9 +1,10 @@
-package blocksync
+package exchange
 
 import (
+	"time"
+
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/store"
-	"time"
 
 	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log"
@@ -12,9 +13,17 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
-var log = logging.Logger("blocksync")
+var log = logging.Logger("chainxchg")
 
-const BlockSyncProtocolID = "/fil/sync/blk/0.0.1"
+const (
+	// BlockSyncProtocolID is the protocol ID of the former blocksync protocol.
+	// Deprecated.
+	BlockSyncProtocolID = "/fil/sync/blk/0.0.1"
+
+	// ChainExchangeProtocolID is the protocol ID of the chain exchange
+	// protocol.
+	ChainExchangeProtocolID = "/fil/chain/xchg/0.0.1"
+)
 
 // FIXME: Bumped from original 800 to this to accommodate `syncFork()`
 //  use of `GetBlocks()`. It seems the expectation of that API is to
@@ -24,14 +33,16 @@ const BlockSyncProtocolID = "/fil/sync/blk/0.0.1"
 //   qualifier to avoid "const initializer [...] is not a constant" error.)
 var MaxRequestLength = uint64(build.ForkLengthThreshold)
 
-// Extracted constants from the code.
-// FIXME: Should be reviewed and confirmed.
-const SUCCESS_PEER_TAG_VALUE = 25
-const WRITE_REQ_DEADLINE = 5 * time.Second
-const READ_RES_DEADLINE = WRITE_REQ_DEADLINE
-const READ_RES_MIN_SPEED = 50 << 10
-const SHUFFLE_PEERS_PREFIX = 5
-const WRITE_RES_DEADLINE = 60 * time.Second
+const (
+	// Extracted constants from the code.
+	// FIXME: Should be reviewed and confirmed.
+	SuccessPeerTagValue = 25
+	WriteReqDeadline    = 5 * time.Second
+	ReadResDeadline     = WriteReqDeadline
+	ReadResMinSpeed     = 50 << 10
+	ShufflePeersPrefix  = 5
+	WriteResDeadline    = 60 * time.Second
+)
 
 // FIXME: Rename. Make private.
 type Request struct {
@@ -116,7 +127,7 @@ func (res *Response) statusToError() error {
 	case NotFound:
 		return xerrors.Errorf("not found")
 	case GoAway:
-		return xerrors.Errorf("not handling 'go away' blocksync responses yet")
+		return xerrors.Errorf("not handling 'go away' chainxchg responses yet")
 	case InternalError:
 		return xerrors.Errorf("block sync peer errored: %s", res.ErrorMessage)
 	case BadRequest:

chain/exchange/server.go

@@ -1,4 +1,4 @@
-package blocksync
+package exchange
 
 import (
 	"bufio"
@@ -18,38 +18,25 @@ import (
 	inet "github.com/libp2p/go-libp2p-core/network"
 )
 
-// BlockSyncService is the component that services BlockSync requests from
-// peers.
-//
-// BlockSync is the basic chain synchronization protocol of Filecoin. BlockSync
-// is an RPC-oriented protocol, with a single operation to request blocks.
-//
-// A request contains a start anchor block (referred to with a CID), and a
-// amount of blocks requested beyond the anchor (including the anchor itself).
-//
-// A client can also pass options, encoded as a 64-bit bitfield. Lotus supports
-// two options at the moment:
-//
-//  - include block contents
-//  - include block messages
-//
-// The response will include a status code, an optional message, and the
-// response payload in case of success. The payload is a slice of serialized
-// tipsets.
-// FIXME: Rename to just `Server` (will be done later, see note on `BlockSync`).
-type BlockSyncService struct {
+// server implements exchange.Server. It services requests for the
+// libp2p ChainExchange protocol.
+type server struct {
 	cs *store.ChainStore
 }
 
-func NewBlockSyncService(cs *store.ChainStore) *BlockSyncService {
-	return &BlockSyncService{
+var _ Server = (*server)(nil)
+
+// NewServer creates a new libp2p-based exchange.Server. It services requests
+// for the libp2p ChainExchange protocol.
+func NewServer(cs *store.ChainStore) Server {
+	return &server{
 		cs: cs,
 	}
 }
 
-// Entry point of the service, handles `Request`s.
-func (server *BlockSyncService) HandleStream(stream inet.Stream) {
-	ctx, span := trace.StartSpan(context.Background(), "blocksync.HandleStream")
+// HandleStream implements Server.HandleStream. Refer to the godocs there.
+func (s *server) HandleStream(stream inet.Stream) {
+	ctx, span := trace.StartSpan(context.Background(), "chainxchg.HandleStream")
 	defer span.End()
 
 	defer stream.Close() //nolint:errcheck
@@ -62,13 +49,13 @@ func (server *BlockSyncService) HandleStream(stream inet.Stream) {
 	log.Infow("block sync request",
 		"start", req.Head, "len", req.Length)
 
-	resp, err := server.processRequest(ctx, &req)
+	resp, err := s.processRequest(ctx, &req)
 	if err != nil {
 		log.Warn("failed to process request: ", err)
 		return
 	}
 
-	_ = stream.SetDeadline(time.Now().Add(WRITE_RES_DEADLINE))
+	_ = stream.SetDeadline(time.Now().Add(WriteResDeadline))
 	if err := cborutil.WriteCborRPC(stream, resp); err != nil {
 		_ = stream.SetDeadline(time.Time{})
 		log.Warnw("failed to write back response for handle stream",
@@ -80,10 +67,7 @@ func (server *BlockSyncService) HandleStream(stream inet.Stream) {
 
 // Validate and service the request. We return either a protocol
 // response or an internal error.
-func (server *BlockSyncService) processRequest(
-	ctx context.Context,
-	req *Request,
-) (*Response, error) {
+func (s *server) processRequest(ctx context.Context, req *Request) (*Response, error) {
 	validReq, errResponse := validateRequest(ctx, req)
 	if errResponse != nil {
 		// The request did not pass validation, return the response
@@ -91,17 +75,14 @@ func (server *BlockSyncService) processRequest(
 		return errResponse, nil
 	}
 
-	return server.serviceRequest(ctx, validReq)
+	return s.serviceRequest(ctx, validReq)
 }
 
 // Validate request. We either return a `validatedRequest`, or an error
 // `Response` indicating why we can't process it. We do not return any
 // internal errors here, we just signal protocol ones.
-func validateRequest(
-	ctx context.Context,
-	req *Request,
-) (*validatedRequest, *Response) {
-	_, span := trace.StartSpan(ctx, "blocksync.ValidateRequest")
+func validateRequest(ctx context.Context, req *Request) (*validatedRequest, *Response) {
+	_, span := trace.StartSpan(ctx, "chainxchg.ValidateRequest")
 	defer span.End()
 
 	validReq := validatedRequest{}
@@ -147,14 +128,11 @@ func validateRequest(
 	return &validReq, nil
 }
 
-func (server *BlockSyncService) serviceRequest(
-	ctx context.Context,
-	req *validatedRequest,
-) (*Response, error) {
-	_, span := trace.StartSpan(ctx, "blocksync.ServiceRequest")
+func (s *server) serviceRequest(ctx context.Context, req *validatedRequest) (*Response, error) {
+	_, span := trace.StartSpan(ctx, "chainxchg.ServiceRequest")
 	defer span.End()
 
-	chain, err := collectChainSegment(server.cs, req)
+	chain, err := collectChainSegment(s.cs, req)
 	if err != nil {
 		log.Warn("block sync request: collectChainSegment failed: ", err)
 		return &Response{
@@ -174,10 +152,7 @@ func (server *BlockSyncService) serviceRequest(
 	}, nil
 }
 
-func collectChainSegment(
-	cs *store.ChainStore,
-	req *validatedRequest,
-) ([]*BSTipSet, error) {
+func collectChainSegment(cs *store.ChainStore, req *validatedRequest) ([]*BSTipSet, error) {
 	var bstips []*BSTipSet
 
 	cur := req.head
@@ -221,37 +196,36 @@ func collectChainSegment(
 func gatherMessages(cs *store.ChainStore, ts *types.TipSet) ([]*types.Message, [][]uint64, []*types.SignedMessage, [][]uint64, error) {
 	blsmsgmap := make(map[cid.Cid]uint64)
 	secpkmsgmap := make(map[cid.Cid]uint64)
-	var secpkmsgs []*types.SignedMessage
-	var blsmsgs []*types.Message
 	var secpkincl, blsincl [][]uint64
 
+	var blscids, secpkcids []cid.Cid
 	for _, block := range ts.Blocks() {
-		bmsgs, smsgs, err := cs.MessagesForBlock(block)
+		bc, sc, err := cs.ReadMsgMetaCids(block.Messages)
 		if err != nil {
 			return nil, nil, nil, nil, err
 		}
 
 		// FIXME: DRY. Use `chain.Message` interface.
-		bmi := make([]uint64, 0, len(bmsgs))
-		for _, m := range bmsgs {
-			i, ok := blsmsgmap[m.Cid()]
+		bmi := make([]uint64, 0, len(bc))
+		for _, m := range bc {
+			i, ok := blsmsgmap[m]
 			if !ok {
-				i = uint64(len(blsmsgs))
-				blsmsgs = append(blsmsgs, m)
-				blsmsgmap[m.Cid()] = i
+				i = uint64(len(blscids))
+				blscids = append(blscids, m)
+				blsmsgmap[m] = i
 			}
 
 			bmi = append(bmi, i)
 		}
 		blsincl = append(blsincl, bmi)
 
-		smi := make([]uint64, 0, len(smsgs))
-		for _, m := range smsgs {
-			i, ok := secpkmsgmap[m.Cid()]
+		smi := make([]uint64, 0, len(sc))
+		for _, m := range sc {
+			i, ok := secpkmsgmap[m]
 			if !ok {
-				i = uint64(len(secpkmsgs))
-				secpkmsgs = append(secpkmsgs, m)
-				secpkmsgmap[m.Cid()] = i
+				i = uint64(len(secpkcids))
+				secpkcids = append(secpkcids, m)
+				secpkmsgmap[m] = i
 			}
 
 			smi = append(smi, i)
@@ -259,5 +233,15 @@ func gatherMessages(cs *store.ChainStore, ts *types.TipSet) ([]*types.Message, [
 		secpkincl = append(secpkincl, smi)
 	}
 
+	blsmsgs, err := cs.LoadMessagesFromCids(blscids)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	secpkmsgs, err := cs.LoadSignedMessagesFromCids(secpkcids)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
 	return blsmsgs, blsincl, secpkmsgs, secpkincl, nil
 }

chain/gen/gen.go

@@ -8,11 +8,13 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/filecoin-project/specs-actors/actors/runtime/proof"
+
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/crypto"
 	saminer "github.com/filecoin-project/specs-actors/actors/builtin/miner"
-	"github.com/filecoin-project/specs-actors/actors/crypto"
 	block "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-cid"
@@ -34,18 +36,19 @@ import (
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/chain/wallet"
 	"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
+	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/lotus/genesis"
 	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/lib/sigs"
 	"github.com/filecoin-project/lotus/node/repo"
-	"github.com/filecoin-project/sector-storage/ffiwrapper"
 )
 
-var log = logging.Logger("gen")
-
 const msgsPerBlock = 20
 
-var ValidWpostForTesting = []abi.PoStProof{{
+//nolint:deadcode,varcheck
+var log = logging.Logger("gen")
+
+var ValidWpostForTesting = []proof.PoStProof{{
 	ProofBytes: []byte("valid proof"),
 }}
 
@@ -56,7 +59,7 @@ type ChainGen struct {
 
 	cs *store.ChainStore
 
-	beacon beacon.RandomBeacon
+	beacon beacon.Schedule
 
 	sm *stmgr.StateManager
 
@@ -92,10 +95,8 @@ func (m mybs) Get(c cid.Cid) (block.Block, error) {
 	return b, nil
 }
 
-var rootkey, _ = address.NewIDAddress(80)
-
 var rootkeyMultisig = genesis.MultisigMeta{
-	Signers:         []address.Address{rootkey},
+	Signers:         []address.Address{remAccTestKey},
 	Threshold:       1,
 	VestingDuration: 0,
 	VestingStart:    0,
@@ -107,6 +108,18 @@ var DefaultVerifregRootkeyActor = genesis.Actor{
 	Meta:    rootkeyMultisig.ActorMeta(),
 }
 
+var remAccTestKey, _ = address.NewFromString("t1ceb34gnsc6qk5dt6n7xg6ycwzasjhbxm3iylkiy")
+var remAccMeta = genesis.MultisigMeta{
+	Signers:   []address.Address{remAccTestKey},
+	Threshold: 1,
+}
+
+var DefaultRemainderAccountActor = genesis.Actor{
+	Type:    genesis.TMultisig,
+	Balance: big.NewInt(0),
+	Meta:    remAccMeta.ActorMeta(),
+}
+
 func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
 	saminer.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{
 		abi.RegisteredSealProof_StackedDrg2KiBV1: {},
@@ -210,9 +223,10 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
 			*genm1,
 			*genm2,
 		},
-		VerifregRootKey: DefaultVerifregRootkeyActor,
-		NetworkName:     "",
-		Timestamp:       uint64(build.Clock.Now().Add(-500 * time.Duration(build.BlockDelaySecs) * time.Second).Unix()),
+		VerifregRootKey:  DefaultVerifregRootkeyActor,
+		RemainderAccount: DefaultRemainderAccountActor,
+		NetworkName:      "",
+		Timestamp:        uint64(build.Clock.Now().Add(-500 * time.Duration(build.BlockDelaySecs) * time.Second).Unix()),
 	}
 
 	genb, err := genesis2.MakeGenesisBlock(context.TODO(), bs, sys, tpl)
@@ -238,7 +252,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
 
 	miners := []address.Address{maddr1, maddr2}
 
-	beac := beacon.NewMockBeacon(time.Second)
+	beac := beacon.Schedule{{Start: 0, Beacon: beacon.NewMockBeacon(time.Second)}}
 	//beac, err := drand.NewDrandBeacon(tpl.Timestamp, build.BlockDelaySecs)
 	//if err != nil {
 	//return nil, xerrors.Errorf("creating drand beacon: %w", err)
@@ -304,7 +318,8 @@ func (cg *ChainGen) GenesisCar() ([]byte, error) {
 
 func CarWalkFunc(nd format.Node) (out []*format.Link, err error) {
 	for _, link := range nd.Links() {
-		if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
+		pref := link.Cid.Prefix()
+		if pref.Codec == cid.FilCommitmentSealed || pref.Codec == cid.FilCommitmentUnsealed {
 			continue
 		}
 		out = append(out, link)
@@ -323,7 +338,7 @@ func (cg *ChainGen) nextBlockProof(ctx context.Context, pts *types.TipSet, m add
 
 	prev := mbi.PrevBeaconEntry
 
-	entries, err := beacon.BeaconEntriesForBlock(ctx, cg.beacon, round, prev)
+	entries, err := beacon.BeaconEntriesForBlock(ctx, cg.beacon, round, pts.Height(), prev)
 	if err != nil {
 		return nil, nil, nil, xerrors.Errorf("get beacon entries for block: %w", err)
 	}
@@ -444,7 +459,7 @@ func (cg *ChainGen) NextTipSetFromMinersWithMessages(base *types.TipSet, miners
 
 func (cg *ChainGen) makeBlock(parents *types.TipSet, m address.Address, vrfticket *types.Ticket,
 	eticket *types.ElectionProof, bvals []types.BeaconEntry, height abi.ChainEpoch,
-	wpost []abi.PoStProof, msgs []*types.SignedMessage) (*types.FullBlock, error) {
+	wpost []proof.PoStProof, msgs []*types.SignedMessage) (*types.FullBlock, error) {
 
 	var ts uint64
 	if cg.Timestamper != nil {
@@ -544,7 +559,7 @@ type mca struct {
 	w   *wallet.Wallet
 	sm  *stmgr.StateManager
 	pv  ffiwrapper.Verifier
-	bcn beacon.RandomBeacon
+	bcn beacon.Schedule
 }
 
 func (mca mca) ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
@@ -575,7 +590,7 @@ func (mca mca) WalletSign(ctx context.Context, a address.Address, v []byte) (*cr
 
 type WinningPoStProver interface {
 	GenerateCandidates(context.Context, abi.PoStRandomness, uint64) ([]uint64, error)
-	ComputeProof(context.Context, []abi.SectorInfo, abi.PoStRandomness) ([]abi.PoStProof, error)
+	ComputeProof(context.Context, []proof.SectorInfo, abi.PoStRandomness) ([]proof.PoStProof, error)
 }
 
 type wppProvider struct{}
@@ -584,7 +599,7 @@ func (wpp *wppProvider) GenerateCandidates(ctx context.Context, _ abi.PoStRandom
 	return []uint64{0}, nil
 }
 
-func (wpp *wppProvider) ComputeProof(context.Context, []abi.SectorInfo, abi.PoStRandomness) ([]abi.PoStProof, error) {
+func (wpp *wppProvider) ComputeProof(context.Context, []proof.SectorInfo, abi.PoStRandomness) ([]proof.PoStProof, error) {
 	return ValidWpostForTesting, nil
 }
 
@@ -593,7 +608,7 @@ func IsRoundWinner(ctx context.Context, ts *types.TipSet, round abi.ChainEpoch,
 
 	buf := new(bytes.Buffer)
 	if err := miner.MarshalCBOR(buf); err != nil {
-		return nil, xerrors.Errorf("failed to cbor marshal address: %w")
+		return nil, xerrors.Errorf("failed to cbor marshal address: %w", err)
 	}
 
 	electionRand, err := store.DrawRandomness(brand.Data, crypto.DomainSeparationTag_ElectionProofProduction, round, buf.Bytes())
@@ -651,15 +666,15 @@ type genFakeVerifier struct{}
 
 var _ ffiwrapper.Verifier = (*genFakeVerifier)(nil)
 
-func (m genFakeVerifier) VerifySeal(svi abi.SealVerifyInfo) (bool, error) {
+func (m genFakeVerifier) VerifySeal(svi proof.SealVerifyInfo) (bool, error) {
 	return true, nil
 }
 
-func (m genFakeVerifier) VerifyWinningPoSt(ctx context.Context, info abi.WinningPoStVerifyInfo) (bool, error) {
+func (m genFakeVerifier) VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error) {
 	panic("not supported")
 }
 
-func (m genFakeVerifier) VerifyWindowPoSt(ctx context.Context, info abi.WindowPoStVerifyInfo) (bool, error) {
+func (m genFakeVerifier) VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error) {
 	panic("not supported")
 }
 

chain/gen/gen_test.go

@@ -3,8 +3,8 @@ package gen
 import (
 	"testing"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"

chain/gen/genesis/genblock.go

@@ -0,0 +1,41 @@
+package genesis
+
+import (
+	"encoding/hex"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+)
+
+const genesisMultihashString = "1220107d821c25dc0735200249df94a8bebc9c8e489744f86a4ca8919e81f19dcd72"
+const genesisBlockHex = "a5684461746574696d6573323031372d30352d30352030313a32373a3531674e6574776f726b6846696c65636f696e65546f6b656e6846696c65636f696e6c546f6b656e416d6f756e7473a36b546f74616c537570706c796d322c3030302c3030302c303030664d696e6572736d312c3430302c3030302c3030306c50726f746f636f6c4c616273a36b446576656c6f706d656e746b3330302c3030302c3030306b46756e6472616973696e676b3230302c3030302c3030306a466f756e646174696f6e6b3130302c3030302c303030674d657373616765784854686973206973207468652047656e6573697320426c6f636b206f66207468652046696c65636f696e20446563656e7472616c697a65642053746f72616765204e6574776f726b2e"
+
+var cidBuilder = cid.V1Builder{Codec: cid.DagCBOR, MhType: multihash.SHA2_256}
+
+func expectedCid() cid.Cid {
+	mh, err := multihash.FromHexString(genesisMultihashString)
+	if err != nil {
+		panic(err)
+	}
+	return cid.NewCidV1(cidBuilder.Codec, mh)
+}
+
+func getGenesisBlock() (blocks.Block, error) {
+	genesisBlockData, err := hex.DecodeString(genesisBlockHex)
+	if err != nil {
+		return nil, err
+	}
+
+	genesisCid, err := cidBuilder.Sum(genesisBlockData)
+	if err != nil {
+		return nil, err
+	}
+
+	block, err := blocks.NewBlockWithCid(genesisBlockData, genesisCid)
+	if err != nil {
+		return nil, err
+	}
+
+	return block, nil
+}

chain/gen/genesis/genesis.go

@@ -2,7 +2,11 @@ package genesis
 
 import (
 	"context"
+	"crypto/rand"
 	"encoding/json"
+	"fmt"
+
+	"github.com/filecoin-project/go-state-types/network"
 
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
@@ -12,13 +16,13 @@ import (
 
 	"github.com/filecoin-project/go-address"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/account"
 	"github.com/filecoin-project/specs-actors/actors/builtin/multisig"
 	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
-	"github.com/filecoin-project/specs-actors/actors/crypto"
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
 
 	"github.com/filecoin-project/lotus/build"
@@ -117,11 +121,6 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
 		return nil, nil, xerrors.Errorf("making new state tree: %w", err)
 	}
 
-	emptyobject, err := cst.Put(context.TODO(), []struct{}{})
-	if err != nil {
-		return nil, nil, xerrors.Errorf("failed putting empty object: %w", err)
-	}
-
 	// Create system actor
 
 	sysact, err := SetupSystemActor(bs)
@@ -134,7 +133,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
 
 	// Create init actor
 
-	initact, keyIDs, err := SetupInitActor(bs, template.NetworkName, template.Accounts, template.VerifregRootKey)
+	idStart, initact, keyIDs, err := SetupInitActor(bs, template.NetworkName, template.Accounts, template.VerifregRootKey)
 	if err != nil {
 		return nil, nil, xerrors.Errorf("setup init actor: %w", err)
 	}
@@ -190,31 +189,47 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
 		return nil, nil, xerrors.Errorf("set market actor: %w", err)
 	}
 
+	burntRoot, err := cst.Put(ctx, &account.State{
+		Address: builtin.BurntFundsActorAddr,
+	})
+	if err != nil {
+		return nil, nil, xerrors.Errorf("failed to setup burnt funds actor state: %w", err)
+	}
+
 	// Setup burnt-funds
 	err = state.SetActor(builtin.BurntFundsActorAddr, &types.Actor{
 		Code:    builtin.AccountActorCodeID,
 		Balance: types.NewInt(0),
-		Head:    emptyobject,
+		Head:    burntRoot,
 	})
 	if err != nil {
 		return nil, nil, xerrors.Errorf("set burnt funds account actor: %w", err)
 	}
 
 	// Create accounts
-	for id, info := range template.Accounts {
-		if info.Type != genesis.TAccount && info.Type != genesis.TMultisig {
+	for _, info := range template.Accounts {
+
+		switch info.Type {
+		case genesis.TAccount:
+			if err := createAccountActor(ctx, cst, state, info, keyIDs); err != nil {
+				return nil, nil, xerrors.Errorf("failed to create account actor: %w", err)
+			}
+
+		case genesis.TMultisig:
+
+			ida, err := address.NewIDAddress(uint64(idStart))
+			if err != nil {
+				return nil, nil, err
+			}
+			idStart++
+
+			if err := createMultisigAccount(ctx, bs, cst, state, ida, info, keyIDs); err != nil {
+				return nil, nil, err
+			}
+		default:
 			return nil, nil, xerrors.New("unsupported account type")
 		}
 
-		ida, err := address.NewIDAddress(uint64(AccountStart + id))
-		if err != nil {
-			return nil, nil, err
-		}
-
-		if err = createAccount(ctx, bs, cst, state, ida, info); err != nil {
-			return nil, nil, err
-		}
-
 	}
 
 	vregroot, err := address.NewIDAddress(80)
@@ -222,8 +237,8 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
 		return nil, nil, err
 	}
 
-	if err = createAccount(ctx, bs, cst, state, vregroot, template.VerifregRootKey); err != nil {
-		return nil, nil, err
+	if err = createMultisigAccount(ctx, bs, cst, state, vregroot, template.VerifregRootKey, keyIDs); err != nil {
+		return nil, nil, xerrors.Errorf("failed to set up verified registry signer: %w", err)
 	}
 
 	// Setup the first verifier as ID-address 81
@@ -258,63 +273,132 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
 		Balance: types.NewInt(0),
 		Head:    verifierState,
 	})
-
 	if err != nil {
 		return nil, nil, xerrors.Errorf("setting account from actmap: %w", err)
 	}
 
+	totalFilAllocated := big.Zero()
+
+	// flush as ForEach works on the HAMT
+	if _, err := state.Flush(ctx); err != nil {
+		return nil, nil, err
+	}
+	err = state.ForEach(func(addr address.Address, act *types.Actor) error {
+		totalFilAllocated = big.Add(totalFilAllocated, act.Balance)
+		return nil
+	})
+	if err != nil {
+		return nil, nil, xerrors.Errorf("summing account balances in state tree: %w", err)
+	}
+
+	totalFil := big.Mul(big.NewInt(int64(build.FilBase)), big.NewInt(int64(build.FilecoinPrecision)))
+	remainingFil := big.Sub(totalFil, totalFilAllocated)
+	if remainingFil.Sign() < 0 {
+		return nil, nil, xerrors.Errorf("somehow overallocated filecoin (allocated = %s)", types.FIL(totalFilAllocated))
+	}
+
+	remAccKey, err := address.NewIDAddress(90)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	template.RemainderAccount.Balance = remainingFil
+
+	if err := createMultisigAccount(ctx, bs, cst, state, remAccKey, template.RemainderAccount, keyIDs); err != nil {
+		return nil, nil, xerrors.Errorf("failed to set up remainder account: %w", err)
+	}
+
 	return state, keyIDs, nil
 }
 
-func createAccount(ctx context.Context, bs bstore.Blockstore, cst cbor.IpldStore, state *state.StateTree, ida address.Address, info genesis.Actor) error {
-	if info.Type == genesis.TAccount {
-		var ainfo genesis.AccountMeta
-		if err := json.Unmarshal(info.Meta, &ainfo); err != nil {
-			return xerrors.Errorf("unmarshaling account meta: %w", err)
-		}
-		st, err := cst.Put(ctx, &account.State{Address: ainfo.Owner})
-		if err != nil {
-			return err
-		}
-		err = state.SetActor(ida, &types.Actor{
-			Code:    builtin.AccountActorCodeID,
-			Balance: info.Balance,
-			Head:    st,
-		})
-		if err != nil {
-			return xerrors.Errorf("setting account from actmap: %w", err)
-		}
-	} else if info.Type == genesis.TMultisig {
-		var ainfo genesis.MultisigMeta
-		if err := json.Unmarshal(info.Meta, &ainfo); err != nil {
-			return xerrors.Errorf("unmarshaling account meta: %w", err)
-		}
-		pending, err := adt.MakeEmptyMap(adt.WrapStore(ctx, cst)).Root()
-		if err != nil {
-			return xerrors.Errorf("failed to create empty map: %v", err)
-		}
-
-		st, err := cst.Put(ctx, &multisig.State{
-			Signers:               ainfo.Signers,
-			NumApprovalsThreshold: uint64(ainfo.Threshold),
-			StartEpoch:            abi.ChainEpoch(ainfo.VestingStart),
-			UnlockDuration:        abi.ChainEpoch(ainfo.VestingDuration),
-			PendingTxns:           pending,
-			InitialBalance:        info.Balance,
-		})
-		if err != nil {
-			return err
-		}
-		err = state.SetActor(ida, &types.Actor{
-			Code:    builtin.MultisigActorCodeID,
-			Balance: info.Balance,
-			Head:    st,
-		})
-		if err != nil {
-			return xerrors.Errorf("setting account from actmap: %w", err)
-		}
+func createAccountActor(ctx context.Context, cst cbor.IpldStore, state *state.StateTree, info genesis.Actor, keyIDs map[address.Address]address.Address) error {
+	var ainfo genesis.AccountMeta
+	if err := json.Unmarshal(info.Meta, &ainfo); err != nil {
+		return xerrors.Errorf("unmarshaling account meta: %w", err)
+	}
+	st, err := cst.Put(ctx, &account.State{Address: ainfo.Owner})
+	if err != nil {
+		return err
 	}
 
+	ida, ok := keyIDs[ainfo.Owner]
+	if !ok {
+		return fmt.Errorf("no registered ID for account actor: %s", ainfo.Owner)
+	}
+
+	err = state.SetActor(ida, &types.Actor{
+		Code:    builtin.AccountActorCodeID,
+		Balance: info.Balance,
+		Head:    st,
+	})
+	if err != nil {
+		return xerrors.Errorf("setting account from actmap: %w", err)
+	}
+	return nil
+}
+
+func createMultisigAccount(ctx context.Context, bs bstore.Blockstore, cst cbor.IpldStore, state *state.StateTree, ida address.Address, info genesis.Actor, keyIDs map[address.Address]address.Address) error {
+	if info.Type != genesis.TMultisig {
+		return fmt.Errorf("can only call createMultisigAccount with multisig Actor info")
+	}
+	var ainfo genesis.MultisigMeta
+	if err := json.Unmarshal(info.Meta, &ainfo); err != nil {
+		return xerrors.Errorf("unmarshaling account meta: %w", err)
+	}
+	pending, err := adt.MakeEmptyMap(adt.WrapStore(ctx, cst)).Root()
+	if err != nil {
+		return xerrors.Errorf("failed to create empty map: %v", err)
+	}
+
+	var signers []address.Address
+
+	for _, e := range ainfo.Signers {
+		idAddress, ok := keyIDs[e]
+		if !ok {
+			return fmt.Errorf("no registered key ID for signer: %s", e)
+		}
+
+		// Check if actor already exists
+		_, err := state.GetActor(e)
+		if err == nil {
+			signers = append(signers, idAddress)
+			continue
+		}
+
+		st, err := cst.Put(ctx, &account.State{Address: e})
+		if err != nil {
+			return err
+		}
+		err = state.SetActor(idAddress, &types.Actor{
+			Code:    builtin.AccountActorCodeID,
+			Balance: types.NewInt(0),
+			Head:    st,
+		})
+		if err != nil {
+			return xerrors.Errorf("setting account from actmap: %w", err)
+		}
+		signers = append(signers, idAddress)
+	}
+
+	st, err := cst.Put(ctx, &multisig.State{
+		Signers:               signers,
+		NumApprovalsThreshold: uint64(ainfo.Threshold),
+		StartEpoch:            abi.ChainEpoch(ainfo.VestingStart),
+		UnlockDuration:        abi.ChainEpoch(ainfo.VestingDuration),
+		PendingTxns:           pending,
+		InitialBalance:        info.Balance,
+	})
+	if err != nil {
+		return err
+	}
+	err = state.SetActor(ida, &types.Actor{
+		Code:    builtin.MultisigActorCodeID,
+		Balance: info.Balance,
+		Head:    st,
+	})
+	if err != nil {
+		return xerrors.Errorf("setting account from actmap: %w", err)
+	}
 	return nil
 }
 
@@ -322,6 +406,10 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci
 	verifNeeds := make(map[address.Address]abi.PaddedPieceSize)
 	var sum abi.PaddedPieceSize
 
+	nwv := func(context.Context, abi.ChainEpoch) network.Version {
+		return build.NewestNetworkVersion
+	}
+
 	vmopt := vm.VMOpts{
 		StateBase:      stateroot,
 		Epoch:          0,
@@ -329,6 +417,7 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci
 		Bstore:         cs.Blockstore(),
 		Syscalls:       mkFakedSigSyscalls(cs.VMSys()),
 		CircSupplyCalc: nil,
+		NtwkVersion:    nwv,
 		BaseFee:        types.NewInt(0),
 	}
 	vm, err := vm.NewVM(&vmopt)
@@ -336,11 +425,12 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci
 		return cid.Undef, xerrors.Errorf("failed to create NewVM: %w", err)
 	}
 
-	for _, m := range template.Miners {
+	for mi, m := range template.Miners {
+		for si, s := range m.Sectors {
+			if s.Deal.Provider != m.ID {
+				return cid.Undef, xerrors.Errorf("Sector %d in miner %d in template had mismatch in provider and miner ID: %s != %s", si, mi, s.Deal.Provider, m.ID)
+			}
 
-		// Add the miner to the market actor's balance table
-		_, err = doExec(ctx, vm, builtin.StorageMarketActorAddr, m.Owner, builtin.MethodsMarket.AddBalance, mustEnc(adt.Empty))
-		for _, s := range m.Sectors {
 			amt := s.Deal.PieceSize
 			verifNeeds[keyIDs[s.Deal.Client]] += amt
 			sum += amt
@@ -430,14 +520,38 @@ func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallB
 
 	log.Infof("Empty Genesis root: %s", emptyroot)
 
+	tickBuf := make([]byte, 32)
+	_, _ = rand.Read(tickBuf)
 	genesisticket := &types.Ticket{
-		VRFProof: []byte("vrf proof0000000vrf proof0000000"),
+		VRFProof: tickBuf,
+	}
+
+	filecoinGenesisCid, err := cid.Decode("bafyreiaqpwbbyjo4a42saasj36kkrpv4tsherf2e7bvezkert2a7dhonoi")
+	if err != nil {
+		return nil, xerrors.Errorf("failed to decode filecoin genesis block CID: %w", err)
+	}
+
+	if !expectedCid().Equals(filecoinGenesisCid) {
+		return nil, xerrors.Errorf("expectedCid != filecoinGenesisCid")
+	}
+
+	gblk, err := getGenesisBlock()
+	if err != nil {
+		return nil, xerrors.Errorf("failed to construct filecoin genesis block: %w", err)
+	}
+
+	if !filecoinGenesisCid.Equals(gblk.Cid()) {
+		return nil, xerrors.Errorf("filecoinGenesisCid != gblk.Cid")
+	}
+
+	if err := bs.Put(gblk); err != nil {
+		return nil, xerrors.Errorf("failed writing filecoin genesis block to blockstore: %w", err)
 	}
 
 	b := &types.BlockHeader{
 		Miner:                 builtin.SystemActorAddr,
 		Ticket:                genesisticket,
-		Parents:               []cid.Cid{},
+		Parents:               []cid.Cid{filecoinGenesisCid},
 		Height:                0,
 		ParentWeight:          types.NewInt(0),
 		ParentStateRoot:       stateroot,

chain/gen/genesis/miners.go

@@ -6,6 +6,10 @@ import (
 	"fmt"
 	"math/rand"
 
+	"github.com/filecoin-project/lotus/build"
+
+	"github.com/filecoin-project/go-state-types/network"
+
 	"github.com/filecoin-project/lotus/chain/state"
 
 	"github.com/ipfs/go-cid"
@@ -14,15 +18,15 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/sector-storage/ffiwrapper"
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/market"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 	"github.com/filecoin-project/specs-actors/actors/builtin/reward"
-	"github.com/filecoin-project/specs-actors/actors/crypto"
 	"github.com/filecoin-project/specs-actors/actors/runtime"
 
 	"github.com/filecoin-project/lotus/chain/store"
@@ -61,6 +65,10 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 		return big.Zero(), nil
 	}
 
+	nwv := func(context.Context, abi.ChainEpoch) network.Version {
+		return build.NewestNetworkVersion
+	}
+
 	vmopt := &vm.VMOpts{
 		StateBase:      sroot,
 		Epoch:          0,
@@ -68,6 +76,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 		Bstore:         cs.Blockstore(),
 		Syscalls:       mkFakedSigSyscalls(cs.VMSys()),
 		CircSupplyCalc: csc,
+		NtwkVersion:    nwv,
 		BaseFee:        types.NewInt(0),
 	}
 
@@ -129,6 +138,9 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 
 				return nil
 			})
+			if err != nil {
+				return cid.Undef, xerrors.Errorf("mutating state: %w", err)
+			}
 		}
 
 		// Add market funds
@@ -137,7 +149,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 			params := mustEnc(&minerInfos[i].maddr)
 			_, err := doExecValue(ctx, vm, builtin.StorageMarketActorAddr, m.Worker, m.MarketBalance, builtin.MethodsMarket.AddBalance, params)
 			if err != nil {
-				return cid.Undef, xerrors.Errorf("failed to create genesis miner: %w", err)
+				return cid.Undef, xerrors.Errorf("failed to create genesis miner (add balance): %w", err)
 			}
 		}
 
@@ -149,7 +161,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 
 				ret, err := doExecValue(ctx, vm, builtin.StorageMarketActorAddr, m.Worker, big.Zero(), builtin.MethodsMarket.PublishStorageDeals, mustEnc(params))
 				if err != nil {
-					return xerrors.Errorf("failed to create genesis miner: %w", err)
+					return xerrors.Errorf("failed to create genesis miner (publish deals): %w", err)
 				}
 				var ids market.PublishStorageDealsReturn
 				if err := ids.UnmarshalCBOR(bytes.NewReader(ret)); err != nil {
@@ -217,9 +229,12 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 		}
 
 		err = vm.MutateState(ctx, builtin.RewardActorAddr, func(sct cbor.IpldStore, st *reward.State) error {
-			st = reward.ConstructState(qaPow)
+			*st = *reward.ConstructState(qaPow)
 			return nil
 		})
+		if err != nil {
+			return cid.Undef, xerrors.Errorf("mutating state: %w", err)
+		}
 	}
 
 	for i, m := range miners {
@@ -244,7 +259,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 
 				// we've added fake power for this sector above, remove it now
 				err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *power.State) error {
-					st.TotalQualityAdjPower = types.BigSub(st.TotalQualityAdjPower, sectorWeight)
+					st.TotalQualityAdjPower = types.BigSub(st.TotalQualityAdjPower, sectorWeight) //nolint:scopelint
 					st.TotalRawBytePower = types.BigSub(st.TotalRawBytePower, types.NewInt(uint64(m.SectorSize)))
 					return nil
 				})
@@ -262,6 +277,8 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 					return cid.Undef, xerrors.Errorf("getting current total power: %w", err)
 				}
 
+				pcd := miner.PreCommitDepositForPower(epochReward.ThisEpochRewardSmoothed, tpow.QualityAdjPowerSmoothed, sectorWeight)
+
 				pledge := miner.InitialPledgeForPower(
 					sectorWeight,
 					epochReward.ThisEpochBaselinePower,
@@ -271,6 +288,8 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 					circSupply(ctx, vm, minerInfos[i].maddr),
 				)
 
+				pledge = big.Add(pcd, pledge)
+
 				fmt.Println(types.FIL(pledge))
 				_, err = doExecValue(ctx, vm, minerInfos[i].maddr, m.Worker, pledge, builtin.MethodsMiner.PreCommitSector, mustEnc(params))
 				if err != nil {
@@ -320,13 +339,13 @@ type fakeRand struct{}
 
 func (fr *fakeRand) GetChainRandomness(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
 	out := make([]byte, 32)
-	_, _ = rand.New(rand.NewSource(int64(randEpoch * 1000))).Read(out)
+	_, _ = rand.New(rand.NewSource(int64(randEpoch * 1000))).Read(out) //nolint
 	return out, nil
 }
 
 func (fr *fakeRand) GetBeaconRandomness(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
 	out := make([]byte, 32)
-	_, _ = rand.New(rand.NewSource(int64(randEpoch))).Read(out)
+	_, _ = rand.New(rand.NewSource(int64(randEpoch))).Read(out) //nolint
 	return out, nil
 }
 

chain/gen/genesis/t01_init.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+
 	"github.com/filecoin-project/go-address"
 
 	"github.com/filecoin-project/specs-actors/actors/builtin"
@@ -19,9 +20,9 @@ import (
 	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
-func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesis.Actor, rootVerifier genesis.Actor) (*types.Actor, map[address.Address]address.Address, error) {
+func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesis.Actor, rootVerifier genesis.Actor) (int64, *types.Actor, map[address.Address]address.Address, error) {
 	if len(initialActors) > MaxAccounts {
-		return nil, nil, xerrors.New("too many initial actors")
+		return 0, nil, nil, xerrors.New("too many initial actors")
 	}
 
 	var ias init_.State
@@ -32,55 +33,105 @@ func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesi
 	amap := adt.MakeEmptyMap(store)
 
 	keyToId := map[address.Address]address.Address{}
+	counter := int64(AccountStart)
 
-	for i, a := range initialActors {
+	for _, a := range initialActors {
 		if a.Type == genesis.TMultisig {
+			var ainfo genesis.MultisigMeta
+			if err := json.Unmarshal(a.Meta, &ainfo); err != nil {
+				return 0, nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
+			}
+			for _, e := range ainfo.Signers {
+
+				if _, ok := keyToId[e]; ok {
+					continue
+				}
+
+				fmt.Printf("init set %s t0%d\n", e, counter)
+
+				value := cbg.CborInt(counter)
+				if err := amap.Put(adt.AddrKey(e), &value); err != nil {
+					return 0, nil, nil, err
+				}
+				counter = counter + 1
+				var err error
+				keyToId[e], err = address.NewIDAddress(uint64(value))
+				if err != nil {
+					return 0, nil, nil, err
+				}
+
+			}
+			// Need to add actors for all multisigs too
 			continue
 		}
 
 		if a.Type != genesis.TAccount {
-			return nil, nil, xerrors.Errorf("unsupported account type: %s", a.Type) // TODO: Support msig (skip here)
+			return 0, nil, nil, xerrors.Errorf("unsupported account type: %s", a.Type)
 		}
 
 		var ainfo genesis.AccountMeta
 		if err := json.Unmarshal(a.Meta, &ainfo); err != nil {
-			return nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
+			return 0, nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
 		}
 
-		fmt.Printf("init set %s t0%d\n", ainfo.Owner, AccountStart+int64(i))
+		fmt.Printf("init set %s t0%d\n", ainfo.Owner, counter)
 
-		value := cbg.CborInt(AccountStart + int64(i))
+		value := cbg.CborInt(counter)
 		if err := amap.Put(adt.AddrKey(ainfo.Owner), &value); err != nil {
-			return nil, nil, err
+			return 0, nil, nil, err
 		}
+		counter = counter + 1
 
 		var err error
 		keyToId[ainfo.Owner], err = address.NewIDAddress(uint64(value))
 		if err != nil {
-			return nil, nil, err
+			return 0, nil, nil, err
 		}
 	}
 
 	if rootVerifier.Type == genesis.TAccount {
 		var ainfo genesis.AccountMeta
 		if err := json.Unmarshal(rootVerifier.Meta, &ainfo); err != nil {
-			return nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
+			return 0, nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
 		}
 		value := cbg.CborInt(80)
 		if err := amap.Put(adt.AddrKey(ainfo.Owner), &value); err != nil {
-			return nil, nil, err
+			return 0, nil, nil, err
+		}
+	} else if rootVerifier.Type == genesis.TMultisig {
+		var ainfo genesis.MultisigMeta
+		if err := json.Unmarshal(rootVerifier.Meta, &ainfo); err != nil {
+			return 0, nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
+		}
+		for _, e := range ainfo.Signers {
+			if _, ok := keyToId[e]; ok {
+				continue
+			}
+			fmt.Printf("init set %s t0%d\n", e, counter)
+
+			value := cbg.CborInt(counter)
+			if err := amap.Put(adt.AddrKey(e), &value); err != nil {
+				return 0, nil, nil, err
+			}
+			counter = counter + 1
+			var err error
+			keyToId[e], err = address.NewIDAddress(uint64(value))
+			if err != nil {
+				return 0, nil, nil, err
+			}
+
 		}
 	}
 
 	amapaddr, err := amap.Root()
 	if err != nil {
-		return nil, nil, err
+		return 0, nil, nil, err
 	}
 	ias.AddressMap = amapaddr
 
 	statecid, err := store.Put(store.Context(), &ias)
 	if err != nil {
-		return nil, nil, err
+		return 0, nil, nil, err
 	}
 
 	act := &types.Actor{
@@ -88,5 +139,5 @@ func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesi
 		Head: statecid,
 	}
 
-	return act, keyToId, nil
+	return counter, act, keyToId, nil
 }

chain/gen/genesis/t02_reward.go

@@ -3,7 +3,7 @@ package genesis
 import (
 	"context"
 
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/go-state-types/big"
 
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/reward"

chain/gen/genesis/util.go

@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"golang.org/x/xerrors"
 
@@ -21,14 +21,10 @@ func mustEnc(i cbg.CBORMarshaler) []byte {
 	return enc
 }
 
-func doExec(ctx context.Context, vm *vm.VM, to, from address.Address, method abi.MethodNum, params []byte) ([]byte, error) {
-	return doExecValue(ctx, vm, to, from, types.NewInt(0), method, params)
-}
-
 func doExecValue(ctx context.Context, vm *vm.VM, to, from address.Address, value types.BigInt, method abi.MethodNum, params []byte) ([]byte, error) {
 	act, err := vm.StateTree().GetActor(from)
 	if err != nil {
-		return nil, xerrors.Errorf("doExec failed to get from actor: %w", err)
+		return nil, xerrors.Errorf("doExec failed to get from actor (%s): %w", from, err)
 	}
 
 	ret, err := vm.ApplyImplicitMessage(ctx, &types.Message{

chain/gen/mining.go

@@ -3,7 +3,7 @@ package gen
 import (
 	"context"
 
-	"github.com/filecoin-project/specs-actors/actors/crypto"
+	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
 	cid "github.com/ipfs/go-cid"
 	cbor "github.com/ipfs/go-ipld-cbor"

chain/gen/slashfilter/slashfilter.go

@@ -9,8 +9,8 @@ import (
 	ds "github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
 
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/specs-actors/actors/abi"
 )
 
 type SlashFilter struct {

chain/market/fundmgr.go

@@ -4,8 +4,8 @@ import (
 	"context"
 	"sync"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
 	"go.uber.org/fx"
 
 	"github.com/filecoin-project/specs-actors/actors/builtin"
@@ -14,7 +14,6 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/events"
 	"github.com/filecoin-project/lotus/chain/events/state"
@@ -52,7 +51,7 @@ func StartFundManager(lc fx.Lifecycle, api API) *FundMgr {
 			match := func(oldTs, newTs *types.TipSet) (bool, events.StateChange, error) {
 				return dealDiffFn(ctx, oldTs.Key(), newTs.Key())
 			}
-			return ev.StateChanged(fm.checkFunc, fm.stateChanged, fm.revert, int(build.MessageConfidence), events.NoTimeout, match)
+			return ev.StateChanged(fm.checkFunc, fm.stateChanged, fm.revert, 0, events.NoTimeout, match)
 		},
 	})
 	return fm
@@ -92,6 +91,10 @@ func (fm *FundMgr) stateChanged(ts *types.TipSet, ts2 *types.TipSet, states even
 	// overwrite our in memory cache with new values from chain (chain is canonical)
 	fm.lk.Lock()
 	for addr, balanceChange := range changedBalances {
+		if fm.available[addr].Int != nil {
+			log.Infof("State balance change recorded, prev: %s, new: %s", fm.available[addr].String(), balanceChange.To.String())
+		}
+
 		fm.available[addr] = balanceChange.To
 	}
 	fm.lk.Unlock()
@@ -116,15 +119,17 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Add
 		return cid.Undef, err
 	}
 	fm.lk.Lock()
+	bal, err := fm.api.StateMarketBalance(ctx, addr, types.EmptyTSK)
+	if err != nil {
+		fm.lk.Unlock()
+		return cid.Undef, err
+	}
+
+	stateAvail := types.BigSub(bal.Escrow, bal.Locked)
+
 	avail, ok := fm.available[idAddr]
 	if !ok {
-		bal, err := fm.api.StateMarketBalance(ctx, addr, types.EmptyTSK)
-		if err != nil {
-			fm.lk.Unlock()
-			return cid.Undef, err
-		}
-
-		avail = types.BigSub(bal.Escrow, bal.Locked)
+		avail = stateAvail
 	}
 
 	toAdd := types.BigSub(amt, avail)
@@ -134,6 +139,8 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Add
 	fm.available[idAddr] = big.Add(avail, toAdd)
 	fm.lk.Unlock()
 
+	log.Infof("Funds operation w/ Expected Balance: %s, In State: %s, Requested: %s, Adding: %s", avail.String(), stateAvail.String(), amt.String(), toAdd.String())
+
 	if toAdd.LessThanEqual(big.Zero()) {
 		return cid.Undef, nil
 	}

chain/market/fundmgr_test.go

@@ -10,9 +10,9 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
-	"github.com/filecoin-project/specs-actors/actors/crypto"
 	tutils "github.com/filecoin-project/specs-actors/support/testing"
 
 	"github.com/filecoin-project/lotus/api"
@@ -147,6 +147,7 @@ func TestAddFunds(t *testing.T) {
 	}
 
 	for testCase, data := range testCases {
+		//nolint:scopelint
 		t.Run(testCase, func(t *testing.T) {
 			ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
 			defer cancel()

chain/messagepool/block_proba.go

@@ -26,6 +26,28 @@ func noWinnersProb() []float64 {
 	return noWinnersProbCache
 }
 
+var noWinnersProbAssumingCache []float64
+var noWinnersProbAssumingOnce sync.Once
+
+func noWinnersProbAssumingMoreThanOne() []float64 {
+	noWinnersProbAssumingOnce.Do(func() {
+		cond := math.Log(-1 + math.Exp(5))
+		poissPdf := func(x float64) float64 {
+			const Mu = 5
+			lg, _ := math.Lgamma(x + 1)
+			result := math.Exp((math.Log(Mu) * x) - lg - cond)
+			return result
+		}
+
+		out := make([]float64, 0, MaxBlocks)
+		for i := 0; i < MaxBlocks; i++ {
+			out = append(out, poissPdf(float64(i+1)))
+		}
+		noWinnersProbAssumingCache = out
+	})
+	return noWinnersProbAssumingCache
+}
+
 func binomialCoefficient(n, k float64) float64 {
 	if k > n {
 		return math.NaN()
@@ -34,13 +56,13 @@ func binomialCoefficient(n, k float64) float64 {
 	for d := 1.0; d <= k; d++ {
 		r *= n
 		r /= d
-		n -= 1
+		n--
 	}
 	return r
 }
 
 func (mp *MessagePool) blockProbabilities(tq float64) []float64 {
-	noWinners := noWinnersProb() // cache this
+	noWinners := noWinnersProbAssumingMoreThanOne()
 
 	p := 1 - tq
 	binoPdf := func(x, trials float64) float64 {
@@ -72,7 +94,7 @@ func (mp *MessagePool) blockProbabilities(tq float64) []float64 {
 	for place := 0; place < MaxBlocks; place++ {
 		var pPlace float64
 		for otherWinners, pCase := range noWinners {
-			pPlace += pCase * binoPdf(float64(place), float64(otherWinners+1))
+			pPlace += pCase * binoPdf(float64(place), float64(otherWinners))
 		}
 		out = append(out, pPlace)
 	}

chain/messagepool/block_proba_test.go

@@ -1,6 +1,11 @@
 package messagepool
 
-import "testing"
+import (
+	"math"
+	"math/rand"
+	"testing"
+	"time"
+)
 
 func TestBlockProbability(t *testing.T) {
 	mp := &MessagePool{}
@@ -13,3 +18,26 @@ func TestBlockProbability(t *testing.T) {
 		}
 	}
 }
+
+func TestWinnerProba(t *testing.T) {
+	rand.Seed(time.Now().UnixNano())
+	const N = 1000000
+	winnerProba := noWinnersProb()
+	sum := 0
+	for i := 0; i < N; i++ {
+		minersRand := rand.Float64()
+		j := 0
+		for ; j < MaxBlocks; j++ {
+			minersRand -= winnerProba[j]
+			if minersRand < 0 {
+				break
+			}
+		}
+		sum += j
+	}
+
+	if avg := float64(sum) / N; math.Abs(avg-5) > 0.01 {
+		t.Fatalf("avg too far off: %f", avg)
+	}
+
+}

chain/messagepool/gasguess/guessgas.go

@@ -9,7 +9,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 )
 

chain/messagepool/messagepool.go

@@ -6,12 +6,14 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	stdbig "math/big"
 	"sort"
 	"sync"
 	"time"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/crypto"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/hashicorp/go-multierror"
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
@@ -26,6 +28,7 @@ import (
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/lib/sigs"
@@ -36,14 +39,21 @@ import (
 
 var log = logging.Logger("messagepool")
 
-const futureDebug = false
+var futureDebug = false
 
 var rbfNumBig = types.NewInt(uint64((ReplaceByFeeRatioDefault - 1) * RbfDenom))
 var rbfDenomBig = types.NewInt(RbfDenom)
 
 const RbfDenom = 256
 
-var RepublishInterval = pubsub.TimeCacheDuration + time.Duration(5*build.BlockDelaySecs+build.PropagationDelaySecs)*time.Second
+var RepublishInterval = time.Duration(10*build.BlockDelaySecs+build.PropagationDelaySecs) * time.Second
+
+var minimumBaseFee = types.NewInt(uint64(build.MinimumBaseFee))
+var baseFeeLowerBoundFactor = types.NewInt(10)
+
+var MaxActorPendingMessages = 1000
+
+var MaxNonceGap = uint64(4)
 
 var (
 	ErrMessageTooBig = errors.New("message too big")
@@ -52,14 +62,18 @@ var (
 
 	ErrNonceTooLow = errors.New("message nonce too low")
 
+	ErrGasFeeCapTooLow = errors.New("gas fee cap too low")
+
 	ErrNotEnoughFunds = errors.New("not enough funds to execute transaction")
 
 	ErrInvalidToAddr = errors.New("message had invalid to address")
 
-	ErrBroadcastAnyway  = errors.New("broadcasting message despite validation fail")
-	ErrRBFTooLowPremium = errors.New("replace by fee has too low GasPremium")
+	ErrSoftValidationFailure  = errors.New("validation failure")
+	ErrRBFTooLowPremium       = errors.New("replace by fee has too low GasPremium")
+	ErrTooManyPendingMessages = errors.New("too many pending messages for actor")
+	ErrNonceGap               = errors.New("unfulfilled nonce gap")
 
-	ErrTryAgain = errors.New("state inconsistency while signing message; please try again")
+	ErrTryAgain = errors.New("state inconsistency while pushing message; please try again")
 )
 
 const (
@@ -68,6 +82,18 @@ const (
 	localUpdates = "update"
 )
 
+// this is *temporary* mutilation until we have implemented uncapped miner penalties -- it will go
+// away in the next fork.
+var strictBaseFeeValidation = false
+
+func init() {
+	// if the republish interval is too short compared to the pubsub timecache, adjust it
+	minInterval := pubsub.TimeCacheDuration + time.Duration(build.PropagationDelaySecs)
+	if RepublishInterval < minInterval {
+		RepublishInterval = minInterval
+	}
+}
+
 type MessagePool struct {
 	lk sync.Mutex
 
@@ -75,8 +101,12 @@ type MessagePool struct {
 
 	addSema chan struct{}
 
-	closer  chan struct{}
-	repubTk *clock.Ticker
+	closer chan struct{}
+
+	repubTk      *clock.Ticker
+	repubTrigger chan struct{}
+
+	republished map[cid.Cid]struct{}
 
 	localAddrs map[address.Address]struct{}
 
@@ -112,27 +142,52 @@ type MessagePool struct {
 }
 
 type msgSet struct {
-	msgs      map[uint64]*types.SignedMessage
-	nextNonce uint64
+	msgs          map[uint64]*types.SignedMessage
+	nextNonce     uint64
+	requiredFunds *stdbig.Int
 }
 
-func newMsgSet() *msgSet {
+func newMsgSet(nonce uint64) *msgSet {
 	return &msgSet{
-		msgs: make(map[uint64]*types.SignedMessage),
+		msgs:          make(map[uint64]*types.SignedMessage),
+		nextNonce:     nonce,
+		requiredFunds: stdbig.NewInt(0),
 	}
 }
 
-func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool) (bool, error) {
-	if len(ms.msgs) == 0 || m.Message.Nonce >= ms.nextNonce {
-		ms.nextNonce = m.Message.Nonce + 1
+func ComputeMinRBF(curPrem abi.TokenAmount) abi.TokenAmount {
+	minPrice := types.BigAdd(curPrem, types.BigDiv(types.BigMul(curPrem, rbfNumBig), rbfDenomBig))
+	return types.BigAdd(minPrice, types.NewInt(1))
+}
+
+func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool, strict bool) (bool, error) {
+	nextNonce := ms.nextNonce
+	nonceGap := false
+	switch {
+	case m.Message.Nonce == nextNonce:
+		nextNonce++
+		// advance if we are filling a gap
+		for _, fillGap := ms.msgs[nextNonce]; fillGap; _, fillGap = ms.msgs[nextNonce] {
+			nextNonce++
+		}
+
+	case strict && m.Message.Nonce > nextNonce+MaxNonceGap:
+		return false, xerrors.Errorf("message nonce has too big a gap from expected nonce (Nonce: %d, nextNonce: %d): %w", m.Message.Nonce, nextNonce, ErrNonceGap)
+
+	case m.Message.Nonce > nextNonce:
+		nonceGap = true
 	}
+
 	exms, has := ms.msgs[m.Message.Nonce]
 	if has {
+		// refuse RBF if we have a gap
+		if strict && nonceGap {
+			return false, xerrors.Errorf("rejecting replace by fee because of nonce gap (Nonce: %d, nextNonce: %d): %w", m.Message.Nonce, nextNonce, ErrNonceGap)
+		}
+
 		if m.Cid() != exms.Cid() {
 			// check if RBF passes
-			minPrice := exms.Message.GasPremium
-			minPrice = types.BigAdd(minPrice, types.BigDiv(types.BigMul(minPrice, rbfNumBig), rbfDenomBig))
-			minPrice = types.BigAdd(minPrice, types.NewInt(1))
+			minPrice := ComputeMinRBF(exms.Message.GasPremium)
 			if types.BigCmp(m.Message.GasPremium, minPrice) >= 0 {
 				log.Infow("add with RBF", "oldpremium", exms.Message.GasPremium,
 					"newpremium", m.Message.GasPremium, "addr", m.Message.From, "nonce", m.Message.Nonce)
@@ -143,22 +198,87 @@ func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool) (bool, error) {
 					m.Message.From, m.Message.Nonce, minPrice, m.Message.GasPremium,
 					ErrRBFTooLowPremium)
 			}
+		} else {
+			return false, xerrors.Errorf("message from %s with nonce %d already in mpool: %w",
+				m.Message.From, m.Message.Nonce, ErrSoftValidationFailure)
 		}
+
+		ms.requiredFunds.Sub(ms.requiredFunds, exms.Message.RequiredFunds().Int)
+		//ms.requiredFunds.Sub(ms.requiredFunds, exms.Message.Value.Int)
 	}
+
+	if !has && strict && len(ms.msgs) > MaxActorPendingMessages {
+		log.Errorf("too many pending messages from actor %s", m.Message.From)
+		return false, ErrTooManyPendingMessages
+	}
+
+	if strict && nonceGap {
+		log.Warnf("adding nonce-gapped message from %s (nonce: %d, nextNonce: %d)",
+			m.Message.From, m.Message.Nonce, nextNonce)
+	}
+
+	ms.nextNonce = nextNonce
 	ms.msgs[m.Message.Nonce] = m
+	ms.requiredFunds.Add(ms.requiredFunds, m.Message.RequiredFunds().Int)
+	//ms.requiredFunds.Add(ms.requiredFunds, m.Message.Value.Int)
 
 	return !has, nil
 }
 
+func (ms *msgSet) rm(nonce uint64, applied bool) {
+	m, has := ms.msgs[nonce]
+	if !has {
+		if applied && nonce >= ms.nextNonce {
+			// we removed a message we did not know about because it was applied
+			// we need to adjust the nonce and check if we filled a gap
+			ms.nextNonce = nonce + 1
+			for _, fillGap := ms.msgs[ms.nextNonce]; fillGap; _, fillGap = ms.msgs[ms.nextNonce] {
+				ms.nextNonce++
+			}
+		}
+		return
+	}
+
+	ms.requiredFunds.Sub(ms.requiredFunds, m.Message.RequiredFunds().Int)
+	//ms.requiredFunds.Sub(ms.requiredFunds, m.Message.Value.Int)
+	delete(ms.msgs, nonce)
+
+	// adjust next nonce
+	if applied {
+		// we removed a (known) message because it was applied in a tipset
+		// we can't possibly have filled a gap in this case
+		if nonce >= ms.nextNonce {
+			ms.nextNonce = nonce + 1
+		}
+		return
+	}
+
+	// we removed a message because it was pruned
+	// we have to adjust the nonce if it creates a gap or rewinds state
+	if nonce < ms.nextNonce {
+		ms.nextNonce = nonce
+	}
+}
+
+func (ms *msgSet) getRequiredFunds(nonce uint64) types.BigInt {
+	requiredFunds := new(stdbig.Int).Set(ms.requiredFunds)
+
+	m, has := ms.msgs[nonce]
+	if has {
+		requiredFunds.Sub(requiredFunds, m.Message.RequiredFunds().Int)
+		//requiredFunds.Sub(requiredFunds, m.Message.Value.Int)
+	}
+
+	return types.BigInt{Int: requiredFunds}
+}
+
 func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*MessagePool, error) {
 	cache, _ := lru.New2Q(build.BlsSignatureCacheSize)
 	verifcache, _ := lru.New2Q(build.VerifSigCacheSize)
 
 	cfg, err := loadConfig(ds)
 	if err != nil {
-		if err != nil {
-			return nil, xerrors.Errorf("error loading mpool config: %w", err)
-		}
+		return nil, xerrors.Errorf("error loading mpool config: %w", err)
 	}
 
 	mp := &MessagePool{
@@ -166,6 +286,7 @@ func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*Messa
 		addSema:       make(chan struct{}, 1),
 		closer:        make(chan struct{}),
 		repubTk:       build.Clock.Ticker(RepublishInterval),
+		repubTrigger:  make(chan struct{}, 1),
 		localAddrs:    make(map[address.Address]struct{}),
 		pending:       make(map[address.Address]*msgSet),
 		minGasPrice:   types.NewInt(0),
@@ -183,12 +304,7 @@ func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*Messa
 	// enable initial prunes
 	mp.pruneCooldown <- struct{}{}
 
-	if err := mp.loadLocal(); err != nil {
-		log.Errorf("loading local messages: %+v", err)
-	}
-
-	go mp.runLoop()
-
+	// load the current tipset and subscribe to head changes _before_ loading local messages
 	mp.curTs = api.SubscribeHeadChanges(func(rev, app []*types.TipSet) error {
 		err := mp.HeadChange(rev, app)
 		if err != nil {
@@ -197,6 +313,12 @@ func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*Messa
 		return err
 	})
 
+	if err := mp.loadLocal(); err != nil {
+		log.Errorf("loading local messages: %+v", err)
+	}
+
+	go mp.runLoop()
+
 	return mp, nil
 }
 
@@ -224,6 +346,10 @@ func (mp *MessagePool) runLoop() {
 			if err := mp.republishPendingMessages(); err != nil {
 				log.Errorf("error while republishing messages: %s", err)
 			}
+		case <-mp.repubTrigger:
+			if err := mp.republishPendingMessages(); err != nil {
+				log.Errorf("error while republishing messages: %s", err)
+			}
 		case <-mp.pruneTrigger:
 			if err := mp.pruneExcessMessages(); err != nil {
 				log.Errorf("failed to prune excess messages from mempool: %s", err)
@@ -245,37 +371,74 @@ func (mp *MessagePool) addLocal(m *types.SignedMessage, msgb []byte) error {
 	return nil
 }
 
-func (mp *MessagePool) verifyMsgBeforePush(m *types.SignedMessage, epoch abi.ChainEpoch) error {
+// verifyMsgBeforeAdd verifies that the message meets the minimum criteria for block inclusio
+// and whether the message has enough funds to be included in the next 20 blocks.
+// If the message is not valid for block inclusion, it returns an error.
+// For local messages, if the message can be included in the next 20 blocks, it returns true to
+// signal that it should be immediately published. If the message cannot be included in the next 20
+// blocks, it returns false so that the message doesn't immediately get published (and ignored by our
+// peers); instead it will be published through the republish loop, once the base fee has fallen
+// sufficiently.
+// For non local messages, if the message cannot be included in the next 20 blocks it returns
+// a (soft) validation error.
+func (mp *MessagePool) verifyMsgBeforeAdd(m *types.SignedMessage, curTs *types.TipSet, local bool) (bool, error) {
+	epoch := curTs.Height()
 	minGas := vm.PricelistByEpoch(epoch).OnChainMessage(m.ChainLength())
 
 	if err := m.VMMessage().ValidForBlockInclusion(minGas.Total()); err != nil {
-		return xerrors.Errorf("message will not be included in a block: %w", err)
+		return false, xerrors.Errorf("message will not be included in a block: %w", err)
 	}
-	return nil
+
+	// this checks if the GasFeeCap is suffisciently high for inclusion in the next 20 blocks
+	// if the GasFeeCap is too low, we soft reject the message (Ignore in pubsub) and rely
+	// on republish to push it through later, if the baseFee has fallen.
+	// this is a defensive check that stops minimum baseFee spam attacks from overloading validation
+	// queues.
+	// Note that for local messages, we always add them so that they can be accepted and republished
+	// automatically.
+	publish := local
+	if strictBaseFeeValidation && len(curTs.Blocks()) > 0 {
+		baseFee := curTs.Blocks()[0].ParentBaseFee
+		baseFeeLowerBound := types.BigDiv(baseFee, baseFeeLowerBoundFactor)
+		if m.Message.GasFeeCap.LessThan(baseFeeLowerBound) {
+			if local {
+				log.Warnf("local message will not be immediately published because GasFeeCap doesn't meet the lower bound for inclusion in the next 20 blocks (GasFeeCap: %s, baseFeeLowerBound: %s)",
+					m.Message.GasFeeCap, baseFeeLowerBound)
+				publish = false
+			} else {
+				return false, xerrors.Errorf("GasFeeCap doesn't meet base fee lower bound for inclusion in the next 20 blocks (GasFeeCap: %s, baseFeeLowerBound: %s): %w",
+					m.Message.GasFeeCap, baseFeeLowerBound, ErrSoftValidationFailure)
+			}
+		}
+	}
+
+	return publish, nil
 }
 
 func (mp *MessagePool) Push(m *types.SignedMessage) (cid.Cid, error) {
+	err := mp.checkMessage(m)
+	if err != nil {
+		return cid.Undef, err
+	}
+
 	// serialize push access to reduce lock contention
 	mp.addSema <- struct{}{}
 	defer func() {
 		<-mp.addSema
 	}()
 
-	mp.curTsLk.Lock()
-	epoch := mp.curTs.Height()
-	mp.curTsLk.Unlock()
-	if err := mp.verifyMsgBeforePush(m, epoch); err != nil {
-		return cid.Undef, err
-	}
-
 	msgb, err := m.Serialize()
 	if err != nil {
 		return cid.Undef, err
 	}
 
-	if err := mp.Add(m); err != nil {
+	mp.curTsLk.Lock()
+	publish, err := mp.addTs(m, mp.curTs, true)
+	if err != nil {
+		mp.curTsLk.Unlock()
 		return cid.Undef, err
 	}
+	mp.curTsLk.Unlock()
 
 	mp.lk.Lock()
 	if err := mp.addLocal(m, msgb); err != nil {
@@ -284,15 +447,24 @@ func (mp *MessagePool) Push(m *types.SignedMessage) (cid.Cid, error) {
 	}
 	mp.lk.Unlock()
 
-	return m.Cid(), mp.api.PubSubPublish(build.MessagesTopic(mp.netName), msgb)
+	if publish {
+		err = mp.api.PubSubPublish(build.MessagesTopic(mp.netName), msgb)
+	}
+
+	return m.Cid(), err
 }
 
-func (mp *MessagePool) Add(m *types.SignedMessage) error {
+func (mp *MessagePool) checkMessage(m *types.SignedMessage) error {
 	// big messages are bad, anti DOS
 	if m.Size() > 32*1024 {
 		return xerrors.Errorf("mpool message too large (%dB): %w", m.Size(), ErrMessageTooBig)
 	}
 
+	// Perform syntactic validation, minGas=0 as we check the actual mingas before we add it
+	if err := m.Message.ValidForBlockInclusion(0); err != nil {
+		return xerrors.Errorf("message not valid for block inclusion: %w", err)
+	}
+
 	if m.Message.To == address.Undef {
 		return ErrInvalidToAddr
 	}
@@ -301,8 +473,21 @@ func (mp *MessagePool) Add(m *types.SignedMessage) error {
 		return ErrMessageValueTooHigh
 	}
 
+	if m.Message.GasFeeCap.LessThan(minimumBaseFee) {
+		return ErrGasFeeCapTooLow
+	}
+
 	if err := mp.VerifyMsgSig(m); err != nil {
-		log.Warnf("mpooladd signature verification failed: %s", err)
+		log.Warnf("signature verification failed: %s", err)
+		return err
+	}
+
+	return nil
+}
+
+func (mp *MessagePool) Add(m *types.SignedMessage) error {
+	err := mp.checkMessage(m)
+	if err != nil {
 		return err
 	}
 
@@ -314,7 +499,9 @@ func (mp *MessagePool) Add(m *types.SignedMessage) error {
 
 	mp.curTsLk.Lock()
 	defer mp.curTsLk.Unlock()
-	return mp.addTs(m, mp.curTs)
+
+	_, err = mp.addTs(m, mp.curTs, false)
+	return err
 }
 
 func sigCacheKey(m *types.SignedMessage) (string, error) {
@@ -353,48 +540,107 @@ func (mp *MessagePool) VerifyMsgSig(m *types.SignedMessage) error {
 	return nil
 }
 
-func (mp *MessagePool) addTs(m *types.SignedMessage, curTs *types.TipSet) error {
+func (mp *MessagePool) checkBalance(m *types.SignedMessage, curTs *types.TipSet) error {
+	balance, err := mp.getStateBalance(m.Message.From, curTs)
+	if err != nil {
+		return xerrors.Errorf("failed to check sender balance: %s: %w", err, ErrSoftValidationFailure)
+	}
+
+	requiredFunds := m.Message.RequiredFunds()
+	if balance.LessThan(requiredFunds) {
+		return xerrors.Errorf("not enough funds (required: %s, balance: %s): %w", types.FIL(requiredFunds), types.FIL(balance), ErrNotEnoughFunds)
+	}
+
+	// add Value for soft failure check
+	//requiredFunds = types.BigAdd(requiredFunds, m.Message.Value)
+
+	mset, ok := mp.pending[m.Message.From]
+	if ok {
+		requiredFunds = types.BigAdd(requiredFunds, mset.getRequiredFunds(m.Message.Nonce))
+	}
+
+	if balance.LessThan(requiredFunds) {
+		// Note: we fail here for ErrSoftValidationFailure to signal a soft failure because we might
+		// be out of sync.
+		return xerrors.Errorf("not enough funds including pending messages (required: %s, balance: %s): %w", types.FIL(requiredFunds), types.FIL(balance), ErrSoftValidationFailure)
+	}
+
+	return nil
+}
+
+func (mp *MessagePool) addTs(m *types.SignedMessage, curTs *types.TipSet, local bool) (bool, error) {
 	snonce, err := mp.getStateNonce(m.Message.From, curTs)
 	if err != nil {
-		return xerrors.Errorf("failed to look up actor state nonce: %s: %w", err, ErrBroadcastAnyway)
+		return false, xerrors.Errorf("failed to look up actor state nonce: %s: %w", err, ErrSoftValidationFailure)
+	}
+
+	if snonce > m.Message.Nonce {
+		return false, xerrors.Errorf("minimum expected nonce is %d: %w", snonce, ErrNonceTooLow)
+	}
+
+	mp.lk.Lock()
+	defer mp.lk.Unlock()
+
+	publish, err := mp.verifyMsgBeforeAdd(m, curTs, local)
+	if err != nil {
+		return false, err
+	}
+
+	if err := mp.checkBalance(m, curTs); err != nil {
+		return false, err
+	}
+
+	return publish, mp.addLocked(m, !local)
+}
+
+func (mp *MessagePool) addLoaded(m *types.SignedMessage) error {
+	err := mp.checkMessage(m)
+	if err != nil {
+		return err
+	}
+
+	mp.curTsLk.Lock()
+	defer mp.curTsLk.Unlock()
+
+	curTs := mp.curTs
+
+	snonce, err := mp.getStateNonce(m.Message.From, curTs)
+	if err != nil {
+		return xerrors.Errorf("failed to look up actor state nonce: %s: %w", err, ErrSoftValidationFailure)
 	}
 
 	if snonce > m.Message.Nonce {
 		return xerrors.Errorf("minimum expected nonce is %d: %w", snonce, ErrNonceTooLow)
 	}
 
-	balance, err := mp.getStateBalance(m.Message.From, curTs)
-	if err != nil {
-		return xerrors.Errorf("failed to check sender balance: %s: %w", err, ErrBroadcastAnyway)
-	}
-
-	if balance.LessThan(m.Message.RequiredFunds()) {
-		return xerrors.Errorf("not enough funds (required: %s, balance: %s): %w", types.FIL(m.Message.RequiredFunds()), types.FIL(balance), ErrNotEnoughFunds)
-	}
-
 	mp.lk.Lock()
 	defer mp.lk.Unlock()
 
-	return mp.addLocked(m)
+	_, err = mp.verifyMsgBeforeAdd(m, curTs, true)
+	if err != nil {
+		return err
+	}
+
+	if err := mp.checkBalance(m, curTs); err != nil {
+		return err
+	}
+
+	return mp.addLocked(m, false)
 }
 
 func (mp *MessagePool) addSkipChecks(m *types.SignedMessage) error {
 	mp.lk.Lock()
 	defer mp.lk.Unlock()
 
-	return mp.addLocked(m)
+	return mp.addLocked(m, false)
 }
 
-func (mp *MessagePool) addLocked(m *types.SignedMessage) error {
+func (mp *MessagePool) addLocked(m *types.SignedMessage, strict bool) error {
 	log.Debugf("mpooladd: %s %d", m.Message.From, m.Message.Nonce)
 	if m.Signature.Type == crypto.SigTypeBLS {
 		mp.blsSigCache.Add(m.Cid(), m.Signature)
 	}
 
-	if m.Message.GasLimit > build.BlockGasLimit {
-		return xerrors.Errorf("given message has too high of a gas limit")
-	}
-
 	if _, err := mp.api.PutMessage(m); err != nil {
 		log.Warnf("mpooladd cs.PutMessage failed: %s", err)
 		return err
@@ -407,13 +653,18 @@ func (mp *MessagePool) addLocked(m *types.SignedMessage) error {
 
 	mset, ok := mp.pending[m.Message.From]
 	if !ok {
-		mset = newMsgSet()
+		nonce, err := mp.getStateNonce(m.Message.From, mp.curTs)
+		if err != nil {
+			return xerrors.Errorf("failed to get initial actor nonce: %w", err)
+		}
+
+		mset = newMsgSet(nonce)
 		mp.pending[m.Message.From] = mset
 	}
 
-	incr, err := mset.add(m, mp)
+	incr, err := mset.add(m, mp, strict)
 	if err != nil {
-		log.Info(err)
+		log.Debug(err)
 		return err
 	}
 
@@ -466,42 +717,16 @@ func (mp *MessagePool) getNonceLocked(addr address.Address, curTs *types.TipSet)
 }
 
 func (mp *MessagePool) getStateNonce(addr address.Address, curTs *types.TipSet) (uint64, error) {
-	// TODO: this method probably should be cached
-
-	act, err := mp.api.StateGetActor(addr, curTs)
+	act, err := mp.api.GetActorAfter(addr, curTs)
 	if err != nil {
 		return 0, err
 	}
 
-	baseNonce := act.Nonce
-
-	// TODO: the correct thing to do here is probably to set curTs to chain.head
-	// but since we have an accurate view of the world until a head change occurs,
-	// this should be fine
-	if curTs == nil {
-		return baseNonce, nil
-	}
-
-	msgs, err := mp.api.MessagesForTipset(curTs)
-	if err != nil {
-		return 0, xerrors.Errorf("failed to check messages for tipset: %w", err)
-	}
-
-	for _, m := range msgs {
-		msg := m.VMMessage()
-		if msg.From == addr {
-			if msg.Nonce != baseNonce {
-				return 0, xerrors.Errorf("tipset %s has bad nonce ordering (%d != %d)", curTs.Cids(), msg.Nonce, baseNonce)
-			}
-			baseNonce++
-		}
-	}
-
-	return baseNonce, nil
+	return act.Nonce, nil
 }
 
 func (mp *MessagePool) getStateBalance(addr address.Address, ts *types.TipSet) (types.BigInt, error) {
-	act, err := mp.api.StateGetActor(addr, ts)
+	act, err := mp.api.GetActorAfter(addr, ts)
 	if err != nil {
 		return types.EmptyInt, err
 	}
@@ -548,6 +773,16 @@ func (mp *MessagePool) PushWithNonce(ctx context.Context, addr address.Address,
 		return nil, err
 	}
 
+	err = mp.checkMessage(msg)
+	if err != nil {
+		return nil, err
+	}
+
+	msgb, err := msg.Serialize()
+	if err != nil {
+		return nil, err
+	}
+
 	// reacquire the locks and check state for consistency
 	mp.curTsLk.Lock()
 	defer mp.curTsLk.Unlock()
@@ -568,33 +803,37 @@ func (mp *MessagePool) PushWithNonce(ctx context.Context, addr address.Address,
 		return nil, ErrTryAgain
 	}
 
-	if err := mp.verifyMsgBeforePush(msg, mp.curTs.Height()); err != nil {
-		return nil, err
-	}
-
-	msgb, err := msg.Serialize()
+	publish, err := mp.verifyMsgBeforeAdd(msg, curTs, true)
 	if err != nil {
 		return nil, err
 	}
 
-	if err := mp.addLocked(msg); err != nil {
+	if err := mp.checkBalance(msg, curTs); err != nil {
+		return nil, err
+	}
+
+	if err := mp.addLocked(msg, false); err != nil {
 		return nil, xerrors.Errorf("add locked failed: %w", err)
 	}
 	if err := mp.addLocal(msg, msgb); err != nil {
 		log.Errorf("addLocal failed: %+v", err)
 	}
 
-	return msg, mp.api.PubSubPublish(build.MessagesTopic(mp.netName), msgb)
+	if publish {
+		err = mp.api.PubSubPublish(build.MessagesTopic(mp.netName), msgb)
+	}
+
+	return msg, err
 }
 
-func (mp *MessagePool) Remove(from address.Address, nonce uint64) {
+func (mp *MessagePool) Remove(from address.Address, nonce uint64, applied bool) {
 	mp.lk.Lock()
 	defer mp.lk.Unlock()
 
-	mp.remove(from, nonce)
+	mp.remove(from, nonce, applied)
 }
 
-func (mp *MessagePool) remove(from address.Address, nonce uint64) {
+func (mp *MessagePool) remove(from address.Address, nonce uint64, applied bool) {
 	mset, ok := mp.pending[from]
 	if !ok {
 		return
@@ -611,22 +850,10 @@ func (mp *MessagePool) remove(from address.Address, nonce uint64) {
 
 	// NB: This deletes any message with the given nonce. This makes sense
 	// as two messages with the same sender cannot have the same nonce
-	delete(mset.msgs, nonce)
+	mset.rm(nonce, applied)
 
 	if len(mset.msgs) == 0 {
 		delete(mp.pending, from)
-	} else {
-		var max uint64
-		for nonce := range mset.msgs {
-			if max < nonce {
-				max = nonce
-			}
-		}
-		if max < nonce {
-			max = nonce // we could have not seen the removed message before
-		}
-
-		mset.nextNonce = max + 1
 	}
 }
 
@@ -637,6 +864,10 @@ func (mp *MessagePool) Pending() ([]*types.SignedMessage, *types.TipSet) {
 	mp.lk.Lock()
 	defer mp.lk.Unlock()
 
+	return mp.allPending()
+}
+
+func (mp *MessagePool) allPending() ([]*types.SignedMessage, *types.TipSet) {
 	out := make([]*types.SignedMessage, 0)
 	for a := range mp.pending {
 		out = append(out, mp.pendingFor(a)...)
@@ -644,6 +875,7 @@ func (mp *MessagePool) Pending() ([]*types.SignedMessage, *types.TipSet) {
 
 	return out, mp.curTs
 }
+
 func (mp *MessagePool) PendingFor(a address.Address) ([]*types.SignedMessage, *types.TipSet) {
 	mp.curTsLk.Lock()
 	defer mp.curTsLk.Unlock()
@@ -676,6 +908,7 @@ func (mp *MessagePool) HeadChange(revert []*types.TipSet, apply []*types.TipSet)
 	mp.curTsLk.Lock()
 	defer mp.curTsLk.Unlock()
 
+	repubTrigger := false
 	rmsgs := make(map[address.Address]map[uint64]*types.SignedMessage)
 	add := func(m *types.SignedMessage) {
 		s, ok := rmsgs[m.Message.From]
@@ -688,7 +921,7 @@ func (mp *MessagePool) HeadChange(revert []*types.TipSet, apply []*types.TipSet)
 	rm := func(from address.Address, nonce uint64) {
 		s, ok := rmsgs[from]
 		if !ok {
-			mp.Remove(from, nonce)
+			mp.Remove(from, nonce, true)
 			return
 		}
 
@@ -697,43 +930,73 @@ func (mp *MessagePool) HeadChange(revert []*types.TipSet, apply []*types.TipSet)
 			return
 		}
 
-		mp.Remove(from, nonce)
+		mp.Remove(from, nonce, true)
 	}
 
+	maybeRepub := func(cid cid.Cid) {
+		if !repubTrigger {
+			mp.lk.Lock()
+			_, republished := mp.republished[cid]
+			mp.lk.Unlock()
+			if republished {
+				repubTrigger = true
+			}
+		}
+	}
+
+	var merr error
+
 	for _, ts := range revert {
 		pts, err := mp.api.LoadTipSet(ts.Parents())
 		if err != nil {
-			return err
-		}
-
-		msgs, err := mp.MessagesForBlocks(ts.Blocks())
-		if err != nil {
-			return err
+			log.Errorf("error loading reverted tipset parent: %s", err)
+			merr = multierror.Append(merr, err)
+			continue
 		}
 
 		mp.curTs = pts
 
+		msgs, err := mp.MessagesForBlocks(ts.Blocks())
+		if err != nil {
+			log.Errorf("error retrieving messages for reverted block: %s", err)
+			merr = multierror.Append(merr, err)
+			continue
+		}
+
 		for _, msg := range msgs {
 			add(msg)
 		}
 	}
 
 	for _, ts := range apply {
+		mp.curTs = ts
+
 		for _, b := range ts.Blocks() {
 			bmsgs, smsgs, err := mp.api.MessagesForBlock(b)
 			if err != nil {
-				return xerrors.Errorf("failed to get messages for apply block %s(height %d) (msgroot = %s): %w", b.Cid(), b.Height, b.Messages, err)
+				xerr := xerrors.Errorf("failed to get messages for apply block %s(height %d) (msgroot = %s): %w", b.Cid(), b.Height, b.Messages, err)
+				log.Errorf("error retrieving messages for block: %s", xerr)
+				merr = multierror.Append(merr, xerr)
+				continue
 			}
+
 			for _, msg := range smsgs {
 				rm(msg.Message.From, msg.Message.Nonce)
+				maybeRepub(msg.Cid())
 			}
 
 			for _, msg := range bmsgs {
 				rm(msg.From, msg.Nonce)
+				maybeRepub(msg.Cid())
 			}
 		}
+	}
 
-		mp.curTs = ts
+	if repubTrigger {
+		select {
+		case mp.repubTrigger <- struct{}{}:
+		default:
+		}
 	}
 
 	for _, s := range rmsgs {
@@ -745,7 +1008,9 @@ func (mp *MessagePool) HeadChange(revert []*types.TipSet, apply []*types.TipSet)
 	}
 
 	if len(revert) > 0 && futureDebug {
-		msgs, ts := mp.Pending()
+		mp.lk.Lock()
+		msgs, ts := mp.allPending()
+		mp.lk.Unlock()
 
 		buckets := map[address.Address]*statBucket{}
 
@@ -762,7 +1027,8 @@ func (mp *MessagePool) HeadChange(revert []*types.TipSet, apply []*types.TipSet)
 		}
 
 		for a, bkt := range buckets {
-			act, err := mp.api.StateGetActor(a, ts)
+			// TODO that might not be correct with GatActorAfter but it is only debug code
+			act, err := mp.api.GetActorAfter(a, ts)
 			if err != nil {
 				log.Debugf("%s, err: %s\n", a, err)
 				continue
@@ -809,7 +1075,72 @@ func (mp *MessagePool) HeadChange(revert []*types.TipSet, apply []*types.TipSet)
 		}
 	}
 
-	return nil
+	return merr
+}
+
+func (mp *MessagePool) runHeadChange(from *types.TipSet, to *types.TipSet, rmsgs map[address.Address]map[uint64]*types.SignedMessage) error {
+	add := func(m *types.SignedMessage) {
+		s, ok := rmsgs[m.Message.From]
+		if !ok {
+			s = make(map[uint64]*types.SignedMessage)
+			rmsgs[m.Message.From] = s
+		}
+		s[m.Message.Nonce] = m
+	}
+	rm := func(from address.Address, nonce uint64) {
+		s, ok := rmsgs[from]
+		if !ok {
+			return
+		}
+
+		if _, ok := s[nonce]; ok {
+			delete(s, nonce)
+			return
+		}
+
+	}
+
+	revert, apply, err := store.ReorgOps(mp.api.LoadTipSet, from, to)
+	if err != nil {
+		return xerrors.Errorf("failed to compute reorg ops for mpool pending messages: %w", err)
+	}
+
+	var merr error
+
+	for _, ts := range revert {
+		msgs, err := mp.MessagesForBlocks(ts.Blocks())
+		if err != nil {
+			log.Errorf("error retrieving messages for reverted block: %s", err)
+			merr = multierror.Append(merr, err)
+			continue
+		}
+
+		for _, msg := range msgs {
+			add(msg)
+		}
+	}
+
+	for _, ts := range apply {
+		for _, b := range ts.Blocks() {
+			bmsgs, smsgs, err := mp.api.MessagesForBlock(b)
+			if err != nil {
+				xerr := xerrors.Errorf("failed to get messages for apply block %s(height %d) (msgroot = %s): %w", b.Cid(), b.Height, b.Messages, err)
+				log.Errorf("error retrieving messages for block: %s", xerr)
+				merr = multierror.Append(merr, xerr)
+				continue
+			}
+
+			for _, msg := range smsgs {
+				rm(msg.Message.From, msg.Message.Nonce)
+			}
+
+			for _, msg := range bmsgs {
+				rm(msg.From, msg.Nonce)
+			}
+		}
+	}
+
+	return merr
 }
 
 type statBucket struct {
@@ -862,6 +1193,7 @@ func (mp *MessagePool) Updates(ctx context.Context) (<-chan api.MpoolUpdate, err
 
 	go func() {
 		defer mp.changes.Unsub(sub, localUpdates)
+		defer close(out)
 
 		for {
 			select {
@@ -870,9 +1202,13 @@ func (mp *MessagePool) Updates(ctx context.Context) (<-chan api.MpoolUpdate, err
 				case out <- u.(api.MpoolUpdate):
 				case <-ctx.Done():
 					return
+				case <-mp.closer:
+					return
 				}
 			case <-ctx.Done():
 				return
+			case <-mp.closer:
+				return
 			}
 		}
 	}()
@@ -896,7 +1232,7 @@ func (mp *MessagePool) loadLocal() error {
 			return xerrors.Errorf("unmarshaling local message: %w", err)
 		}
 
-		if err := mp.Add(&sm); err != nil {
+		if err := mp.addLoaded(&sm); err != nil {
 			if xerrors.Is(err, ErrNonceTooLow) {
 				continue // todo: drop the message from local cache (if above certain confidence threshold)
 			}
@@ -909,3 +1245,40 @@ func (mp *MessagePool) loadLocal() error {
 
 	return nil
 }
+
+func (mp *MessagePool) Clear(local bool) {
+	mp.lk.Lock()
+	defer mp.lk.Unlock()
+
+	// remove everything if local is true, including removing local messages from
+	// the datastore
+	if local {
+		for a := range mp.localAddrs {
+			mset, ok := mp.pending[a]
+			if !ok {
+				continue
+			}
+
+			for _, m := range mset.msgs {
+				err := mp.localMsgs.Delete(datastore.NewKey(string(m.Cid().Bytes())))
+				if err != nil {
+					log.Warnf("error deleting local message: %s", err)
+				}
+			}
+		}
+
+		mp.pending = make(map[address.Address]*msgSet)
+		mp.republished = nil
+
+		return
+	}
+
+	// remove everything except the local messages
+	for a := range mp.pending {
+		_, isLocal := mp.localAddrs[a]
+		if isLocal {
+			continue
+		}
+		delete(mp.pending, a)
+	}
+}

chain/messagepool/messagepool_test.go

@@ -3,16 +3,19 @@ package messagepool
 import (
 	"context"
 	"fmt"
+	"sort"
 	"testing"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/lotus/chain/messagepool/gasguess"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/types/mock"
 	"github.com/filecoin-project/lotus/chain/wallet"
 	_ "github.com/filecoin-project/lotus/lib/sigs/bls"
 	_ "github.com/filecoin-project/lotus/lib/sigs/secp"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
-	"github.com/filecoin-project/specs-actors/actors/crypto"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
 	logging "github.com/ipfs/go-log/v2"
@@ -30,14 +33,35 @@ type testMpoolAPI struct {
 	balance    map[address.Address]types.BigInt
 
 	tipsets []*types.TipSet
+
+	published int
+
+	baseFee types.BigInt
 }
 
 func newTestMpoolAPI() *testMpoolAPI {
-	return &testMpoolAPI{
+	tma := &testMpoolAPI{
 		bmsgs:      make(map[cid.Cid][]*types.SignedMessage),
 		statenonce: make(map[address.Address]uint64),
 		balance:    make(map[address.Address]types.BigInt),
+		baseFee:    types.NewInt(100),
 	}
+	genesis := mock.MkBlock(nil, 1, 1)
+	tma.tipsets = append(tma.tipsets, mock.TipSet(genesis))
+	return tma
+}
+
+func (tma *testMpoolAPI) nextBlock() *types.BlockHeader {
+	newBlk := mock.MkBlock(tma.tipsets[len(tma.tipsets)-1], 1, 1)
+	tma.tipsets = append(tma.tipsets, mock.TipSet(newBlk))
+	return newBlk
+}
+
+func (tma *testMpoolAPI) nextBlockWithHeight(height uint64) *types.BlockHeader {
+	newBlk := mock.MkBlock(tma.tipsets[len(tma.tipsets)-1], 1, 1)
+	newBlk.Height = abi.ChainEpoch(height)
+	tma.tipsets = append(tma.tipsets, mock.TipSet(newBlk))
+	return newBlk
 }
 
 func (tma *testMpoolAPI) applyBlock(t *testing.T, b *types.BlockHeader) {
@@ -68,12 +92,11 @@ func (tma *testMpoolAPI) setBalanceRaw(addr address.Address, v types.BigInt) {
 
 func (tma *testMpoolAPI) setBlockMessages(h *types.BlockHeader, msgs ...*types.SignedMessage) {
 	tma.bmsgs[h.Cid()] = msgs
-	tma.tipsets = append(tma.tipsets, mock.TipSet(h))
 }
 
 func (tma *testMpoolAPI) SubscribeHeadChanges(cb func(rev, app []*types.TipSet) error) *types.TipSet {
 	tma.cb = cb
-	return nil
+	return tma.tipsets[0]
 }
 
 func (tma *testMpoolAPI) PutMessage(m types.ChainMsg) (cid.Cid, error) {
@@ -81,18 +104,47 @@ func (tma *testMpoolAPI) PutMessage(m types.ChainMsg) (cid.Cid, error) {
 }
 
 func (tma *testMpoolAPI) PubSubPublish(string, []byte) error {
+	tma.published++
 	return nil
 }
 
-func (tma *testMpoolAPI) StateGetActor(addr address.Address, ts *types.TipSet) (*types.Actor, error) {
+func (tma *testMpoolAPI) GetActorAfter(addr address.Address, ts *types.TipSet) (*types.Actor, error) {
+	// regression check for load bug
+	if ts == nil {
+		panic("GetActorAfter called with nil tipset")
+	}
+
 	balance, ok := tma.balance[addr]
 	if !ok {
 		balance = types.NewInt(1000e6)
 		tma.balance[addr] = balance
 	}
+
+	msgs := make([]*types.SignedMessage, 0)
+	for _, b := range ts.Blocks() {
+		for _, m := range tma.bmsgs[b.Cid()] {
+			if m.Message.From == addr {
+				msgs = append(msgs, m)
+			}
+		}
+	}
+
+	sort.Slice(msgs, func(i, j int) bool {
+		return msgs[i].Message.Nonce < msgs[j].Message.Nonce
+	})
+
+	nonce := tma.statenonce[addr]
+
+	for _, m := range msgs {
+		if m.Message.Nonce != nonce {
+			break
+		}
+		nonce++
+	}
+
 	return &types.Actor{
 		Code:    builtin.StorageMarketActorCodeID,
-		Nonce:   tma.statenonce[addr],
+		Nonce:   nonce,
 		Balance: balance,
 	}, nil
 }
@@ -141,7 +193,7 @@ func (tma *testMpoolAPI) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error)
 }
 
 func (tma *testMpoolAPI) ChainComputeBaseFee(ctx context.Context, ts *types.TipSet) (types.BigInt, error) {
-	return types.NewInt(100), nil
+	return tma.baseFee, nil
 }
 
 func assertNonce(t *testing.T, mp *MessagePool, addr address.Address, val uint64) {
@@ -178,7 +230,7 @@ func TestMessagePool(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	a := mock.MkBlock(nil, 1, 1)
+	a := tma.nextBlock()
 
 	sender, err := w.GenerateKey(crypto.SigTypeBLS)
 	if err != nil {
@@ -204,7 +256,7 @@ func TestMessagePool(t *testing.T) {
 	assertNonce(t, mp, sender, 2)
 }
 
-func TestRevertMessages(t *testing.T) {
+func TestMessagePoolMessagesInEachBlock(t *testing.T) {
 	tma := newTestMpoolAPI()
 
 	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
@@ -219,8 +271,57 @@ func TestRevertMessages(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	a := mock.MkBlock(nil, 1, 1)
-	b := mock.MkBlock(mock.TipSet(a), 1, 1)
+	a := tma.nextBlock()
+
+	sender, err := w.GenerateKey(crypto.SigTypeBLS)
+	if err != nil {
+		t.Fatal(err)
+	}
+	target := mock.Address(1001)
+
+	var msgs []*types.SignedMessage
+	for i := 0; i < 5; i++ {
+		m := mock.MkMessage(sender, target, uint64(i), w)
+		msgs = append(msgs, m)
+		mustAdd(t, mp, m)
+	}
+
+	tma.setStateNonce(sender, 0)
+
+	tma.setBlockMessages(a, msgs[0], msgs[1])
+	tma.applyBlock(t, a)
+	tsa := mock.TipSet(a)
+
+	_, _ = mp.Pending()
+
+	selm, _ := mp.SelectMessages(tsa, 1)
+	if len(selm) == 0 {
+		t.Fatal("should have returned the rest of the messages")
+	}
+}
+
+func TestRevertMessages(t *testing.T) {
+	futureDebug = true
+	defer func() {
+		futureDebug = false
+	}()
+
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(tma, ds, "mptest")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a := tma.nextBlock()
+	b := tma.nextBlock()
 
 	sender, err := w.GenerateKey(crypto.SigTypeBLS)
 	if err != nil {
@@ -254,6 +355,7 @@ func TestRevertMessages(t *testing.T) {
 	assertNonce(t, mp, sender, 4)
 
 	p, _ := mp.Pending()
+	fmt.Printf("%+v\n", p)
 	if len(p) != 3 {
 		t.Fatal("expected three messages in mempool")
 	}
@@ -261,6 +363,12 @@ func TestRevertMessages(t *testing.T) {
 }
 
 func TestPruningSimple(t *testing.T) {
+	oldMaxNonceGap := MaxNonceGap
+	MaxNonceGap = 1000
+	defer func() {
+		MaxNonceGap = oldMaxNonceGap
+	}()
+
 	tma := newTestMpoolAPI()
 
 	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
@@ -275,13 +383,14 @@ func TestPruningSimple(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	a := mock.MkBlock(nil, 1, 1)
+	a := tma.nextBlock()
 	tma.applyBlock(t, a)
 
 	sender, err := w.GenerateKey(crypto.SigTypeBLS)
 	if err != nil {
 		t.Fatal(err)
 	}
+	tma.setBalance(sender, 1) // in FIL
 	target := mock.Address(1001)
 
 	for i := 0; i < 5; i++ {
@@ -308,3 +417,257 @@ func TestPruningSimple(t *testing.T) {
 		t.Fatal("expected only 5 messages in pool, got: ", len(msgs))
 	}
 }
+
+func TestLoadLocal(t *testing.T) {
+	tma := newTestMpoolAPI()
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(tma, ds, "mptest")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// the actors
+	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(a1, 1) // in FIL
+	tma.setBalance(a2, 1) // in FIL
+	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
+	msgs := make(map[cid.Cid]struct{})
+	for i := 0; i < 10; i++ {
+		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		cid, err := mp.Push(m)
+		if err != nil {
+			t.Fatal(err)
+		}
+		msgs[cid] = struct{}{}
+	}
+	err = mp.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mp, err = New(tma, ds, "mptest")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pmsgs, _ := mp.Pending()
+	if len(msgs) != len(pmsgs) {
+		t.Fatalf("expected %d messages, but got %d", len(msgs), len(pmsgs))
+	}
+
+	for _, m := range pmsgs {
+		cid := m.Cid()
+		_, ok := msgs[cid]
+		if !ok {
+			t.Fatal("unknown message")
+		}
+
+		delete(msgs, cid)
+	}
+
+	if len(msgs) > 0 {
+		t.Fatalf("not all messages were laoded; missing %d messages", len(msgs))
+	}
+}
+
+func TestClearAll(t *testing.T) {
+	tma := newTestMpoolAPI()
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(tma, ds, "mptest")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// the actors
+	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(a1, 1) // in FIL
+	tma.setBalance(a2, 1) // in FIL
+	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
+	for i := 0; i < 10; i++ {
+		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		_, err := mp.Push(m)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	for i := 0; i < 10; i++ {
+		m := makeTestMessage(w2, a2, a1, uint64(i), gasLimit, uint64(i+1))
+		mustAdd(t, mp, m)
+	}
+
+	mp.Clear(true)
+
+	pending, _ := mp.Pending()
+	if len(pending) > 0 {
+		t.Fatalf("cleared the mpool, but got %d pending messages", len(pending))
+	}
+}
+
+func TestClearNonLocal(t *testing.T) {
+	tma := newTestMpoolAPI()
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(tma, ds, "mptest")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// the actors
+	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(a1, 1) // in FIL
+	tma.setBalance(a2, 1) // in FIL
+
+	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
+	for i := 0; i < 10; i++ {
+		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		_, err := mp.Push(m)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	for i := 0; i < 10; i++ {
+		m := makeTestMessage(w2, a2, a1, uint64(i), gasLimit, uint64(i+1))
+		mustAdd(t, mp, m)
+	}
+
+	mp.Clear(false)
+
+	pending, _ := mp.Pending()
+	if len(pending) != 10 {
+		t.Fatalf("expected 10 pending messages, but got %d instead", len(pending))
+	}
+
+	for _, m := range pending {
+		if m.Message.From != a1 {
+			t.Fatalf("expected message from %s but got one from %s instead", a1, m.Message.From)
+		}
+	}
+}
+
+func TestUpdates(t *testing.T) {
+	tma := newTestMpoolAPI()
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(tma, ds, "mptest")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// the actors
+	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx, cancel := context.WithCancel(context.TODO())
+	defer cancel()
+
+	ch, err := mp.Updates(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
+
+	tma.setBalance(a1, 1) // in FIL
+	tma.setBalance(a2, 1) // in FIL
+
+	for i := 0; i < 10; i++ {
+		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		_, err := mp.Push(m)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		_, ok := <-ch
+		if !ok {
+			t.Fatal("expected update, but got a closed channel instead")
+		}
+	}
+
+	err = mp.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, ok := <-ch
+	if ok {
+		t.Fatal("expected closed channel, but got an update instead")
+	}
+}

chain/messagepool/provider.go

@@ -16,7 +16,7 @@ type Provider interface {
 	SubscribeHeadChanges(func(rev, app []*types.TipSet) error) *types.TipSet
 	PutMessage(m types.ChainMsg) (cid.Cid, error)
 	PubSubPublish(string, []byte) error
-	StateGetActor(address.Address, *types.TipSet) (*types.Actor, error)
+	GetActorAfter(address.Address, *types.TipSet) (*types.Actor, error)
 	StateAccountKey(context.Context, address.Address, *types.TipSet) (address.Address, error)
 	MessagesForBlock(*types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error)
 	MessagesForTipset(*types.TipSet) ([]types.ChainMsg, error)
@@ -43,12 +43,17 @@ func (mpp *mpoolProvider) PutMessage(m types.ChainMsg) (cid.Cid, error) {
 }
 
 func (mpp *mpoolProvider) PubSubPublish(k string, v []byte) error {
-	return mpp.ps.Publish(k, v)
+	return mpp.ps.Publish(k, v) //nolint
 }
 
-func (mpp *mpoolProvider) StateGetActor(addr address.Address, ts *types.TipSet) (*types.Actor, error) {
+func (mpp *mpoolProvider) GetActorAfter(addr address.Address, ts *types.TipSet) (*types.Actor, error) {
 	var act types.Actor
-	return &act, mpp.sm.WithParentState(ts, mpp.sm.WithActor(addr, stmgr.GetActor(&act)))
+	stcid, _, err := mpp.sm.TipSetState(context.TODO(), ts)
+	if err != nil {
+		return nil, xerrors.Errorf("computing tipset state for GetActor: %w", err)
+	}
+
+	return &act, mpp.sm.WithStateTree(stcid, mpp.sm.WithActor(addr, stmgr.GetActor(&act)))
 }
 
 func (mpp *mpoolProvider) StateAccountKey(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) {

chain/messagepool/pruning.go

@@ -98,7 +98,7 @@ keepLoop:
 	// and remove all messages that are still in pruneMsgs after processing the chains
 	log.Infof("Pruning %d messages", len(pruneMsgs))
 	for _, m := range pruneMsgs {
-		mp.remove(m.Message.From, m.Message.Nonce)
+		mp.remove(m.Message.From, m.Message.Nonce, false)
 	}
 
 	return nil

chain/messagepool/repub.go

@@ -3,6 +3,7 @@ package messagepool
 import (
 	"context"
 	"sort"
+	"time"
 
 	"golang.org/x/xerrors"
 
@@ -10,10 +11,13 @@ import (
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/messagepool/gasguess"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/ipfs/go-cid"
 )
 
 const repubMsgLimit = 30
 
+var RepublishBatchDelay = 100 * time.Millisecond
+
 func (mp *MessagePool) republishPendingMessages() error {
 	mp.curTsLk.Lock()
 	ts := mp.curTs
@@ -24,8 +28,14 @@ func (mp *MessagePool) republishPendingMessages() error {
 		return xerrors.Errorf("computing basefee: %w", err)
 	}
 
+	baseFeeLowerBound := types.BigDiv(baseFee, baseFeeLowerBoundFactor)
+	if baseFeeLowerBoundFactor.LessThan(minimumBaseFee) {
+		baseFeeLowerBound = minimumBaseFee
+	}
+
 	pending := make(map[address.Address]map[uint64]*types.SignedMessage)
 	mp.lk.Lock()
+	mp.republished = nil // clear this to avoid races triggering an early republish
 	for actor := range mp.localAddrs {
 		mset, ok := mp.pending[actor]
 		if !ok {
@@ -50,7 +60,11 @@ func (mp *MessagePool) republishPendingMessages() error {
 
 	var chains []*msgChain
 	for actor, mset := range pending {
-		next := mp.createMessageChains(actor, mset, baseFee, ts)
+		// We use the baseFee lower bound for createChange so that we optimistically include
+		// chains that might become profitable in the next 20 blocks.
+		// We still check the lowerBound condition for individual messages so that we don't send
+		// messages that will be rejected by the mpool spam protector, so this is safe to do.
+		next := mp.createMessageChains(actor, mset, baseFeeLowerBound, ts)
 		chains = append(chains, next...)
 	}
 
@@ -62,15 +76,10 @@ func (mp *MessagePool) republishPendingMessages() error {
 		return chains[i].Before(chains[j])
 	})
 
-	// we don't republish negative performing chains; this is an error that will be screamed
-	// at the user
-	if chains[0].gasPerf < 0 {
-		return xerrors.Errorf("skipping republish: all message chains have negative gas performance; best gas performance: %f", chains[0].gasPerf)
-	}
-
 	gasLimit := int64(build.BlockGasLimit)
 	minGas := int64(gasguess.MinGas)
 	var msgs []*types.SignedMessage
+loop:
 	for i := 0; i < len(chains); {
 		chain := chains[i]
 
@@ -84,12 +93,6 @@ func (mp *MessagePool) republishPendingMessages() error {
 			break
 		}
 
-		// we don't republish negative performing chains, as they won't be included in
-		// a block anyway
-		if chain.gasPerf < 0 {
-			break
-		}
-
 		// has the chain been invalidated?
 		if !chain.valid {
 			i++
@@ -98,15 +101,25 @@ func (mp *MessagePool) republishPendingMessages() error {
 
 		// does it fit in a block?
 		if chain.gasLimit <= gasLimit {
-			gasLimit -= chain.gasLimit
-			msgs = append(msgs, chain.msgs...)
+			// check the baseFee lower bound -- only republish messages that can be included in the chain
+			// within the next 20 blocks.
+			for _, m := range chain.msgs {
+				if !allowNegativeChains(ts.Height()) && m.Message.GasFeeCap.LessThan(baseFeeLowerBound) {
+					chain.Invalidate()
+					continue loop
+				}
+				gasLimit -= m.Message.GasLimit
+				msgs = append(msgs, m)
+			}
+
+			// we processed the whole chain, advance
 			i++
 			continue
 		}
 
 		// we can't fit the current chain but there is gas to spare
 		// trim it and push it down
-		chain.Trim(gasLimit, mp, baseFee, ts, false)
+		chain.Trim(gasLimit, mp, baseFee, true)
 		for j := i; j < len(chains)-1; j++ {
 			if chains[j].Before(chains[j+1]) {
 				break
@@ -115,6 +128,7 @@ func (mp *MessagePool) republishPendingMessages() error {
 		}
 	}
 
+	count := 0
 	log.Infof("republishing %d messages", len(msgs))
 	for _, m := range msgs {
 		mb, err := m.Serialize()
@@ -126,7 +140,26 @@ func (mp *MessagePool) republishPendingMessages() error {
 		if err != nil {
 			return xerrors.Errorf("cannot publish: %w", err)
 		}
+
+		count++
+
+		if count < len(msgs) {
+			// this delay is here to encourage the pubsub subsystem to process the messages serially
+			// and avoid creating nonce gaps because of concurrent validation.
+			time.Sleep(RepublishBatchDelay)
+		}
 	}
 
+	// track most recently republished messages
+	republished := make(map[cid.Cid]struct{})
+	for _, m := range msgs[:count] {
+		republished[m.Cid()] = struct{}{}
+	}
+
+	mp.lk.Lock()
+	// update the republished set so that we can trigger early republish from head changes
+	mp.republished = republished
+	mp.lk.Unlock()
+
 	return nil
 }

chain/messagepool/repub_test.go

@@ -0,0 +1,72 @@
+package messagepool
+
+import (
+	"testing"
+	"time"
+
+	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/lotus/chain/messagepool/gasguess"
+	"github.com/filecoin-project/lotus/chain/wallet"
+	"github.com/filecoin-project/specs-actors/actors/builtin"
+	"github.com/ipfs/go-datastore"
+)
+
+func TestRepubMessages(t *testing.T) {
+	oldRepublishBatchDelay := RepublishBatchDelay
+	RepublishBatchDelay = time.Microsecond
+	defer func() {
+		RepublishBatchDelay = oldRepublishBatchDelay
+	}()
+
+	tma := newTestMpoolAPI()
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(tma, ds, "mptest")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// the actors
+	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
+
+	tma.setBalance(a1, 1) // in FIL
+
+	for i := 0; i < 10; i++ {
+		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		_, err := mp.Push(m)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if tma.published != 10 {
+		t.Fatalf("expected to have published 10 messages, but got %d instead", tma.published)
+	}
+
+	mp.repubTrigger <- struct{}{}
+	time.Sleep(100 * time.Millisecond)
+
+	if tma.published != 20 {
+		t.Fatalf("expected to have published 20 messages, but got %d instead", tma.published)
+	}
+}

chain/messagepool/selection.go

@@ -3,22 +3,29 @@ package messagepool
 import (
 	"context"
 	"math/big"
+	"math/rand"
 	"sort"
 	"time"
 
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	tbig "github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/messagepool/gasguess"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
-	abig "github.com/filecoin-project/specs-actors/actors/abi/big"
-	"github.com/ipfs/go-cid"
 )
 
 var bigBlockGasLimit = big.NewInt(build.BlockGasLimit)
 
+// this is *temporary* mutilation until we have implemented uncapped miner penalties -- it will go
+// away in the next fork.
+func allowNegativeChains(epoch abi.ChainEpoch) bool {
+	return epoch < build.UpgradeBreezeHeight+5
+}
+
 const MaxBlocks = 15
 
 type msgChain struct {
@@ -101,9 +108,9 @@ func (mp *MessagePool) selectMessagesOptimal(curTs, ts *types.TipSet, tq float64
 		return chains[i].Before(chains[j])
 	})
 
-	if len(chains) != 0 && chains[0].gasPerf < 0 {
+	if !allowNegativeChains(curTs.Height()) && len(chains) != 0 && chains[0].gasPerf < 0 {
 		log.Warnw("all messages in mpool have non-positive gas performance", "bestGasPerf", chains[0].gasPerf)
-		return nil, nil
+		return result, nil
 	}
 
 	// 3. Parition chains into blocks (without trimming)
@@ -154,7 +161,7 @@ func (mp *MessagePool) selectMessagesOptimal(curTs, ts *types.TipSet, tq float64
 	last := len(chains)
 	for i, chain := range chains {
 		// did we run out of performing chains?
-		if chain.gasPerf < 0 {
+		if !allowNegativeChains(curTs.Height()) && chain.gasPerf < 0 {
 			break
 		}
 
@@ -177,24 +184,26 @@ func (mp *MessagePool) selectMessagesOptimal(curTs, ts *types.TipSet, tq float64
 			for i := len(chainDeps) - 1; i >= 0; i-- {
 				curChain := chainDeps[i]
 				curChain.merged = true
-				// adjust the next chain for the parent, which is being merged
-				if next := curChain.next; next != nil && next.effPerf > 0 {
-					next.effPerf += next.parentOffset
-				}
 				result = append(result, curChain.msgs...)
 			}
 
 			chain.merged = true
+			// adjust the effective pefromance for all subsequent chains
 			if next := chain.next; next != nil && next.effPerf > 0 {
 				next.effPerf += next.parentOffset
+				for next = next.next; next != nil && next.effPerf > 0; next = next.next {
+					next.setEffPerf()
+				}
 			}
 			result = append(result, chain.msgs...)
 			gasLimit -= chainGasLimit
 
 			// resort to account for already merged chains and effective performance adjustments
-			sort.Slice(chains[i+1:], func(i, j int) bool {
+			// the sort *must* be stable or we end up getting negative gasPerfs pushed up.
+			sort.SliceStable(chains[i+1:], func(i, j int) bool {
 				return chains[i].BeforeEffective(chains[j])
 			})
+
 			continue
 		}
 
@@ -218,7 +227,7 @@ tailLoop:
 	for gasLimit >= minGas && last < len(chains) {
 		// trim if necessary
 		if chains[last].gasLimit > gasLimit {
-			chains[last].Trim(gasLimit, mp, baseFee, ts, false)
+			chains[last].Trim(gasLimit, mp, baseFee, allowNegativeChains(curTs.Height()))
 		}
 
 		// push down if it hasn't been invalidated
@@ -244,7 +253,7 @@ tailLoop:
 			}
 
 			// if gasPerf < 0 we have no more profitable chains
-			if chain.gasPerf < 0 {
+			if !allowNegativeChains(curTs.Height()) && chain.gasPerf < 0 {
 				break tailLoop
 			}
 
@@ -285,7 +294,7 @@ tailLoop:
 			}
 
 			// dependencies fit, just trim it
-			chain.Trim(gasLimit-depGasLimit, mp, baseFee, ts, false)
+			chain.Trim(gasLimit-depGasLimit, mp, baseFee, allowNegativeChains(curTs.Height()))
 			last += i
 			continue tailLoop
 		}
@@ -298,6 +307,79 @@ tailLoop:
 		log.Infow("pack tail chains done", "took", dt)
 	}
 
+	// if we have gasLimit to spare, pick some random (non-negative) chains to fill the block
+	// we pick randomly so that we minimize the probability of duplication among all miners
+	if gasLimit >= minGas {
+		randomCount := 0
+
+		startRandom := time.Now()
+		shuffleChains(chains)
+
+		for _, chain := range chains {
+			// have we filled the block
+			if gasLimit < minGas {
+				break
+			}
+
+			// has it been merged or invalidated?
+			if chain.merged || !chain.valid {
+				continue
+			}
+
+			// is it negative?
+			if !allowNegativeChains(curTs.Height()) && chain.gasPerf < 0 {
+				continue
+			}
+
+			// compute the dependencies that must be merged and the gas limit including deps
+			chainGasLimit := chain.gasLimit
+			depGasLimit := int64(0)
+			var chainDeps []*msgChain
+			for curChain := chain.prev; curChain != nil && !curChain.merged; curChain = curChain.prev {
+				chainDeps = append(chainDeps, curChain)
+				chainGasLimit += curChain.gasLimit
+				depGasLimit += curChain.gasLimit
+			}
+
+			// do the deps fit? if the deps won't fit, invalidate the chain
+			if depGasLimit > gasLimit {
+				chain.Invalidate()
+				continue
+			}
+
+			// do they fit as is? if it doesn't, trim to make it fit if possible
+			if chainGasLimit > gasLimit {
+				chain.Trim(gasLimit-depGasLimit, mp, baseFee, allowNegativeChains(curTs.Height()))
+
+				if !chain.valid {
+					continue
+				}
+			}
+
+			// include it together with all dependencies
+			for i := len(chainDeps) - 1; i >= 0; i-- {
+				curChain := chainDeps[i]
+				curChain.merged = true
+				result = append(result, curChain.msgs...)
+				randomCount += len(curChain.msgs)
+			}
+
+			chain.merged = true
+			result = append(result, chain.msgs...)
+			randomCount += len(chain.msgs)
+			gasLimit -= chainGasLimit
+		}
+
+		if dt := time.Since(startRandom); dt > time.Millisecond {
+			log.Infow("pack random tail chains done", "took", dt)
+		}
+
+		if randomCount > 0 {
+			log.Warnf("optimal selection failed to pack a block; picked %d messages with random selection",
+				randomCount)
+		}
+	}
+
 	return result, nil
 }
 
@@ -350,9 +432,9 @@ func (mp *MessagePool) selectMessagesGreedy(curTs, ts *types.TipSet) ([]*types.S
 		return chains[i].Before(chains[j])
 	})
 
-	if len(chains) != 0 && chains[0].gasPerf < 0 {
+	if !allowNegativeChains(curTs.Height()) && len(chains) != 0 && chains[0].gasPerf < 0 {
 		log.Warnw("all messages in mpool have non-positive gas performance", "bestGasPerf", chains[0].gasPerf)
-		return nil, nil
+		return result, nil
 	}
 
 	// 3. Merge the head chains to produce the list of messages selected for inclusion, subject to
@@ -361,7 +443,7 @@ func (mp *MessagePool) selectMessagesGreedy(curTs, ts *types.TipSet) ([]*types.S
 	last := len(chains)
 	for i, chain := range chains {
 		// did we run out of performing chains?
-		if chain.gasPerf < 0 {
+		if !allowNegativeChains(curTs.Height()) && chain.gasPerf < 0 {
 			break
 		}
 
@@ -390,7 +472,7 @@ func (mp *MessagePool) selectMessagesGreedy(curTs, ts *types.TipSet) ([]*types.S
 tailLoop:
 	for gasLimit >= minGas && last < len(chains) {
 		// trim
-		chains[last].Trim(gasLimit, mp, baseFee, ts, false)
+		chains[last].Trim(gasLimit, mp, baseFee, allowNegativeChains(curTs.Height()))
 
 		// push down if it hasn't been invalidated
 		if chains[last].valid {
@@ -410,7 +492,7 @@ tailLoop:
 			}
 
 			// if gasPerf < 0 we have no more profitable chains
-			if chain.gasPerf < 0 {
+			if !allowNegativeChains(curTs.Height()) && chain.gasPerf < 0 {
 				break tailLoop
 			}
 
@@ -463,15 +545,27 @@ func (mp *MessagePool) selectPriorityMessages(pending map[address.Address]map[ui
 		}
 	}
 
+	if len(chains) == 0 {
+		return nil, gasLimit
+	}
+
 	// 2. Sort the chains
 	sort.Slice(chains, func(i, j int) bool {
 		return chains[i].Before(chains[j])
 	})
 
-	// 3. Merge chains until the block limit; we are willing to include negative performing chains
-	//    as these are messages from our own miners
+	if !allowNegativeChains(ts.Height()) && len(chains) != 0 && chains[0].gasPerf < 0 {
+		log.Warnw("all priority messages in mpool have negative gas performance", "bestGasPerf", chains[0].gasPerf)
+		return nil, gasLimit
+	}
+
+	// 3. Merge chains until the block limit, as long as they have non-negative gas performance
 	last := len(chains)
 	for i, chain := range chains {
+		if !allowNegativeChains(ts.Height()) && chain.gasPerf < 0 {
+			break
+		}
+
 		if chain.gasLimit <= gasLimit {
 			gasLimit -= chain.gasLimit
 			result = append(result, chain.msgs...)
@@ -485,8 +579,8 @@ func (mp *MessagePool) selectPriorityMessages(pending map[address.Address]map[ui
 
 tailLoop:
 	for gasLimit >= minGas && last < len(chains) {
-		// trim, without discarding negative performing messages
-		chains[last].Trim(gasLimit, mp, baseFee, ts, true)
+		// trim, discarding negative performing messages
+		chains[last].Trim(gasLimit, mp, baseFee, allowNegativeChains(ts.Height()))
 
 		// push down if it hasn't been invalidated
 		if chains[last].valid {
@@ -504,6 +598,12 @@ tailLoop:
 			if !chain.valid {
 				continue
 			}
+
+			// if gasPerf < 0 we have no more profitable chains
+			if !allowNegativeChains(ts.Height()) && chain.gasPerf < 0 {
+				break tailLoop
+			}
+
 			// does it fit in the bock?
 			if chain.gasLimit <= gasLimit {
 				gasLimit -= chain.gasLimit
@@ -516,9 +616,9 @@ tailLoop:
 			continue tailLoop
 		}
 
-		// the merge loop ended after processing all the chains and we probably still have gas to spare
-		// -- mark the end.
-		last = len(chains)
+		// the merge loop ended after processing all the chains and we probably still have gas to spare;
+		// end the loop
+		break
 	}
 
 	return result, gasLimit
@@ -528,7 +628,6 @@ func (mp *MessagePool) getPendingMessages(curTs, ts *types.TipSet) (map[address.
 	start := time.Now()
 
 	result := make(map[address.Address]map[uint64]*types.SignedMessage)
-	haveCids := make(map[cid.Cid]struct{})
 	defer func() {
 		if dt := time.Since(start); dt > time.Millisecond {
 			log.Infow("get pending messages done", "took", dt)
@@ -554,10 +653,6 @@ func (mp *MessagePool) getPendingMessages(curTs, ts *types.TipSet) (map[address.
 			}
 			result[a] = msetCopy
 
-			// mark the messages as seen
-			for _, m := range mset.msgs {
-				haveCids[m.Cid()] = struct{}{}
-			}
 		}
 	}
 
@@ -566,84 +661,25 @@ func (mp *MessagePool) getPendingMessages(curTs, ts *types.TipSet) (map[address.
 		return result, nil
 	}
 
-	// nope, we need to sync the tipsets
-	for {
-		if curTs.Height() == ts.Height() {
-			if curTs.Equals(ts) {
-				return result, nil
-			}
-
-			// different blocks in tipsets -- we mark them as seen so that they are not included in
-			// in the message set we return, but *neither me (vyzo) nor why understand why*
-			// this code is also probably completely untested in production, so I am adding a big fat
-			// warning to revisit this case and sanity check this decision.
-			log.Warnf("mpool tipset has same height as target tipset but it's not equal; beware of dragons!")
-
-			have, err := mp.MessagesForBlocks(ts.Blocks())
-			if err != nil {
-				return nil, xerrors.Errorf("error retrieving messages for tipset: %w", err)
-			}
-
-			for _, m := range have {
-				haveCids[m.Cid()] = struct{}{}
-			}
-		}
-
-		msgs, err := mp.MessagesForBlocks(ts.Blocks())
-		if err != nil {
-			return nil, xerrors.Errorf("error retrieving messages for tipset: %w", err)
-		}
-
-		for _, m := range msgs {
-			if _, have := haveCids[m.Cid()]; have {
-				continue
-			}
-
-			haveCids[m.Cid()] = struct{}{}
-			mset, ok := result[m.Message.From]
-			if !ok {
-				mset = make(map[uint64]*types.SignedMessage)
-				result[m.Message.From] = mset
-			}
-
-			other, dupNonce := mset[m.Message.Nonce]
-			if dupNonce {
-				// duplicate nonce, selfishly keep the message with the highest GasPrice
-				// if the gas prices are the same, keep the one with the highest GasLimit
-				switch m.Message.GasPremium.Int.Cmp(other.Message.GasPremium.Int) {
-				case 0:
-					if m.Message.GasLimit > other.Message.GasLimit {
-						mset[m.Message.Nonce] = m
-					}
-				case 1:
-					mset[m.Message.Nonce] = m
-				}
-			} else {
-				mset[m.Message.Nonce] = m
-			}
-		}
-
-		if curTs.Height() >= ts.Height() {
-			return result, nil
-		}
-
-		ts, err = mp.api.LoadTipSet(ts.Parents())
-		if err != nil {
-			return nil, xerrors.Errorf("error loading parent tipset: %w", err)
-		}
+	if err := mp.runHeadChange(curTs, ts, result); err != nil {
+		return nil, xerrors.Errorf("failed to process difference between mpool head and given head: %w", err)
 	}
+
+	return result, nil
 }
 
-func (mp *MessagePool) getGasReward(msg *types.SignedMessage, baseFee types.BigInt, ts *types.TipSet) *big.Int {
+func (*MessagePool) getGasReward(msg *types.SignedMessage, baseFee types.BigInt) *big.Int {
 	maxPremium := types.BigSub(msg.Message.GasFeeCap, baseFee)
-	if types.BigCmp(maxPremium, msg.Message.GasPremium) < 0 {
+
+	if types.BigCmp(maxPremium, msg.Message.GasPremium) > 0 {
 		maxPremium = msg.Message.GasPremium
 	}
-	gasReward := abig.Mul(maxPremium, types.NewInt(uint64(msg.Message.GasLimit)))
+
+	gasReward := tbig.Mul(maxPremium, types.NewInt(uint64(msg.Message.GasLimit)))
 	return gasReward.Int
 }
 
-func (mp *MessagePool) getGasPerf(gasReward *big.Int, gasLimit int64) float64 {
+func (*MessagePool) getGasPerf(gasReward *big.Int, gasLimit int64) float64 {
 	// gasPerf = gasReward * build.BlockGasLimit / gasLimit
 	a := new(big.Rat).SetInt(new(big.Int).Mul(gasReward, bigBlockGasLimit))
 	b := big.NewRat(1, gasLimit)
@@ -671,7 +707,12 @@ func (mp *MessagePool) createMessageChains(actor address.Address, mset map[uint6
 	//   cannot exceed the block limit; drop all messages that exceed the limit
 	// - the total gasReward cannot exceed the actor's balance; drop all messages that exceed
 	//   the balance
-	a, _ := mp.api.StateGetActor(actor, ts)
+	a, err := mp.api.GetActorAfter(actor, ts)
+	if err != nil {
+		log.Errorf("failed to load actor state, not building chain for %s: %w", actor, err)
+		return nil
+	}
+
 	curNonce := a.Nonce
 	balance := a.Balance.Int
 	gasLimit := int64(0)
@@ -716,7 +757,7 @@ func (mp *MessagePool) createMessageChains(actor address.Address, mset map[uint6
 			balance = new(big.Int).Sub(balance, value)
 		}
 
-		gasReward := mp.getGasReward(m, baseFee, ts)
+		gasReward := mp.getGasReward(m, baseFee)
 		rewards = append(rewards, gasReward)
 	}
 
@@ -817,10 +858,10 @@ func (mc *msgChain) Before(other *msgChain) bool {
 		(mc.gasPerf == other.gasPerf && mc.gasReward.Cmp(other.gasReward) > 0)
 }
 
-func (mc *msgChain) Trim(gasLimit int64, mp *MessagePool, baseFee types.BigInt, ts *types.TipSet, priority bool) {
+func (mc *msgChain) Trim(gasLimit int64, mp *MessagePool, baseFee types.BigInt, allowNegative bool) {
 	i := len(mc.msgs) - 1
-	for i >= 0 && (mc.gasLimit > gasLimit || (!priority && mc.gasPerf < 0)) {
-		gasReward := mp.getGasReward(mc.msgs[i], baseFee, ts)
+	for i >= 0 && (mc.gasLimit > gasLimit || (!allowNegative && mc.gasPerf < 0)) {
+		gasReward := mp.getGasReward(mc.msgs[i], baseFee)
 		mc.gasReward = new(big.Int).Sub(mc.gasReward, gasReward)
 		mc.gasLimit -= mc.msgs[i].Message.GasLimit
 		if mc.gasLimit > 0 {
@@ -883,7 +924,16 @@ func (mc *msgChain) SetNullEffectivePerf() {
 
 func (mc *msgChain) BeforeEffective(other *msgChain) bool {
 	// move merged chains to the front so we can discard them earlier
-	return (mc.merged && !other.merged) || mc.effPerf > other.effPerf ||
+	return (mc.merged && !other.merged) ||
+		(mc.gasPerf >= 0 && other.gasPerf < 0) ||
+		mc.effPerf > other.effPerf ||
 		(mc.effPerf == other.effPerf && mc.gasPerf > other.gasPerf) ||
 		(mc.effPerf == other.effPerf && mc.gasPerf == other.gasPerf && mc.gasReward.Cmp(other.gasReward) > 0)
 }
+
+func shuffleChains(lst []*msgChain) {
+	for i := range lst {
+		j := rand.Intn(i + 1)
+		lst[i], lst[j] = lst[j], lst[i]
+	}
+}

chain/messagepool/selection_test.go

@@ -1,20 +1,26 @@
 package messagepool
 
 import (
+	"compress/gzip"
 	"context"
+	"encoding/json"
+	"fmt"
+	"io"
 	"math"
 	"math/big"
 	"math/rand"
+	"os"
+	"sort"
 	"testing"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/messagepool/gasguess"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/types/mock"
 	"github.com/filecoin-project/lotus/chain/wallet"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
-	"github.com/filecoin-project/specs-actors/actors/crypto"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
 
@@ -23,6 +29,11 @@ import (
 	logging "github.com/ipfs/go-log"
 )
 
+func init() {
+	// bump this for the selection tests
+	MaxActorPendingMessages = 1000000
+}
+
 func makeTestMessage(w *wallet.Wallet, from, to address.Address, nonce uint64, gasLimit int64, gasPrice uint64) *types.SignedMessage {
 	msg := &types.Message{
 		From:       from,
@@ -79,7 +90,7 @@ func TestMessageChains(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	block := mock.MkBlock(nil, 1, 1)
+	block := tma.nextBlock()
 	ts := mock.TipSet(block)
 
 	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
@@ -317,7 +328,7 @@ func TestMessageChainSkipping(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	block := mock.MkBlock(nil, 1, 1)
+	block := tma.nextBlock()
 	ts := mock.TipSet(block)
 
 	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
@@ -364,6 +375,12 @@ func TestMessageChainSkipping(t *testing.T) {
 }
 
 func TestBasicMessageSelection(t *testing.T) {
+	oldMaxNonceGap := MaxNonceGap
+	MaxNonceGap = 1000
+	defer func() {
+		MaxNonceGap = oldMaxNonceGap
+	}()
+
 	mp, tma := makeTestMpool()
 
 	// the actors
@@ -387,7 +404,7 @@ func TestBasicMessageSelection(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	block := mock.MkBlock(nil, 1, 1)
+	block := tma.nextBlock()
 	ts := mock.TipSet(block)
 	tma.applyBlock(t, block)
 
@@ -440,12 +457,12 @@ func TestBasicMessageSelection(t *testing.T) {
 	}
 
 	// now we make a block with all the messages and advance the chain
-	block2 := mock.MkBlock(ts, 2, 2)
+	block2 := tma.nextBlock()
 	tma.setBlockMessages(block2, msgs...)
 	tma.applyBlock(t, block2)
 
 	// we should have no pending messages in the mpool
-	pend, ts2 := mp.Pending()
+	pend, _ := mp.Pending()
 	if len(pend) != 0 {
 		t.Fatalf("expected no pending messages, but got %d", len(pend))
 	}
@@ -458,13 +475,13 @@ func TestBasicMessageSelection(t *testing.T) {
 		m = makeTestMessage(w2, a2, a1, uint64(i), gasLimit, uint64(i+1))
 		msgs = append(msgs, m)
 	}
-	block3 := mock.MkBlock(ts2, 3, 3)
+	block3 := tma.nextBlock()
 	tma.setBlockMessages(block3, msgs...)
 	ts3 := mock.TipSet(block3)
 
 	// now create another set of messages and add them to the mpool
 	for i := 20; i < 30; i++ {
-		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(2*i+1))
+		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(2*i+200))
 		mustAdd(t, mp, m)
 		m = makeTestMessage(w2, a2, a1, uint64(i), gasLimit, uint64(i+1))
 		mustAdd(t, mp, m)
@@ -480,12 +497,12 @@ func TestBasicMessageSelection(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(msgs) != 40 {
-		t.Fatalf("expected 40 messages, got %d", len(msgs))
+	if len(msgs) != 20 {
+		t.Fatalf("expected 20 messages, got %d", len(msgs))
 	}
 
-	nextNonce = 10
-	for i := 0; i < 20; i++ {
+	nextNonce = 20
+	for i := 0; i < 10; i++ {
 		if msgs[i].Message.From != a1 {
 			t.Fatalf("expected message from actor a1")
 		}
@@ -495,8 +512,8 @@ func TestBasicMessageSelection(t *testing.T) {
 		nextNonce++
 	}
 
-	nextNonce = 10
-	for i := 20; i < 40; i++ {
+	nextNonce = 20
+	for i := 10; i < 20; i++ {
 		if msgs[i].Message.From != a2 {
 			t.Fatalf("expected message from actor a2")
 		}
@@ -531,7 +548,7 @@ func TestMessageSelectionTrimming(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	block := mock.MkBlock(nil, 1, 1)
+	block := tma.nextBlock()
 	ts := mock.TipSet(block)
 	tma.applyBlock(t, block)
 
@@ -594,7 +611,7 @@ func TestPriorityMessageSelection(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	block := mock.MkBlock(nil, 1, 1)
+	block := tma.nextBlock()
 	ts := mock.TipSet(block)
 	tma.applyBlock(t, block)
 
@@ -649,6 +666,169 @@ func TestPriorityMessageSelection(t *testing.T) {
 	}
 }
 
+func TestPriorityMessageSelection2(t *testing.T) {
+	mp, tma := makeTestMpool()
+
+	// the actors
+	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	block := tma.nextBlock()
+	ts := mock.TipSet(block)
+	tma.applyBlock(t, block)
+
+	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
+
+	tma.setBalance(a1, 1) // in FIL
+	tma.setBalance(a2, 1) // in FIL
+
+	mp.cfg.PriorityAddrs = []address.Address{a1}
+
+	nMessages := int(2 * build.BlockGasLimit / gasLimit)
+	for i := 0; i < nMessages; i++ {
+		bias := (nMessages - i) / 3
+		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(1+i%3+bias))
+		mustAdd(t, mp, m)
+		m = makeTestMessage(w2, a2, a1, uint64(i), gasLimit, uint64(1+i%3+bias))
+		mustAdd(t, mp, m)
+	}
+
+	msgs, err := mp.SelectMessages(ts, 1.0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedMsgs := int(build.BlockGasLimit / gasLimit)
+	if len(msgs) != expectedMsgs {
+		t.Fatalf("expected %d messages but got %d", expectedMsgs, len(msgs))
+	}
+
+	// all messages must be from a1
+	nextNonce := uint64(0)
+	for _, m := range msgs {
+		if m.Message.From != a1 {
+			t.Fatal("expected messages from a1 before messages from a2")
+		}
+		if m.Message.Nonce != nextNonce {
+			t.Fatalf("expected nonce %d but got %d", nextNonce, m.Message.Nonce)
+		}
+		nextNonce++
+	}
+}
+
+func TestPriorityMessageSelection3(t *testing.T) {
+	t.Skip("reenable after removing allow negative")
+
+	mp, tma := makeTestMpool()
+
+	// the actors
+	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	block := tma.nextBlock()
+	ts := mock.TipSet(block)
+	tma.applyBlock(t, block)
+
+	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
+
+	tma.setBalance(a1, 1) // in FIL
+	tma.setBalance(a2, 1) // in FIL
+
+	mp.cfg.PriorityAddrs = []address.Address{a1}
+
+	tma.baseFee = types.NewInt(1000)
+	nMessages := 10
+	for i := 0; i < nMessages; i++ {
+		bias := (nMessages - i) / 3
+		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(1000+i%3+bias))
+		mustAdd(t, mp, m)
+		// messages from a2 have negative performance
+		m = makeTestMessage(w2, a2, a1, uint64(i), gasLimit, 100)
+		mustAdd(t, mp, m)
+	}
+
+	// test greedy selection
+	msgs, err := mp.SelectMessages(ts, 1.0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedMsgs := 10
+	if len(msgs) != expectedMsgs {
+		t.Fatalf("expected %d messages but got %d", expectedMsgs, len(msgs))
+	}
+
+	// all messages must be from a1
+	nextNonce := uint64(0)
+	for _, m := range msgs {
+		if m.Message.From != a1 {
+			t.Fatal("expected messages from a1 before messages from a2")
+		}
+		if m.Message.Nonce != nextNonce {
+			t.Fatalf("expected nonce %d but got %d", nextNonce, m.Message.Nonce)
+		}
+		nextNonce++
+	}
+
+	// test optimal selection
+	msgs, err = mp.SelectMessages(ts, 0.1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedMsgs = 10
+	if len(msgs) != expectedMsgs {
+		t.Fatalf("expected %d messages but got %d", expectedMsgs, len(msgs))
+	}
+
+	// all messages must be from a1
+	nextNonce = uint64(0)
+	for _, m := range msgs {
+		if m.Message.From != a1 {
+			t.Fatal("expected messages from a1 before messages from a2")
+		}
+		if m.Message.Nonce != nextNonce {
+			t.Fatalf("expected nonce %d but got %d", nextNonce, m.Message.Nonce)
+		}
+		nextNonce++
+	}
+
+}
+
 func TestOptimalMessageSelection1(t *testing.T) {
 	// this test uses just a single actor sending messages with a low tq
 	// the chain depenent merging algorithm should pick messages from the actor
@@ -676,7 +856,7 @@ func TestOptimalMessageSelection1(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	block := mock.MkBlock(nil, 1, 1)
+	block := tma.nextBlock()
 	ts := mock.TipSet(block)
 	tma.applyBlock(t, block)
 
@@ -743,7 +923,7 @@ func TestOptimalMessageSelection2(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	block := mock.MkBlock(nil, 1, 1)
+	block := tma.nextBlock()
 	ts := mock.TipSet(block)
 	tma.applyBlock(t, block)
 
@@ -771,16 +951,26 @@ func TestOptimalMessageSelection2(t *testing.T) {
 		t.Fatalf("expected %d messages, but got %d", expectedMsgs, len(msgs))
 	}
 
-	nextNonce := uint64(0)
+	var nFrom1, nFrom2 int
+	var nextNonce1, nextNonce2 uint64
 	for _, m := range msgs {
-		if m.Message.From != a2 {
-			t.Fatal("expected message from a2")
+		if m.Message.From == a1 {
+			if m.Message.Nonce != nextNonce1 {
+				t.Fatalf("expected nonce %d but got %d", nextNonce1, m.Message.Nonce)
+			}
+			nextNonce1++
+			nFrom1++
+		} else {
+			if m.Message.Nonce != nextNonce2 {
+				t.Fatalf("expected nonce %d but got %d", nextNonce2, m.Message.Nonce)
+			}
+			nextNonce2++
+			nFrom2++
 		}
+	}
 
-		if m.Message.Nonce != nextNonce {
-			t.Fatalf("expected nonce %d but got %d", nextNonce, m.Message.Nonce)
-		}
-		nextNonce++
+	if nFrom1 > nFrom2 {
+		t.Fatalf("expected more messages from a2 than a1; nFrom1=%d nFrom2=%d", nFrom1, nFrom2)
 	}
 }
 
@@ -811,7 +1001,7 @@ func TestOptimalMessageSelection3(t *testing.T) {
 		wallets = append(wallets, w)
 	}
 
-	block := mock.MkBlock(nil, 1, 1)
+	block := tma.nextBlock()
 	ts := mock.TipSet(block)
 	tma.applyBlock(t, block)
 
@@ -824,7 +1014,7 @@ func TestOptimalMessageSelection3(t *testing.T) {
 	nMessages := int(build.BlockGasLimit/gasLimit) + 1
 	for i := 0; i < nMessages; i++ {
 		for j := 0; j < nActors; j++ {
-			premium := 500000 + 20000*(nActors-j) + (nMessages+2-i)/(3*nActors) + i%3
+			premium := 500000 + 10000*(nActors-j) + (nMessages+2-i)/(30*nActors) + i%3
 			m := makeTestMessage(wallets[j], actors[j], actors[j%nActors], uint64(i), gasLimit, uint64(premium))
 			mustAdd(t, mp, m)
 		}
@@ -864,12 +1054,11 @@ func TestOptimalMessageSelection3(t *testing.T) {
 	}
 }
 
-func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) (float64, float64) {
-	// in this test we use 100 actors and send 10 blocks of messages.
-	// actors send with an exponentially decreasing premium.
+func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand, getPremium func() uint64) (float64, float64, float64) {
+	// in this test we use 300 actors and send 10 blocks of messages.
+	// actors send with an randomly distributed premium dictated by the getPremium function.
 	// a number of miners select with varying ticket quality and we compare the
 	// capacity and rewards of greedy selection -vs- optimal selection
-
 	mp, tma := makeTestMpool()
 
 	nActors := 300
@@ -892,7 +1081,7 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) (float64, flo
 		wallets = append(wallets, w)
 	}
 
-	block := mock.MkBlock(nil, 1, 1)
+	block := tma.nextBlock()
 	ts := mock.TipSet(block)
 	tma.applyBlock(t, block)
 
@@ -909,10 +1098,10 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) (float64, flo
 	for i := 0; i < nMessages; i++ {
 		from := rng.Intn(nActors)
 		to := rng.Intn(nActors)
-		premium := 20000*math.Exp(-3.*rand.Float64()) + 5000
 		nonce := nonces[from]
 		nonces[from]++
-		m := makeTestMessage(wallets[from], actors[from], actors[to], uint64(nonce), gasLimit, uint64(premium))
+		premium := getPremium()
+		m := makeTestMessage(wallets[from], actors[from], actors[to], nonce, gasLimit, premium)
 		mustAdd(t, mp, m)
 	}
 
@@ -924,8 +1113,11 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) (float64, flo
 		t.Fatal(err)
 	}
 
-	capacityBoost := 0.0
-	rewardBoost := 0.0
+	totalGreedyCapacity := 0.0
+	totalGreedyReward := 0.0
+	totalOptimalCapacity := 0.0
+	totalOptimalReward := 0.0
+	totalBestTQReward := 0.0
 	const runs = 1
 	for i := 0; i < runs; i++ {
 		// 2. optimal selection
@@ -935,76 +1127,336 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) (float64, flo
 		for ; i < MaxBlocks && minersRand > 0; i++ {
 			minersRand -= winerProba[i]
 		}
-		nMiners := i
-		if nMiners == 0 {
+		nMiners := i - 1
+		if nMiners < 1 {
 			nMiners = 1
 		}
 
 		optMsgs := make(map[cid.Cid]*types.SignedMessage)
+		bestTq := 0.0
+		var bestMsgs []*types.SignedMessage
 		for j := 0; j < nMiners; j++ {
 			tq := rng.Float64()
 			msgs, err := mp.SelectMessages(ts, tq)
 			if err != nil {
 				t.Fatal(err)
 			}
+			if tq > bestTq {
+				bestMsgs = msgs
+			}
+
 			for _, m := range msgs {
 				optMsgs[m.Cid()] = m
 			}
 		}
 
+		totalGreedyCapacity += float64(len(greedyMsgs))
+		totalOptimalCapacity += float64(len(optMsgs))
 		boost := float64(len(optMsgs)) / float64(len(greedyMsgs))
-		capacityBoost += boost
 
 		t.Logf("nMiners: %d", nMiners)
 		t.Logf("greedy capacity %d, optimal capacity %d (x %.1f )", len(greedyMsgs),
 			len(optMsgs), boost)
 		if len(greedyMsgs) > len(optMsgs) {
-			t.Fatal("greedy capacity higher than optimal capacity; wtf")
+			t.Errorf("greedy capacity higher than optimal capacity; wtf")
 		}
 
 		greedyReward := big.NewInt(0)
 		for _, m := range greedyMsgs {
-			greedyReward.Add(greedyReward, mp.getGasReward(m, baseFee, ts))
+			greedyReward.Add(greedyReward, mp.getGasReward(m, baseFee))
 		}
 
 		optReward := big.NewInt(0)
 		for _, m := range optMsgs {
-			optReward.Add(optReward, mp.getGasReward(m, baseFee, ts))
+			optReward.Add(optReward, mp.getGasReward(m, baseFee))
 		}
 
+		bestTqReward := big.NewInt(0)
+		for _, m := range bestMsgs {
+			bestTqReward.Add(bestTqReward, mp.getGasReward(m, baseFee))
+		}
+
+		totalBestTQReward += float64(bestTqReward.Uint64())
+
 		nMinersBig := big.NewInt(int64(nMiners))
 		greedyAvgReward, _ := new(big.Rat).SetFrac(greedyReward, nMinersBig).Float64()
+		totalGreedyReward += greedyAvgReward
 		optimalAvgReward, _ := new(big.Rat).SetFrac(optReward, nMinersBig).Float64()
+		totalOptimalReward += optimalAvgReward
 
 		boost = optimalAvgReward / greedyAvgReward
-		rewardBoost += boost
 		t.Logf("greedy reward: %.0f, optimal reward: %.0f (x %.1f )", greedyAvgReward,
 			optimalAvgReward, boost)
 
 	}
 
-	capacityBoost /= runs
-	rewardBoost /= runs
+	capacityBoost := totalOptimalCapacity / totalGreedyCapacity
+	rewardBoost := totalOptimalReward / totalGreedyReward
 	t.Logf("Average capacity boost: %f", capacityBoost)
 	t.Logf("Average reward boost: %f", rewardBoost)
+	t.Logf("Average best tq reward: %f", totalBestTQReward/runs/1e12)
 
 	logging.SetLogLevel("messagepool", "info")
 
-	return capacityBoost, rewardBoost
+	return capacityBoost, rewardBoost, totalBestTQReward / runs / 1e12
 }
 
-func TestCompetitiveMessageSelection(t *testing.T) {
-	var capacityBoost, rewardBoost float64
+func makeExpPremiumDistribution(rng *rand.Rand) func() uint64 {
+	return func() uint64 {
+		premium := 20000*math.Exp(-3.*rng.Float64()) + 5000
+		return uint64(premium)
+	}
+}
+
+func makeZipfPremiumDistribution(rng *rand.Rand) func() uint64 {
+	zipf := rand.NewZipf(rng, 1.001, 1, 40000)
+	return func() uint64 {
+		return zipf.Uint64() + 10000
+	}
+}
+
+func TestCompetitiveMessageSelectionExp(t *testing.T) {
+	var capacityBoost, rewardBoost, tqReward float64
 	seeds := []int64{1947, 1976, 2020, 2100, 10000, 143324, 432432, 131, 32, 45}
 	for _, seed := range seeds {
-		t.Log("running competitve message selection with seed", seed)
-		cb, rb := testCompetitiveMessageSelection(t, rand.New(rand.NewSource(seed)))
+		t.Log("running competitive message selection with Exponential premium distribution and seed", seed)
+		rng := rand.New(rand.NewSource(seed))
+		cb, rb, tqR := testCompetitiveMessageSelection(t, rng, makeExpPremiumDistribution(rng))
 		capacityBoost += cb
 		rewardBoost += rb
+		tqReward += tqR
 	}
 
 	capacityBoost /= float64(len(seeds))
 	rewardBoost /= float64(len(seeds))
+	tqReward /= float64(len(seeds))
 	t.Logf("Average capacity boost across all seeds: %f", capacityBoost)
 	t.Logf("Average reward boost across all seeds: %f", rewardBoost)
+	t.Logf("Average reward of best ticket across all seeds: %f", tqReward)
+}
+
+func TestCompetitiveMessageSelectionZipf(t *testing.T) {
+	var capacityBoost, rewardBoost, tqReward float64
+	seeds := []int64{1947, 1976, 2020, 2100, 10000, 143324, 432432, 131, 32, 45}
+	for _, seed := range seeds {
+		t.Log("running competitive message selection with Zipf premium distribution and seed", seed)
+		rng := rand.New(rand.NewSource(seed))
+		cb, rb, tqR := testCompetitiveMessageSelection(t, rng, makeZipfPremiumDistribution(rng))
+		capacityBoost += cb
+		rewardBoost += rb
+		tqReward += tqR
+	}
+
+	tqReward /= float64(len(seeds))
+	capacityBoost /= float64(len(seeds))
+	rewardBoost /= float64(len(seeds))
+	t.Logf("Average capacity boost across all seeds: %f", capacityBoost)
+	t.Logf("Average reward boost across all seeds: %f", rewardBoost)
+	t.Logf("Average reward of best ticket across all seeds: %f", tqReward)
+}
+
+func TestGasReward(t *testing.T) {
+	tests := []struct {
+		Premium   uint64
+		FeeCap    uint64
+		BaseFee   uint64
+		GasReward int64
+	}{
+		{Premium: 100, FeeCap: 200, BaseFee: 100, GasReward: 100},
+		{Premium: 100, FeeCap: 200, BaseFee: 210, GasReward: -10},
+		{Premium: 200, FeeCap: 250, BaseFee: 210, GasReward: 40},
+		{Premium: 200, FeeCap: 250, BaseFee: 2000, GasReward: -1750},
+	}
+
+	mp := new(MessagePool)
+	for _, test := range tests {
+		test := test
+		t.Run(fmt.Sprintf("%v", test), func(t *testing.T) {
+			msg := &types.SignedMessage{
+				Message: types.Message{
+					GasLimit:   10,
+					GasFeeCap:  types.NewInt(test.FeeCap),
+					GasPremium: types.NewInt(test.Premium),
+				},
+			}
+			rew := mp.getGasReward(msg, types.NewInt(test.BaseFee))
+			if rew.Cmp(big.NewInt(test.GasReward*10)) != 0 {
+				t.Errorf("bad reward: expected %d, got %s", test.GasReward*10, rew)
+			}
+		})
+	}
+}
+
+func TestRealWorldSelection(t *testing.T) {
+	// load test-messages.json.gz and rewrite the messages so that
+	// 1) we map each real actor to a test actor so that we can sign the messages
+	// 2) adjust the nonces so that they start from 0
+	file, err := os.Open("test-messages.json.gz")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	gzr, err := gzip.NewReader(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dec := json.NewDecoder(gzr)
+
+	var msgs []*types.SignedMessage
+	baseNonces := make(map[address.Address]uint64)
+
+readLoop:
+	for {
+		m := new(types.SignedMessage)
+		err := dec.Decode(m)
+		switch err {
+		case nil:
+			msgs = append(msgs, m)
+			nonce, ok := baseNonces[m.Message.From]
+			if !ok || m.Message.Nonce < nonce {
+				baseNonces[m.Message.From] = m.Message.Nonce
+			}
+
+		case io.EOF:
+			break readLoop
+
+		default:
+			t.Fatal(err)
+		}
+	}
+
+	actorMap := make(map[address.Address]address.Address)
+	actorWallets := make(map[address.Address]*wallet.Wallet)
+
+	for _, m := range msgs {
+		baseNonce := baseNonces[m.Message.From]
+
+		localActor, ok := actorMap[m.Message.From]
+		if !ok {
+			w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			a, err := w.GenerateKey(crypto.SigTypeSecp256k1)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			actorMap[m.Message.From] = a
+			actorWallets[a] = w
+			localActor = a
+		}
+
+		w, ok := actorWallets[localActor]
+		if !ok {
+			t.Fatalf("failed to lookup wallet for actor %s", localActor)
+		}
+
+		m.Message.From = localActor
+		m.Message.Nonce -= baseNonce
+
+		sig, err := w.Sign(context.TODO(), localActor, m.Message.Cid().Bytes())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		m.Signature = *sig
+	}
+
+	mp, tma := makeTestMpool()
+
+	block := tma.nextBlockWithHeight(build.UpgradeBreezeHeight + 10)
+	ts := mock.TipSet(block)
+	tma.applyBlock(t, block)
+
+	for _, a := range actorMap {
+		tma.setBalance(a, 1000000)
+	}
+
+	tma.baseFee = types.NewInt(800_000_000)
+
+	sort.Slice(msgs, func(i, j int) bool {
+		return msgs[i].Message.Nonce < msgs[j].Message.Nonce
+	})
+
+	// add the messages
+	for _, m := range msgs {
+		mustAdd(t, mp, m)
+	}
+
+	// do message selection and check block packing
+	minGasLimit := int64(0.9 * float64(build.BlockGasLimit))
+
+	// greedy first
+	selected, err := mp.SelectMessages(ts, 1.0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	gasLimit := int64(0)
+	for _, m := range selected {
+		gasLimit += m.Message.GasLimit
+	}
+	if gasLimit < minGasLimit {
+		t.Fatalf("failed to pack with tq=1.0; packed %d, minimum packing: %d", gasLimit, minGasLimit)
+	}
+
+	// high quality ticket
+	selected, err = mp.SelectMessages(ts, .8)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	gasLimit = int64(0)
+	for _, m := range selected {
+		gasLimit += m.Message.GasLimit
+	}
+	if gasLimit < minGasLimit {
+		t.Fatalf("failed to pack with tq=0.8; packed %d, minimum packing: %d", gasLimit, minGasLimit)
+	}
+
+	// mid quality ticket
+	selected, err = mp.SelectMessages(ts, .4)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	gasLimit = int64(0)
+	for _, m := range selected {
+		gasLimit += m.Message.GasLimit
+	}
+	if gasLimit < minGasLimit {
+		t.Fatalf("failed to pack with tq=0.4; packed %d, minimum packing: %d", gasLimit, minGasLimit)
+	}
+
+	// low quality ticket
+	selected, err = mp.SelectMessages(ts, .1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	gasLimit = int64(0)
+	for _, m := range selected {
+		gasLimit += m.Message.GasLimit
+	}
+	if gasLimit < minGasLimit {
+		t.Fatalf("failed to pack with tq=0.1; packed %d, minimum packing: %d", gasLimit, minGasLimit)
+	}
+
+	// very low quality ticket
+	selected, err = mp.SelectMessages(ts, .01)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	gasLimit = int64(0)
+	for _, m := range selected {
+		gasLimit += m.Message.GasLimit
+	}
+	if gasLimit < minGasLimit {
+		t.Fatalf("failed to pack with tq=0.01; packed %d, minimum packing: %d", gasLimit, minGasLimit)
+	}
+
 }

chain/metrics/consensus.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"encoding/json"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log/v2"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
@@ -44,7 +44,7 @@ func SendHeadNotifs(nickname string) func(mctx helpers.MetricsCtx, lc fx.Lifecyc
 					}
 				}()
 				go func() {
-					sub, err := ps.Subscribe(topic)
+					sub, err := ps.Subscribe(topic) //nolint
 					if err != nil {
 						return
 					}
@@ -116,6 +116,7 @@ func sendHeadNotifs(ctx context.Context, ps *pubsub.PubSub, topic string, chain
 				return err
 			}
 
+			//nolint
 			if err := ps.Publish(topic, b); err != nil {
 				return err
 			}

chain/state/statetree.go

@@ -246,7 +246,7 @@ func (st *StateTree) DeleteActor(addr address.Address) error {
 }
 
 func (st *StateTree) Flush(ctx context.Context) (cid.Cid, error) {
-	ctx, span := trace.StartSpan(ctx, "stateTree.Flush")
+	ctx, span := trace.StartSpan(ctx, "stateTree.Flush") //nolint:staticcheck
 	defer span.End()
 	if len(st.snaps.layers) != 1 {
 		return cid.Undef, xerrors.Errorf("tried to flush state tree with snapshots on the stack")
@@ -268,7 +268,7 @@ func (st *StateTree) Flush(ctx context.Context) (cid.Cid, error) {
 }
 
 func (st *StateTree) Snapshot(ctx context.Context) error {
-	ctx, span := trace.StartSpan(ctx, "stateTree.SnapShot")
+	ctx, span := trace.StartSpan(ctx, "stateTree.SnapShot") //nolint:staticcheck
 	defer span.End()
 
 	st.snaps.addLayer()

chain/stmgr/call.go

@@ -5,8 +5,8 @@ import (
 	"fmt"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/crypto"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/ipfs/go-cid"
 	"go.opencensus.io/trace"
 	"golang.org/x/xerrors"
@@ -29,6 +29,7 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate
 		Bstore:         sm.cs.Blockstore(),
 		Syscalls:       sm.cs.VMSys(),
 		CircSupplyCalc: sm.GetCirculatingSupply,
+		NtwkVersion:    sm.GetNtwkVersion,
 		BaseFee:        types.NewInt(0),
 	}
 
@@ -95,7 +96,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
 
 	state := ts.ParentState()
 
-	r := store.NewChainRand(sm.cs, ts.Cids(), ts.Height())
+	r := store.NewChainRand(sm.cs, ts.Cids())
 
 	return sm.CallRaw(ctx, msg, state, r, ts.Height())
 }
@@ -113,7 +114,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 		return nil, xerrors.Errorf("computing tipset state: %w", err)
 	}
 
-	r := store.NewChainRand(sm.cs, ts.Cids(), ts.Height())
+	r := store.NewChainRand(sm.cs, ts.Cids())
 
 	if span.IsRecordingEvents() {
 		span.AddAttributes(
@@ -130,6 +131,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 		Bstore:         sm.cs.Blockstore(),
 		Syscalls:       sm.cs.VMSys(),
 		CircSupplyCalc: sm.GetCirculatingSupply,
+		NtwkVersion:    sm.GetNtwkVersion,
 		BaseFee:        ts.Blocks()[0].ParentBaseFee,
 	}
 	vmi, err := vm.NewVM(vmopt)

chain/stmgr/forks.go

@@ -3,16 +3,28 @@ package stmgr
 import (
 	"context"
 
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/state"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/builtin"
+	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
+	"github.com/filecoin-project/specs-actors/actors/builtin/power"
+	"github.com/filecoin-project/specs-actors/actors/util/adt"
+	cbor "github.com/ipfs/go-ipld-cbor"
+	"golang.org/x/xerrors"
 )
 
-var ForksAtHeight = map[abi.ChainEpoch]func(context.Context, *StateManager, types.StateTree) error{}
+var ForksAtHeight = map[abi.ChainEpoch]func(context.Context, *StateManager, types.StateTree, *types.TipSet) error{
+	build.UpgradeBreezeHeight: UpgradeFaucetBurnRecovery,
+}
 
-func (sm *StateManager) handleStateForks(ctx context.Context, st types.StateTree, height abi.ChainEpoch) (err error) {
+func (sm *StateManager) handleStateForks(ctx context.Context, st types.StateTree, height abi.ChainEpoch, ts *types.TipSet) (err error) {
 	f, ok := ForksAtHeight[height]
 	if ok {
-		err := f(ctx, sm, st)
+		err := f(ctx, sm, st, ts)
 		if err != nil {
 			return err
 		}
@@ -20,3 +32,291 @@ func (sm *StateManager) handleStateForks(ctx context.Context, st types.StateTree
 
 	return nil
 }
+
+type forEachTree interface {
+	ForEach(func(address.Address, *types.Actor) error) error
+}
+
+func doTransfer(tree types.StateTree, from, to address.Address, amt abi.TokenAmount) error {
+	fromAct, err := tree.GetActor(from)
+	if err != nil {
+		return xerrors.Errorf("failed to get 'from' actor for transfer: %w", err)
+	}
+
+	fromAct.Balance = types.BigSub(fromAct.Balance, amt)
+	if fromAct.Balance.Sign() < 0 {
+		return xerrors.Errorf("(sanity) deducted more funds from target account than it had (%s, %s)", from, types.FIL(amt))
+	}
+
+	if err := tree.SetActor(from, fromAct); err != nil {
+		return xerrors.Errorf("failed to persist from actor: %w", err)
+	}
+
+	toAct, err := tree.GetActor(to)
+	if err != nil {
+		return xerrors.Errorf("failed to get 'to' actor for transfer: %w", err)
+	}
+
+	toAct.Balance = types.BigAdd(toAct.Balance, amt)
+
+	if err := tree.SetActor(to, toAct); err != nil {
+		return xerrors.Errorf("failed to persist to actor: %w", err)
+	}
+
+	return nil
+}
+
+func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types.StateTree, ts *types.TipSet) error {
+	// Some initial parameters
+	FundsForMiners := types.FromFil(1_000_000)
+	LookbackEpoch := abi.ChainEpoch(32000)
+	AccountCap := types.FromFil(0)
+	BaseMinerBalance := types.FromFil(20)
+	DesiredReimbursementBalance := types.FromFil(5_000_000)
+
+	isSystemAccount := func(addr address.Address) (bool, error) {
+		id, err := address.IDFromAddress(addr)
+		if err != nil {
+			return false, xerrors.Errorf("id address: %w", err)
+		}
+
+		if id < 1000 {
+			return true, nil
+		}
+		return false, nil
+	}
+
+	minerFundsAlloc := func(pow, tpow abi.StoragePower) abi.TokenAmount {
+		return types.BigDiv(types.BigMul(pow, FundsForMiners), tpow)
+	}
+
+	// Grab lookback state for account checks
+	lbts, err := sm.ChainStore().GetTipsetByHeight(ctx, LookbackEpoch, ts, false)
+	if err != nil {
+		return xerrors.Errorf("failed to get tipset at lookback height: %w", err)
+	}
+
+	var lbtree *state.StateTree
+	if err = sm.WithStateTree(lbts.ParentState(), func(state *state.StateTree) error {
+		lbtree = state
+		return nil
+	}); err != nil {
+		return xerrors.Errorf("loading state tree failed: %w", err)
+	}
+
+	ReserveAddress, err := address.NewFromString("t090")
+	if err != nil {
+		return xerrors.Errorf("failed to parse reserve address: %w", err)
+	}
+
+	fetree, ok := tree.(forEachTree)
+	if !ok {
+		return xerrors.Errorf("fork transition state tree doesnt support ForEach (%T)", tree)
+	}
+
+	type transfer struct {
+		From address.Address
+		To   address.Address
+		Amt  abi.TokenAmount
+	}
+
+	var transfers []transfer
+
+	// Take all excess funds away, put them into the reserve account
+	err = fetree.ForEach(func(addr address.Address, act *types.Actor) error {
+		switch act.Code {
+		case builtin.AccountActorCodeID, builtin.MultisigActorCodeID, builtin.PaymentChannelActorCodeID:
+			sysAcc, err := isSystemAccount(addr)
+			if err != nil {
+				return xerrors.Errorf("checking system account: %w", err)
+			}
+
+			if !sysAcc {
+				transfers = append(transfers, transfer{
+					From: addr,
+					To:   ReserveAddress,
+					Amt:  act.Balance,
+				})
+			}
+		case builtin.StorageMinerActorCodeID:
+			var st miner.State
+			if err := sm.WithActorState(ctx, &st)(act); err != nil {
+				return xerrors.Errorf("failed to load miner state: %w", err)
+			}
+
+			var available abi.TokenAmount
+			{
+				defer func() {
+					if err := recover(); err != nil {
+						log.Warnf("Get available balance failed (%s, %s, %s): %s", addr, act.Head, act.Balance, err)
+					}
+					available = abi.NewTokenAmount(0)
+				}()
+				// this panics if the miner doesnt have enough funds to cover their locked pledge
+				available = st.GetAvailableBalance(act.Balance)
+			}
+
+			transfers = append(transfers, transfer{
+				From: addr,
+				To:   ReserveAddress,
+				Amt:  available,
+			})
+		}
+		return nil
+	})
+	if err != nil {
+		return xerrors.Errorf("foreach over state tree failed: %w", err)
+	}
+
+	// Execute transfers from previous step
+	for _, t := range transfers {
+		if err := doTransfer(tree, t.From, t.To, t.Amt); err != nil {
+			return xerrors.Errorf("transfer %s %s->%s failed: %w", t.Amt, t.From, t.To, err)
+		}
+	}
+
+	// pull up power table to give miners back some funds proportional to their power
+	var ps power.State
+	powAct, err := tree.GetActor(builtin.StoragePowerActorAddr)
+	if err != nil {
+		return xerrors.Errorf("failed to load power actor: %w", err)
+	}
+
+	cst := cbor.NewCborStore(sm.ChainStore().Blockstore())
+	if err := cst.Get(ctx, powAct.Head, &ps); err != nil {
+		return xerrors.Errorf("failed to get power actor state: %w", err)
+	}
+
+	totalPower := ps.TotalBytesCommitted
+
+	var transfersBack []transfer
+	// Now, we return some funds to places where they are needed
+	err = fetree.ForEach(func(addr address.Address, act *types.Actor) error {
+		lbact, err := lbtree.GetActor(addr)
+		if err != nil {
+			if !xerrors.Is(err, types.ErrActorNotFound) {
+				return xerrors.Errorf("failed to get actor in lookback state")
+			}
+		}
+
+		prevBalance := abi.NewTokenAmount(0)
+		if lbact != nil {
+			prevBalance = lbact.Balance
+		}
+
+		switch act.Code {
+		case builtin.AccountActorCodeID, builtin.MultisigActorCodeID, builtin.PaymentChannelActorCodeID:
+			nbalance := big.Min(prevBalance, AccountCap)
+			if nbalance.Sign() != 0 {
+				transfersBack = append(transfersBack, transfer{
+					From: ReserveAddress,
+					To:   addr,
+					Amt:  nbalance,
+				})
+			}
+		case builtin.StorageMinerActorCodeID:
+			var st miner.State
+			if err := sm.WithActorState(ctx, &st)(act); err != nil {
+				return xerrors.Errorf("failed to load miner state: %w", err)
+			}
+
+			var minfo miner.MinerInfo
+			if err := cst.Get(ctx, st.Info, &minfo); err != nil {
+				return xerrors.Errorf("failed to get miner info: %w", err)
+			}
+
+			sectorsArr, err := adt.AsArray(sm.ChainStore().Store(ctx), st.Sectors)
+			if err != nil {
+				return xerrors.Errorf("failed to load sectors array: %w", err)
+			}
+
+			slen := sectorsArr.Length()
+
+			power := types.BigMul(types.NewInt(slen), types.NewInt(uint64(minfo.SectorSize)))
+
+			mfunds := minerFundsAlloc(power, totalPower)
+			transfersBack = append(transfersBack, transfer{
+				From: ReserveAddress,
+				To:   minfo.Worker,
+				Amt:  mfunds,
+			})
+
+			// Now make sure to give each miner who had power at the lookback some FIL
+			lbact, err := lbtree.GetActor(addr)
+			if err == nil {
+				var lbst miner.State
+				if err := sm.WithActorState(ctx, &lbst)(lbact); err != nil {
+					return xerrors.Errorf("failed to load miner state: %w", err)
+				}
+
+				lbsectors, err := adt.AsArray(sm.ChainStore().Store(ctx), lbst.Sectors)
+				if err != nil {
+					return xerrors.Errorf("failed to load lb sectors array: %w", err)
+				}
+
+				if lbsectors.Length() > 0 {
+					transfersBack = append(transfersBack, transfer{
+						From: ReserveAddress,
+						To:   minfo.Worker,
+						Amt:  BaseMinerBalance,
+					})
+				}
+
+			} else {
+				log.Warnf("failed to get miner in lookback state: %s", err)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return xerrors.Errorf("foreach over state tree failed: %w", err)
+	}
+
+	for _, t := range transfersBack {
+		if err := doTransfer(tree, t.From, t.To, t.Amt); err != nil {
+			return xerrors.Errorf("transfer %s %s->%s failed: %w", t.Amt, t.From, t.To, err)
+		}
+	}
+
+	// transfer all burnt funds back to the reserve account
+	burntAct, err := tree.GetActor(builtin.BurntFundsActorAddr)
+	if err != nil {
+		return xerrors.Errorf("failed to load burnt funds actor: %w", err)
+	}
+	if err := doTransfer(tree, builtin.BurntFundsActorAddr, ReserveAddress, burntAct.Balance); err != nil {
+		return xerrors.Errorf("failed to unburn funds: %w", err)
+	}
+
+	// Top up the reimbursement service
+	reimbAddr, err := address.NewFromString("t0111")
+	if err != nil {
+		return xerrors.Errorf("failed to parse reimbursement service address")
+	}
+
+	reimb, err := tree.GetActor(reimbAddr)
+	if err != nil {
+		return xerrors.Errorf("failed to load reimbursement account actor: %w", err)
+	}
+
+	difference := types.BigSub(DesiredReimbursementBalance, reimb.Balance)
+	if err := doTransfer(tree, ReserveAddress, reimbAddr, difference); err != nil {
+		return xerrors.Errorf("failed to top up reimbursement account: %w", err)
+	}
+
+	// Now, a final sanity check to make sure the balances all check out
+	total := abi.NewTokenAmount(0)
+	err = fetree.ForEach(func(addr address.Address, act *types.Actor) error {
+		total = types.BigAdd(total, act.Balance)
+		return nil
+	})
+	if err != nil {
+		return xerrors.Errorf("checking final state balance failed: %w", err)
+	}
+
+	exp := types.FromFil(build.FilBase)
+	if !exp.Equals(total) {
+		return xerrors.Errorf("resultant state tree account balance was not correct: %s", total)
+	}
+
+	return nil
+}

chain/stmgr/forks_test.go

@@ -7,15 +7,14 @@ import (
 	"testing"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	init_ "github.com/filecoin-project/specs-actors/actors/builtin/init"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
 	"github.com/filecoin-project/specs-actors/actors/runtime"
-	"github.com/filecoin-project/specs-actors/actors/util/adt"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/chain/actors"
@@ -73,15 +72,15 @@ func (ta *testActor) Exports() []interface{} {
 	}
 }
 
-func (ta *testActor) Constructor(rt runtime.Runtime, params *adt.EmptyValue) *adt.EmptyValue {
+func (ta *testActor) Constructor(rt runtime.Runtime, params *abi.EmptyValue) *abi.EmptyValue {
 	rt.ValidateImmediateCallerAcceptAny()
 	rt.State().Create(&testActorState{11})
 	fmt.Println("NEW ACTOR ADDRESS IS: ", rt.Message().Receiver())
 
-	return adt.Empty
+	return abi.Empty
 }
 
-func (ta *testActor) TestMethod(rt runtime.Runtime, params *adt.EmptyValue) *adt.EmptyValue {
+func (ta *testActor) TestMethod(rt runtime.Runtime, params *abi.EmptyValue) *abi.EmptyValue {
 	rt.ValidateImmediateCallerAcceptAny()
 	var st testActorState
 	rt.State().Readonly(&st)
@@ -96,7 +95,7 @@ func (ta *testActor) TestMethod(rt runtime.Runtime, params *adt.EmptyValue) *adt
 		}
 	}
 
-	return adt.Empty
+	return abi.Empty
 }
 
 func TestForkHeightTriggers(t *testing.T) {
@@ -119,7 +118,7 @@ func TestForkHeightTriggers(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	stmgr.ForksAtHeight[testForkHeight] = func(ctx context.Context, sm *StateManager, st types.StateTree) error {
+	stmgr.ForksAtHeight[testForkHeight] = func(ctx context.Context, sm *StateManager, st types.StateTree, ts *types.TipSet) error {
 		cst := cbor.NewCborStore(sm.ChainStore().Blockstore())
 
 		act, err := st.GetActor(taddr)

chain/stmgr/stmgr.go

@@ -3,9 +3,12 @@ package stmgr
 import (
 	"context"
 	"fmt"
-	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 	"sync"
 
+	"github.com/filecoin-project/go-state-types/network"
+
+	"github.com/filecoin-project/specs-actors/actors/builtin/power"
+
 	"github.com/filecoin-project/specs-actors/actors/builtin/multisig"
 
 	"github.com/filecoin-project/go-address"
@@ -17,8 +20,8 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/market"
 	"github.com/filecoin-project/specs-actors/actors/builtin/reward"
@@ -144,7 +147,7 @@ func (sm *StateManager) ExecutionTrace(ctx context.Context, ts *types.TipSet) (c
 
 type ExecCallback func(cid.Cid, *types.Message, *vm.ApplyRet) error
 
-func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEpoch, pstate cid.Cid, bms []store.BlockMessages, epoch abi.ChainEpoch, r vm.Rand, cb ExecCallback, baseFee abi.TokenAmount) (cid.Cid, cid.Cid, error) {
+func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEpoch, pstate cid.Cid, bms []store.BlockMessages, epoch abi.ChainEpoch, r vm.Rand, cb ExecCallback, baseFee abi.TokenAmount, ts *types.TipSet) (cid.Cid, cid.Cid, error) {
 
 	vmopt := &vm.VMOpts{
 		StateBase:      pstate,
@@ -153,6 +156,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
 		Bstore:         sm.cs.Blockstore(),
 		Syscalls:       sm.cs.VMSys(),
 		CircSupplyCalc: sm.GetCirculatingSupply,
+		NtwkVersion:    sm.GetNtwkVersion,
 		BaseFee:        baseFee,
 	}
 
@@ -197,7 +201,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
 
 	for i := parentEpoch; i < epoch; i++ {
 		// handle state forks
-		err = sm.handleStateForks(ctx, vmi.StateTree(), i)
+		err = sm.handleStateForks(ctx, vmi.StateTree(), i, ts)
 		if err != nil {
 			return cid.Undef, cid.Undef, xerrors.Errorf("error handling state forks: %w", err)
 		}
@@ -323,7 +327,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, ts *types.TipSet
 
 	var parentEpoch abi.ChainEpoch
 	pstate := blks[0].ParentStateRoot
-	if len(blks[0].Parents) > 0 {
+	if blks[0].Height > 0 {
 		parent, err := sm.cs.GetBlock(blks[0].Parents[0])
 		if err != nil {
 			return cid.Undef, cid.Undef, xerrors.Errorf("getting parent block: %w", err)
@@ -337,7 +341,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, ts *types.TipSet
 		cids[i] = v.Cid()
 	}
 
-	r := store.NewChainRand(sm.cs, cids, blks[0].Height)
+	r := store.NewChainRand(sm.cs, cids)
 
 	blkmsgs, err := sm.cs.BlockMsgsForTipset(ts)
 	if err != nil {
@@ -346,7 +350,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, ts *types.TipSet
 
 	baseFee := blks[0].ParentBaseFee
 
-	return sm.ApplyBlocks(ctx, parentEpoch, pstate, blkmsgs, blks[0].Height, r, cb, baseFee)
+	return sm.ApplyBlocks(ctx, parentEpoch, pstate, blkmsgs, blks[0].Height, r, cb, baseFee, ts)
 }
 
 func (sm *StateManager) parentState(ts *types.TipSet) cid.Cid {
@@ -787,7 +791,7 @@ type genesisActor struct {
 	initBal abi.TokenAmount
 }
 
-// sets up information about the non-multisig actors in the genesis state
+// sets up information about the actors in the genesis state
 func (sm *StateManager) setupGenesisActors(ctx context.Context) error {
 
 	gi := genesisInfo{}
@@ -850,21 +854,24 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error {
 			}
 
 		} else if act.Code == builtin.AccountActorCodeID {
+			// should exclude burnt funds actor and "remainder account actor"
 			// should only ever be "faucet" accounts in testnets
 			kaddr, err := address.NewFromBytes([]byte(k))
 			if err != nil {
 				return xerrors.Errorf("decoding address: %w", err)
 			}
 
-			kid, err := sTree.LookupID(kaddr)
-			if err != nil {
-				return xerrors.Errorf("resolving address: %w", err)
-			}
+			if kaddr != builtin.BurntFundsActorAddr {
+				kid, err := sTree.LookupID(kaddr)
+				if err != nil {
+					return xerrors.Errorf("resolving address: %w", err)
+				}
 
-			gi.genesisActors = append(gi.genesisActors, genesisActor{
-				addr:    kid,
-				initBal: act.Balance,
-			})
+				gi.genesisActors = append(gi.genesisActors, genesisActor{
+					addr:    kid,
+					initBal: act.Balance,
+				})
+			}
 		}
 		return nil
 	})
@@ -888,6 +895,83 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error {
 	return nil
 }
 
+// sets up information about the actors in the genesis state
+// For testnet we use a hardcoded set of multisig states, instead of what's actually in the genesis multisigs
+// We also do not consider ANY account actors (including the faucet)
+func (sm *StateManager) setupGenesisActorsTestnet(ctx context.Context) error {
+
+	gi := genesisInfo{}
+
+	gb, err := sm.cs.GetGenesis()
+	if err != nil {
+		return xerrors.Errorf("getting genesis block: %w", err)
+	}
+
+	gts, err := types.NewTipSet([]*types.BlockHeader{gb})
+	if err != nil {
+		return xerrors.Errorf("getting genesis tipset: %w", err)
+	}
+
+	st, _, err := sm.TipSetState(ctx, gts)
+	if err != nil {
+		return xerrors.Errorf("getting genesis tipset state: %w", err)
+	}
+
+	cst := cbor.NewCborStore(sm.cs.Blockstore())
+	sTree, err := state.LoadStateTree(cst, st)
+	if err != nil {
+		return xerrors.Errorf("loading state tree: %w", err)
+	}
+
+	gi.genesisMarketFunds, err = getFilMarketLocked(ctx, sTree)
+	if err != nil {
+		return xerrors.Errorf("setting up genesis market funds: %w", err)
+	}
+
+	gi.genesisPledge, err = getFilPowerLocked(ctx, sTree)
+	if err != nil {
+		return xerrors.Errorf("setting up genesis pledge: %w", err)
+	}
+
+	totalsByEpoch := make(map[abi.ChainEpoch]abi.TokenAmount)
+
+	// 6 months
+	sixMonths := abi.ChainEpoch(183 * builtin.EpochsInDay)
+	totalsByEpoch[sixMonths] = big.NewInt(49_929_341)
+	totalsByEpoch[sixMonths] = big.Add(totalsByEpoch[sixMonths], big.NewInt(32_787_700))
+
+	// 1 year
+	oneYear := abi.ChainEpoch(365 * builtin.EpochsInDay)
+	totalsByEpoch[oneYear] = big.NewInt(22_421_712)
+
+	// 2 years
+	twoYears := abi.ChainEpoch(2 * 365 * builtin.EpochsInDay)
+	totalsByEpoch[twoYears] = big.NewInt(7_223_364)
+
+	// 3 years
+	threeYears := abi.ChainEpoch(3 * 365 * builtin.EpochsInDay)
+	totalsByEpoch[threeYears] = big.NewInt(87_637_883)
+
+	// 6 years
+	sixYears := abi.ChainEpoch(6 * 365 * builtin.EpochsInDay)
+	totalsByEpoch[sixYears] = big.NewInt(100_000_000)
+	totalsByEpoch[sixYears] = big.Add(totalsByEpoch[sixYears], big.NewInt(300_000_000))
+
+	gi.genesisMsigs = make([]multisig.State, 0, len(totalsByEpoch))
+	for k, v := range totalsByEpoch {
+		ns := multisig.State{
+			InitialBalance: v,
+			UnlockDuration: k,
+			PendingTxns:    cid.Undef,
+		}
+		gi.genesisMsigs = append(gi.genesisMsigs, ns)
+	}
+
+	sm.genInfo = &gi
+
+	return nil
+}
+
 // GetVestedFunds returns all funds that have "left" actors that are in the genesis state:
 // - For Multisigs, it counts the actual amounts that have vested at the given epoch
 // - For Accounts, it counts max(currentBalance - genesisBalance, 0).
@@ -898,7 +982,7 @@ func (sm *StateManager) GetFilVested(ctx context.Context, height abi.ChainEpoch,
 		vf = big.Add(vf, au)
 	}
 
-	// these should only ever be "faucet" accounts in testnets
+	// there should not be any such accounts in testnet (and also none in mainnet?)
 	for _, v := range sm.genInfo.genesisActors {
 		act, err := st.GetActor(v.addr)
 		if err != nil {
@@ -984,34 +1068,34 @@ func GetFilBurnt(ctx context.Context, st *state.StateTree) (abi.TokenAmount, err
 	return burnt.Balance, nil
 }
 
-func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.ChainEpoch, st *state.StateTree) (abi.TokenAmount, error) {
+func (sm *StateManager) GetCirculatingSupplyDetailed(ctx context.Context, height abi.ChainEpoch, st *state.StateTree) (api.CirculatingSupply, error) {
 	sm.genesisMsigLk.Lock()
 	defer sm.genesisMsigLk.Unlock()
 	if sm.genInfo == nil {
-		err := sm.setupGenesisActors(ctx)
+		err := sm.setupGenesisActorsTestnet(ctx)
 		if err != nil {
-			return big.Zero(), xerrors.Errorf("failed to setup genesis information: %w", err)
+			return api.CirculatingSupply{}, xerrors.Errorf("failed to setup genesis information: %w", err)
 		}
 	}
 
 	filVested, err := sm.GetFilVested(ctx, height, st)
 	if err != nil {
-		return big.Zero(), xerrors.Errorf("failed to calculate filVested: %w", err)
+		return api.CirculatingSupply{}, xerrors.Errorf("failed to calculate filVested: %w", err)
 	}
 
 	filMined, err := GetFilMined(ctx, st)
 	if err != nil {
-		return big.Zero(), xerrors.Errorf("failed to calculate filMined: %w", err)
+		return api.CirculatingSupply{}, xerrors.Errorf("failed to calculate filMined: %w", err)
 	}
 
 	filBurnt, err := GetFilBurnt(ctx, st)
 	if err != nil {
-		return big.Zero(), xerrors.Errorf("failed to calculate filBurnt: %w", err)
+		return api.CirculatingSupply{}, xerrors.Errorf("failed to calculate filBurnt: %w", err)
 	}
 
 	filLocked, err := sm.GetFilLocked(ctx, st)
 	if err != nil {
-		return big.Zero(), xerrors.Errorf("failed to calculate filLocked: %w", err)
+		return api.CirculatingSupply{}, xerrors.Errorf("failed to calculate filLocked: %w", err)
 	}
 
 	ret := types.BigAdd(filVested, filMined)
@@ -1022,5 +1106,36 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha
 		ret = big.Zero()
 	}
 
-	return ret, nil
+	return api.CirculatingSupply{
+		FilVested:      filVested,
+		FilMined:       filMined,
+		FilBurnt:       filBurnt,
+		FilLocked:      filLocked,
+		FilCirculating: ret,
+	}, nil
+}
+
+func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.ChainEpoch, st *state.StateTree) (abi.TokenAmount, error) {
+	csi, err := sm.GetCirculatingSupplyDetailed(ctx, height, st)
+	if err != nil {
+		return big.Zero(), err
+	}
+
+	return csi.FilCirculating, nil
+}
+
+func (sm *StateManager) GetNtwkVersion(ctx context.Context, height abi.ChainEpoch) network.Version {
+	if build.UseNewestNetwork() {
+		return build.NewestNetworkVersion
+	}
+
+	if height <= build.UpgradeBreezeHeight {
+		return network.Version0
+	}
+
+	if height <= build.UpgradeSmokeHeight {
+		return network.Version1
+	}
+
+	return build.NewestNetworkVersion
 }

chain/stmgr/utils.go

@@ -3,8 +3,14 @@ package stmgr
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"os"
 	"reflect"
+	"runtime"
+	"strings"
+
+	saruntime "github.com/filecoin-project/specs-actors/actors/runtime"
+	"github.com/filecoin-project/specs-actors/actors/runtime/proof"
 
 	cid "github.com/ipfs/go-cid"
 	cbor "github.com/ipfs/go-ipld-cbor"
@@ -13,8 +19,9 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
-	"github.com/filecoin-project/sector-storage/ffiwrapper"
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/account"
 	"github.com/filecoin-project/specs-actors/actors/builtin/cron"
@@ -26,7 +33,6 @@ import (
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 	"github.com/filecoin-project/specs-actors/actors/builtin/reward"
 	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
-	"github.com/filecoin-project/specs-actors/actors/crypto"
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
 
 	"github.com/filecoin-project/lotus/api"
@@ -157,13 +163,13 @@ func MinerSectorInfo(ctx context.Context, sm *StateManager, maddr address.Addres
 		return nil, err
 	}
 	if !ok {
-		return nil, xerrors.New("sector not found")
+		return nil, nil
 	}
 
 	return sectorInfo, nil
 }
 
-func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address, filter *abi.BitField, filterOut bool) ([]*api.ChainSectorInfo, error) {
+func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address, filter *bitfield.BitField, filterOut bool) ([]*api.ChainSectorInfo, error) {
 	var mas miner.State
 	_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
 	if err != nil {
@@ -173,8 +179,8 @@ func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet,
 	return LoadSectorsFromSet(ctx, sm.ChainStore().Blockstore(), mas.Sectors, filter, filterOut)
 }
 
-func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *StateManager, st cid.Cid, maddr address.Address, rand abi.PoStRandomness) ([]abi.SectorInfo, error) {
-	var partsProving []abi.BitField
+func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *StateManager, st cid.Cid, maddr address.Address, rand abi.PoStRandomness) ([]proof.SectorInfo, error) {
+	var partsProving []bitfield.BitField
 	var mas *miner.State
 	var info *miner.MinerInfo
 
@@ -261,7 +267,7 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S
 		return nil, xerrors.Errorf("failed to load sectors amt: %w", err)
 	}
 
-	out := make([]abi.SectorInfo, len(ids))
+	out := make([]proof.SectorInfo, len(ids))
 	for i, n := range ids {
 		sid := sectors[n]
 
@@ -272,7 +278,7 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S
 			return nil, xerrors.Errorf("failed to find sector %d", sid)
 		}
 
-		out[i] = abi.SectorInfo{
+		out[i] = proof.SectorInfo{
 			SealProof:    spt,
 			SectorNumber: sinfo.SectorNumber,
 			SealedCID:    sinfo.SealedCID,
@@ -293,14 +299,8 @@ func StateMinerInfo(ctx context.Context, sm *StateManager, ts *types.TipSet, mad
 }
 
 func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (bool, error) {
-	var mas miner.State
-	_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
-	if err != nil {
-		return false, xerrors.Errorf("(get miner slashed) failed to load miner actor state")
-	}
-
 	var spas power.State
-	_, err = sm.LoadActorState(ctx, builtin.StoragePowerActorAddr, &spas, ts)
+	_, err := sm.LoadActorState(ctx, builtin.StoragePowerActorAddr, &spas, ts)
 	if err != nil {
 		return false, xerrors.Errorf("(get miner slashed) failed to load power actor state")
 	}
@@ -393,7 +393,7 @@ func ListMinerActors(ctx context.Context, sm *StateManager, ts *types.TipSet) ([
 	return miners, nil
 }
 
-func LoadSectorsFromSet(ctx context.Context, bs blockstore.Blockstore, ssc cid.Cid, filter *abi.BitField, filterOut bool) ([]*api.ChainSectorInfo, error) {
+func LoadSectorsFromSet(ctx context.Context, bs blockstore.Blockstore, ssc cid.Cid, filter *bitfield.BitField, filterOut bool) ([]*api.ChainSectorInfo, error) {
 	a, err := adt.AsArray(store.ActorStore(ctx, bs), ssc)
 	if err != nil {
 		return nil, err
@@ -438,7 +438,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
 		return cid.Undef, nil, err
 	}
 
-	r := store.NewChainRand(sm.cs, ts.Cids(), height)
+	r := store.NewChainRand(sm.cs, ts.Cids())
 	vmopt := &vm.VMOpts{
 		StateBase:      base,
 		Epoch:          height,
@@ -446,6 +446,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
 		Bstore:         sm.cs.Blockstore(),
 		Syscalls:       sm.cs.VMSys(),
 		CircSupplyCalc: sm.GetCirculatingSupply,
+		NtwkVersion:    sm.GetNtwkVersion,
 		BaseFee:        ts.Blocks()[0].ParentBaseFee,
 	}
 	vmi, err := vm.NewVM(vmopt)
@@ -455,7 +456,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
 
 	for i := ts.Height(); i < height; i++ {
 		// handle state forks
-		err = sm.handleStateForks(ctx, vmi.StateTree(), i)
+		err = sm.handleStateForks(ctx, vmi.StateTree(), i, ts)
 		if err != nil {
 			return cid.Undef, nil, xerrors.Errorf("error handling state forks: %w", err)
 		}
@@ -501,7 +502,7 @@ func GetLookbackTipSetForRound(ctx context.Context, sm *StateManager, ts *types.
 	return lbts, nil
 }
 
-func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcn beacon.RandomBeacon, tsk types.TipSetKey, round abi.ChainEpoch, maddr address.Address, pv ffiwrapper.Verifier) (*api.MiningBaseInfo, error) {
+func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule, tsk types.TipSetKey, round abi.ChainEpoch, maddr address.Address, pv ffiwrapper.Verifier) (*api.MiningBaseInfo, error) {
 	ts, err := sm.ChainStore().LoadTipSet(tsk)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to load tipset for mining base: %w", err)
@@ -516,7 +517,7 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcn beacon.RandomBe
 		prev = &types.BeaconEntry{}
 	}
 
-	entries, err := beacon.BeaconEntriesForBlock(ctx, bcn, round, *prev)
+	entries, err := beacon.BeaconEntriesForBlock(ctx, bcs, round, ts.Height(), *prev)
 	if err != nil {
 		return nil, err
 	}
@@ -592,14 +593,14 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcn beacon.RandomBe
 	}, nil
 }
 
-type methodMeta struct {
+type MethodMeta struct {
 	Name string
 
 	Params reflect.Type
 	Ret    reflect.Type
 }
 
-var MethodsMap = map[cid.Cid][]methodMeta{}
+var MethodsMap = map[cid.Cid]map[abi.MethodNum]MethodMeta{}
 
 func init() {
 	cidToMethods := map[cid.Cid][2]interface{}{
@@ -617,25 +618,65 @@ func init() {
 	}
 
 	for c, m := range cidToMethods {
-		rt := reflect.TypeOf(m[0])
-		nf := rt.NumField()
+		exports := m[1].(saruntime.Invokee).Exports()
+		methods := make(map[abi.MethodNum]MethodMeta, len(exports))
 
-		MethodsMap[c] = append(MethodsMap[c], methodMeta{
+		// Explicitly add send, it's special.
+		methods[builtin.MethodSend] = MethodMeta{
 			Name:   "Send",
-			Params: reflect.TypeOf(new(adt.EmptyValue)),
-			Ret:    reflect.TypeOf(new(adt.EmptyValue)),
-		})
-
-		exports := m[1].(abi.Invokee).Exports()
-		for i := 0; i < nf; i++ {
-			export := reflect.TypeOf(exports[i+1])
-
-			MethodsMap[c] = append(MethodsMap[c], methodMeta{
-				Name:   rt.Field(i).Name,
-				Params: export.In(1),
-				Ret:    export.Out(0),
-			})
+			Params: reflect.TypeOf(new(abi.EmptyValue)),
+			Ret:    reflect.TypeOf(new(abi.EmptyValue)),
 		}
+
+		// Learn method names from the builtin.Methods* structs.
+		rv := reflect.ValueOf(m[0])
+		rt := rv.Type()
+		nf := rt.NumField()
+		methodToName := make([]string, len(exports))
+		for i := 0; i < nf; i++ {
+			name := rt.Field(i).Name
+			number := rv.Field(i).Interface().(abi.MethodNum)
+			methodToName[number] = name
+		}
+
+		// Iterate over exported methods. Some of these _may_ be nil and
+		// must be skipped.
+		for number, export := range exports {
+			if export == nil {
+				continue
+			}
+
+			ev := reflect.ValueOf(export)
+			et := ev.Type()
+
+			// Make sure the method name is correct.
+			// This is just a nice sanity check.
+			fnName := runtime.FuncForPC(ev.Pointer()).Name()
+			fnName = strings.TrimSuffix(fnName[strings.LastIndexByte(fnName, '.')+1:], "-fm")
+			mName := methodToName[number]
+			if mName != fnName {
+				panic(fmt.Sprintf(
+					"actor method name is %s but exported method name is %s",
+					fnName, mName,
+				))
+			}
+
+			switch abi.MethodNum(number) {
+			case builtin.MethodSend:
+				panic("method 0 is reserved for Send")
+			case builtin.MethodConstructor:
+				if fnName != "Constructor" {
+					panic("method 1 is reserved for Constructor")
+				}
+			}
+
+			methods[abi.MethodNum(number)] = MethodMeta{
+				Name:   fnName,
+				Params: et.In(1),
+				Ret:    et.Out(0),
+			}
+		}
+		MethodsMap[c] = methods
 	}
 }
 
@@ -645,7 +686,10 @@ func GetReturnType(ctx context.Context, sm *StateManager, to address.Address, me
 		return nil, xerrors.Errorf("getting actor: %w", err)
 	}
 
-	m := MethodsMap[act.Code][method]
+	m, found := MethodsMap[act.Code][method]
+	if !found {
+		return nil, fmt.Errorf("unknown method %d for actor %s", method, act.Code)
+	}
 	return reflect.New(m.Ret.Elem()).Interface().(cbg.CBORUnmarshaler), nil
 }
 

chain/store/basefee.go

@@ -3,15 +3,30 @@ package store
 import (
 	"context"
 
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
 )
 
 func computeNextBaseFee(baseFee types.BigInt, gasLimitUsed int64, noOfBlocks int) types.BigInt {
-	delta := gasLimitUsed/int64(noOfBlocks) - build.BlockGasTarget
+	// deta := 1/PackingEfficiency * gasLimitUsed/noOfBlocks - build.BlockGasTarget
+	// change := baseFee * deta / BlockGasTarget / BaseFeeMaxChangeDenom
+	// nextBaseFee = baseFee + change
+	// nextBaseFee = max(nextBaseFee, build.MinimumBaseFee)
+
+	delta := build.PackingEfficiencyDenom * gasLimitUsed / (int64(noOfBlocks) * build.PackingEfficiencyNum)
+	delta -= build.BlockGasTarget
+
+	// cap change at 12.5% (BaseFeeMaxChangeDenom) by capping delta
+	if delta > build.BlockGasTarget {
+		delta = build.BlockGasTarget
+	}
+	if delta < -build.BlockGasTarget {
+		delta = -build.BlockGasTarget
+	}
 
 	change := big.Mul(baseFee, big.NewInt(delta))
 	change = big.Div(change, big.NewInt(build.BlockGasTarget))
@@ -25,18 +40,35 @@ func computeNextBaseFee(baseFee types.BigInt, gasLimitUsed int64, noOfBlocks int
 }
 
 func (cs *ChainStore) ComputeBaseFee(ctx context.Context, ts *types.TipSet) (abi.TokenAmount, error) {
+	if ts.Height() > build.UpgradeBreezeHeight && ts.Height() < build.UpgradeBreezeHeight+build.BreezeGasTampingDuration {
+		return abi.NewTokenAmount(100), nil
+	}
+
 	zero := abi.NewTokenAmount(0)
+
+	// totalLimit is sum of GasLimits of unique messages in a tipset
 	totalLimit := int64(0)
+
+	seen := make(map[cid.Cid]struct{})
+
 	for _, b := range ts.Blocks() {
 		msg1, msg2, err := cs.MessagesForBlock(b)
 		if err != nil {
 			return zero, xerrors.Errorf("error getting messages for: %s: %w", b.Cid(), err)
 		}
 		for _, m := range msg1 {
-			totalLimit += m.GasLimit
+			c := m.Cid()
+			if _, ok := seen[c]; !ok {
+				totalLimit += m.GasLimit
+				seen[c] = struct{}{}
+			}
 		}
 		for _, m := range msg2 {
-			totalLimit += m.Message.GasLimit
+			c := m.Cid()
+			if _, ok := seen[c]; !ok {
+				totalLimit += m.Message.GasLimit
+				seen[c] = struct{}{}
+			}
 		}
 	}
 	parentBaseFee := ts.Blocks()[0].ParentBaseFee

chain/store/basefee_test.go

@@ -18,10 +18,10 @@ func TestBaseFee(t *testing.T) {
 	}{
 		{100e6, 0, 1, 87.5e6},
 		{100e6, 0, 5, 87.5e6},
-		{100e6, build.BlockGasTarget, 1, 100e6},
-		{100e6, build.BlockGasTarget * 2, 2, 100e6},
+		{100e6, build.BlockGasTarget, 1, 103.125e6},
+		{100e6, build.BlockGasTarget * 2, 2, 103.125e6},
 		{100e6, build.BlockGasLimit * 2, 2, 112.5e6},
-		{100e6, build.BlockGasLimit * 1.5, 2, 106.25e6},
+		{100e6, build.BlockGasLimit * 1.5, 2, 110937500},
 	}
 
 	for _, test := range tests {

chain/store/index.go

@@ -5,8 +5,8 @@ import (
 	"os"
 	"strconv"
 
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/specs-actors/actors/abi"
 	lru "github.com/hashicorp/golang-lru"
 	"golang.org/x/xerrors"
 )

chain/store/index_test.go

@@ -5,11 +5,11 @@ import (
 	"context"
 	"testing"
 
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/chain/gen"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types/mock"
 	"github.com/filecoin-project/lotus/lib/blockstore"
-	"github.com/filecoin-project/specs-actors/actors/abi"
 	datastore "github.com/ipfs/go-datastore"
 	syncds "github.com/ipfs/go-datastore/sync"
 	"github.com/stretchr/testify/assert"
@@ -42,7 +42,7 @@ func TestIndexSeeks(t *testing.T) {
 	if err := cs.PutTipSet(ctx, mock.TipSet(gen)); err != nil {
 		t.Fatal(err)
 	}
-	cs.SetGenesis(gen)
+	assert.NoError(t, cs.SetGenesis(gen))
 
 	// Put 113 blocks from genesis
 	for i := 0; i < 113; i++ {

chain/store/store.go

@@ -10,11 +10,11 @@ import (
 	"strconv"
 	"sync"
 
-	"github.com/filecoin-project/specs-actors/actors/crypto"
+	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/minio/blake2b-simd"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
 
 	"github.com/filecoin-project/lotus/api"
@@ -49,6 +49,7 @@ var chainHeadKey = dstore.NewKey("head")
 var blockValidationCacheKeyPrefix = dstore.NewKey("blockValidation")
 
 var DefaultTipSetCacheSize = 8192
+var DefaultMsgMetaCacheSize = 2048
 
 func init() {
 	if s := os.Getenv("LOTUS_CHAIN_TIPSET_CACHE"); s != "" {
@@ -58,6 +59,14 @@ func init() {
 		}
 		DefaultTipSetCacheSize = tscs
 	}
+
+	if s := os.Getenv("LOTUS_CHAIN_MSGMETA_CACHE"); s != "" {
+		mmcs, err := strconv.Atoi(s)
+		if err != nil {
+			log.Errorf("failed to parse 'LOTUS_CHAIN_MSGMETA_CACHE' env var: %s", err)
+		}
+		DefaultMsgMetaCacheSize = mmcs
+	}
 }
 
 // ReorgNotifee represents a callback that gets called upon reorgs.
@@ -97,7 +106,7 @@ type ChainStore struct {
 }
 
 func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder) *ChainStore {
-	c, _ := lru.NewARC(2048)
+	c, _ := lru.NewARC(DefaultMsgMetaCacheSize)
 	tsc, _ := lru.NewARC(DefaultTipSetCacheSize)
 	cs := &ChainStore{
 		bs:       bs,
@@ -462,7 +471,7 @@ func (cs *ChainStore) IsAncestorOf(a, b *types.TipSet) (bool, error) {
 
 	cur := b
 	for !a.Equals(cur) && cur.Height() > a.Height() {
-		next, err := cs.LoadTipSet(b.Parents())
+		next, err := cs.LoadTipSet(cur.Parents())
 		if err != nil {
 			return false, err
 		}
@@ -483,6 +492,10 @@ func (cs *ChainStore) NearestCommonAncestor(a, b *types.TipSet) (*types.TipSet,
 }
 
 func (cs *ChainStore) ReorgOps(a, b *types.TipSet) ([]*types.TipSet, []*types.TipSet, error) {
+	return ReorgOps(cs.LoadTipSet, a, b)
+}
+
+func ReorgOps(lts func(types.TipSetKey) (*types.TipSet, error), a, b *types.TipSet) ([]*types.TipSet, []*types.TipSet, error) {
 	left := a
 	right := b
 
@@ -490,7 +503,7 @@ func (cs *ChainStore) ReorgOps(a, b *types.TipSet) ([]*types.TipSet, []*types.Ti
 	for !left.Equals(right) {
 		if left.Height() > right.Height() {
 			leftChain = append(leftChain, left)
-			par, err := cs.LoadTipSet(left.Parents())
+			par, err := lts(left.Parents())
 			if err != nil {
 				return nil, nil, err
 			}
@@ -498,7 +511,7 @@ func (cs *ChainStore) ReorgOps(a, b *types.TipSet) ([]*types.TipSet, []*types.Ti
 			left = par
 		} else {
 			rightChain = append(rightChain, right)
-			par, err := cs.LoadTipSet(right.Parents())
+			par, err := lts(right.Parents())
 			if err != nil {
 				log.Infof("failed to fetch right.Parents: %s", err)
 				return nil, nil, err
@@ -509,6 +522,7 @@ func (cs *ChainStore) ReorgOps(a, b *types.TipSet) ([]*types.TipSet, []*types.Ti
 	}
 
 	return leftChain, rightChain, nil
+
 }
 
 // GetHeaviestTipSet returns the current heaviest tipset known (i.e. our head).
@@ -829,7 +843,7 @@ type mmCids struct {
 	secpk []cid.Cid
 }
 
-func (cs *ChainStore) readMsgMetaCids(mmc cid.Cid) ([]cid.Cid, []cid.Cid, error) {
+func (cs *ChainStore) ReadMsgMetaCids(mmc cid.Cid) ([]cid.Cid, []cid.Cid, error) {
 	o, ok := cs.mmCache.Get(mmc)
 	if ok {
 		mmcids := o.(*mmCids)
@@ -885,7 +899,7 @@ func (cs *ChainStore) GetPath(ctx context.Context, from types.TipSetKey, to type
 }
 
 func (cs *ChainStore) MessagesForBlock(b *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) {
-	blscids, secpkcids, err := cs.readMsgMetaCids(b.Messages)
+	blscids, secpkcids, err := cs.ReadMsgMetaCids(b.Messages)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -925,7 +939,7 @@ func (cs *ChainStore) LoadMessagesFromCids(cids []cid.Cid) ([]*types.Message, er
 	for i, c := range cids {
 		m, err := cs.GetMessage(c)
 		if err != nil {
-			return nil, xerrors.Errorf("failed to get message: (%s):%d: %w", err, c, i)
+			return nil, xerrors.Errorf("failed to get message: (%s):%d: %w", c, i, err)
 		}
 
 		msgs = append(msgs, m)
@@ -939,7 +953,7 @@ func (cs *ChainStore) LoadSignedMessagesFromCids(cids []cid.Cid) ([]*types.Signe
 	for i, c := range cids {
 		m, err := cs.GetSignedMessage(c)
 		if err != nil {
-			return nil, xerrors.Errorf("failed to get message: (%s):%d: %w", err, c, i)
+			return nil, xerrors.Errorf("failed to get message: (%s):%d: %w", c, i, err)
 		}
 
 		msgs = append(msgs, m)
@@ -1109,7 +1123,7 @@ func (cs *ChainStore) GetTipsetByHeight(ctx context.Context, h abi.ChainEpoch, t
 	return cs.LoadTipSet(lbts.Parents())
 }
 
-func recurseLinks(bs bstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid, error) {
+func recurseLinks(bs bstore.Blockstore, walked *cid.Set, root cid.Cid, in []cid.Cid) ([]cid.Cid, error) {
 	if root.Prefix().Codec != cid.DagCBOR {
 		return in, nil
 	}
@@ -1126,9 +1140,14 @@ func recurseLinks(bs bstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid,
 			return
 		}
 
+		// traversed this already...
+		if !walked.Visit(c) {
+			return
+		}
+
 		in = append(in, c)
 		var err error
-		in, err = recurseLinks(bs, c, in)
+		in, err = recurseLinks(bs, walked, c, in)
 		if err != nil {
 			rerr = err
 		}
@@ -1140,12 +1159,13 @@ func recurseLinks(bs bstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid,
 	return in, rerr
 }
 
-func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, w io.Writer) error {
+func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, w io.Writer) error {
 	if ts == nil {
 		ts = cs.GetHeaviestTipSet()
 	}
 
 	seen := cid.NewSet()
+	walked := cid.NewSet()
 
 	h := &car.CarHeader{
 		Roots:   ts.Cids(),
@@ -1177,19 +1197,24 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, w io.Writer)
 			return xerrors.Errorf("unmarshaling block header (cid=%s): %w", blk, err)
 		}
 
-		for _, p := range b.Parents {
-			blocksToWalk = append(blocksToWalk, p)
-		}
-
-		cids, err := recurseLinks(cs.bs, b.Messages, []cid.Cid{b.Messages})
+		cids, err := recurseLinks(cs.bs, walked, b.Messages, []cid.Cid{b.Messages})
 		if err != nil {
 			return xerrors.Errorf("recursing messages failed: %w", err)
 		}
 
+		if b.Height > 0 {
+			for _, p := range b.Parents {
+				blocksToWalk = append(blocksToWalk, p)
+			}
+		} else {
+			// include the genesis block
+			cids = append(cids, b.Parents...)
+		}
+
 		out := cids
 
-		if b.Height == 0 {
-			cids, err := recurseLinks(cs.bs, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot})
+		if b.Height == 0 || b.Height > ts.Height()-inclRecentRoots {
+			cids, err := recurseLinks(cs.bs, walked, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot})
 			if err != nil {
 				return xerrors.Errorf("recursing genesis state failed: %w", err)
 			}
@@ -1272,14 +1297,12 @@ func (cs *ChainStore) GetLatestBeaconEntry(ts *types.TipSet) (*types.BeaconEntry
 type chainRand struct {
 	cs   *ChainStore
 	blks []cid.Cid
-	bh   abi.ChainEpoch
 }
 
-func NewChainRand(cs *ChainStore, blks []cid.Cid, bheight abi.ChainEpoch) vm.Rand {
+func NewChainRand(cs *ChainStore, blks []cid.Cid) vm.Rand {
 	return &chainRand{
 		cs:   cs,
 		blks: blks,
-		bh:   bheight,
 	}
 }
 

chain/store/store_test.go

@@ -7,12 +7,12 @@ import (
 
 	datastore "github.com/ipfs/go-datastore"
 
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
-	"github.com/filecoin-project/specs-actors/actors/crypto"
 
 	"github.com/filecoin-project/lotus/chain/gen"
 	"github.com/filecoin-project/lotus/chain/store"
@@ -96,7 +96,7 @@ func TestChainExportImport(t *testing.T) {
 	}
 
 	buf := new(bytes.Buffer)
-	if err := cg.ChainStore().Export(context.TODO(), last, buf); err != nil {
+	if err := cg.ChainStore().Export(context.TODO(), last, 0, buf); err != nil {
 		t.Fatal(err)
 	}