Butterfly genesis

.circleci/config.yml

@@ -5,7 +5,7 @@ orbs:
 executors:
   golang:
     docker:
-      - image: circleci/golang:1.14.6
+      - image: circleci/golang:1.14.2
     resource_class: 2xlarge
   ubuntu:
     docker:
@@ -113,7 +113,7 @@ jobs:
   test: &test
     description: |
       Run tests with gotestsum.
-    parameters: &test-params
+    parameters:
       executor:
         type: executor
         default: golang
@@ -134,7 +134,7 @@ jobs:
         description: Test suite name to report to CircleCI.
       gotestsum-format:
         type: string
-        default: pkgname-and-test-fails
+        default: short
         description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
       coverage:
         type: string
@@ -156,28 +156,21 @@ jobs:
       - download-params
       - go/install-gotestsum:
           gobin: $HOME/.local/bin
-          version: 0.5.2
       - run:
           name: go test
           environment:
+            GOTESTSUM_JUNITFILE: /tmp/test-reports/<< parameters.test-suite-name >>/junit.xml
+            GOTESTSUM_FORMAT: << parameters.gotestsum-format >>
             LOTUS_TEST_WINDOW_POST: << parameters.winpost-test >>
-            SKIP_CONFORMANCE: "1"
           command: |
             mkdir -p /tmp/test-reports/<< parameters.test-suite-name >>
-            mkdir -p /tmp/test-artifacts
-            gotestsum \
-              --format << parameters.gotestsum-format >> \
-              --junitfile /tmp/test-reports/<< parameters.test-suite-name >>/junit.xml \
-              --jsonfile /tmp/test-artifacts/<< parameters.test-suite-name >>.json \
-              -- \
+            gotestsum -- \
               << parameters.coverage >> \
               << parameters.go-test-flags >> \
               << parameters.packages >>
           no_output_timeout: 30m
       - store_test_results:
           path: /tmp/test-reports
-      - store_artifacts:
-          path: /tmp/test-artifacts/<< parameters.test-suite-name >>.json
       - when:
           condition: << parameters.codecov-upload >>
           steps:
@@ -192,63 +185,6 @@ jobs:
     <<: *test
   test-window-post:
     <<: *test
-  test-conformance:
-    description: |
-      Run tests using a corpus of interoperable test vectors for Filecoin 
-      implementations to test their correctness and compliance with the Filecoin
-      specifications.
-    parameters:
-      <<: *test-params
-      vectors-branch:
-        type: string
-        default: ""
-        description: |
-          Branch on github.com/filecoin-project/test-vectors to checkout and
-          test with. If empty (the default) the commit defined by the git
-          submodule is used.
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          command: make deps lotus
-          no_output_timeout: 30m
-      - download-params
-      - when:
-          condition:
-            not:
-              equal: [ "", << parameters.vectors-branch >> ]
-          steps:
-            - run:
-                name: checkout vectors branch
-                command: |
-                  cd extern/test-vectors
-                  git fetch
-                  git checkout origin/<< parameters.vectors-branch >>
-            - run:
-                name: go get vectors branch
-                command: go get github.com/filecoin-project/test-vectors@<< parameters.vectors-branch >>
-      - go/install-gotestsum:
-          gobin: $HOME/.local/bin
-          version: 0.5.2
-      - run:
-          name: go test
-          environment:
-            SKIP_CONFORMANCE: "0"
-          command: |
-            mkdir -p /tmp/test-reports
-            mkdir -p /tmp/test-artifacts
-            gotestsum \
-              --format pkgname-and-test-fails \
-              --junitfile /tmp/test-reports/junit.xml \
-              -- \
-              -v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/
-            go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html
-          no_output_timeout: 30m
-      - store_test_results:
-          path: /tmp/test-reports
-      - store_artifacts:
-          path: /tmp/test-artifacts/conformance-coverage.html
 
   build-macos:
     description: build darwin lotus binary
@@ -310,27 +246,6 @@ jobs:
       - run:
           command: "! go fmt ./... 2>&1 | read"
 
-  cbor-gen-check:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run: make deps
-      - run: go install golang.org/x/tools/cmd/goimports
-      - run: go install github.com/hannahhoward/cbor-gen-for
-      - run: go generate ./...
-      - run: git --no-pager diff
-      - run: git --no-pager diff --quiet
-
-  docs-check:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run: make docsgen
-      - run: git --no-pager diff
-      - run: git --no-pager diff --quiet
-
   lint: &lint
     description: |
       Run golangci-lint.
@@ -367,6 +282,9 @@ jobs:
           command: |
             $HOME/.local/bin/golangci-lint run -v --timeout 2m \
               --concurrency << parameters.concurrency >> << parameters.args >>
+  lint-changes:
+    <<: *lint
+
   lint-all:
     <<: *lint
 
@@ -395,33 +313,21 @@ workflows:
   version: 2.1
   ci:
     jobs:
-      - lint-all
+      - lint-changes:
+          args: "--new-from-rev origin/master"
       - mod-tidy-check
       - gofmt
-      - cbor-gen-check
-      - docs-check
       - test:
           codecov-upload: true
-          test-suite-name: full
       - test-window-post:
           go-test-flags: "-run=TestWindowedPost"
           winpost-test: "1"
-          test-suite-name: window-post
       - test-short:
           go-test-flags: "--timeout 10m --short"
-          test-suite-name: short
           filters:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+$/
-      - test-conformance:
-          test-suite-name: conformance
-          packages: "./conformance"
-      - test-conformance:
-          name: test-conformance-bleeding-edge
-          test-suite-name: conformance-bleeding-edge
-          packages: "./conformance"
-          vectors-branch: master
       - build-debug
       - build-all:
           requires:

.codecov.yml

@@ -1,5 +1,3 @@
 comment: off
 ignore:
   - "cbor_gen.go"
-github_checks:
-  annotations: false

.github/ISSUE_TEMPLATE/sealingfailed.md

@@ -17,27 +17,18 @@ A brief description of the problem you encountered while proving (sealing) a sec
 
 Including what commands you ran, and a description of your setup, is very helpful.
 
+**Sectors list**
+
+The output of `./lotus-miner sectors list`.
+
 **Sectors status**
 
-The output of `lotus-miner sectors status --log <sectorId>` for the failed sector(s).
+The output of `./lotus-miner sectors status --log <sectorId>` for the failed sector(s).
 
 **Lotus miner logs**
 
 Please go through the logs of your miner, and include screenshots of any error-like messages you find.
 
-Alternatively please upload full log files and share a link here
-
-**Lotus miner diagnostic info**
-
-Please collect the following diagnostic information, and share a link here
-
-* lotus-miner diagnostic info `lotus-miner info all > allinfo`
-
-** Code modifications **
-
-If you have modified parts of lotus, please describe which areas were modified,
-and the scope of those modifications
-
 **Version**
 
-The output of `lotus --version`.
+The output of `./lotus --version`.

.github/labels.yml

@@ -1,248 +0,0 @@
-###
-### Special magic GitHub labels
-### https://help.github.com/en/github/building-a-strong-community/encouraging-helpful-contributions-to-your-project-with-labels
-#
-- name: "good first issue"
-  color: 7057ff
-  description: "Good for newcomers"
-- name: "help wanted"
-  color: 008672
-  description: "Extra attention is needed"
-
-###
-### Goals
-#
-- name: goal/incentives
-  color: ff004d
-  description: "Incentinet"
-
-###
-### Areas
-#
-- name: area/ux
-  color: 00A4E0
-  description: "Area: UX"
-- name: area/chain/vm
-  color: 00A4E2
-  description: "Area: Chain/VM"
-- name: area/chain/sync
-  color: 00A4E4
-  description: "Area: Chain/Sync"
-- name: area/chain/misc
-  color: 00A4E6
-  description: "Area: Chain/Misc"
-- name: area/markets
-  color: 00A4E8
-  description: "Area: Markets"  
-- name: area/sealing/fsm
-  color: 0bb1ed
-  description: "Area: Sealing/FSM"
-- name: area/sealing/storage
-  color: 0EB4F0
-  description: "Area: Sealing/Storage"
-- name: area/proving
-  color: 0EB4F0
-  description: "Area: Proving"
-- name: area/mining
-  color: 10B6F2
-  description: "Area: Mining"
-- name: area/client/storage
-  color: 13B9F5
-  description: "Area: Client/Storage"
-- name: area/client/retrieval
-  color: 15BBF7
-  description: "Area: Client/Retrieval"
-- name: area/wallet
-  color: 15BBF7
-  description: "Area: Wallet"
-- name: area/payment-channel
-  color: ff6767
-  description: "Area: Payment Channel"
-- name: area/multisig
-  color: fff0ff
-  description: "Area: Multisig"
-- name: area/networking
-  color: 273f8a
-  description: "Area: Networking"
-
-###
-### Kinds
-#
-- name: kind/bug
-  color: c92712
-  description: "Kind: Bug"
-- name: kind/chore
-  color: fcf0b5
-  description: "Kind: Chore"
-- name: kind/feature
-  color: FFF3B8
-  description: "Kind: Feature"
-- name: kind/improvement
-  color: FFF5BA
-  description: "Kind: Improvement"
-- name: kind/test
-  color: FFF8BD
-  description: "Kind: Test"
-- name: kind/question
-  color: FFFDC2
-  description: "Kind: Question"
-- name: kind/enhancement
-  color: FFFFC5
-  description: "Kind: Enhancement"
-- name: kind/discussion
-  color: FFFFC7
-  description: "Kind: Discussion"
-
-###
-### Difficulties
-#
-- name: dif/trivial
-  color: b2b7ff
-  description: "Can be confidently tackled by newcomers, who are widely unfamiliar with lotus"
-- name: dif/easy
-  color: 7886d7
-  description: "An existing lotus user should be able to pick this up"
-- name: dif/medium
-  color: 6574cd
-  description: "Prior development experience with lotus is likely helpful"
-- name: dif/hard
-  color: 5661b3
-  description: "Suggests that having worked on the specific component affected by this issue is important"
-- name: dif/expert
-  color: 2f365f
-  description: "Requires extensive knowledge of the history, implications, ramifications of the issue"
-
-###
-### Efforts
-#
-- name: effort/minutes
-  color: e8fffe
-  description: "Effort: Minutes"
-- name: effort/hours
-  color: a0f0ed
-  description: "Effort: Hours"
-- name: effort/day
-  color: 64d5ca
-  description: "Effort: One Day"
-- name: effort/days
-  color: 4dc0b5
-  description: "Effort: Multiple Days"
-- name: effort/week
-  color: 38a89d
-  description: "Effort: One Week"
-- name: effort/weeks
-  color: 20504f
-  description: "Effort: Multiple Weeks"
-
-###
-### Impacts
-#
-- name: impact/regression
-  color: f1f5f8
-  description: "Impact: Regression"
-- name: impact/api-breakage
-  color: ECF0F3
-  description: "Impact: API Breakage"
-- name: impact/quality
-  color: E7EBEE
-  description: "Impact: Quality"
-- name: impact/dx
-  color: E2E6E9
-  description: "Impact: Developer Experience"
-- name: impact/test-flakiness
-  color: DDE1E4
-  description: "Impact: Test Flakiness"
-- name: impact/consensus
-  color: b20014
-  description: "Impact: Consensus"
-
-###
-### Topics
-#
-- name: topic/interoperability
-  color: bf0f73
-  description: "Topic: Interoperability"
-- name: topic/specs
-  color: CC1C80
-  description: "Topic: Specs"
-- name: topic/docs
-  color: D9298D
-  description: "Topic: Documentation"
-- name: topic/architecture
-  color: E53599
-  description: "Topic: Architecture"
-
-###
-### Priorities
-###
-- name: P0
-  color: dd362a
-  description: "P0: Critical Blocker"
-- name: P1
-  color: ce8048
-  description: "P1: Must be resolved"
-- name: P2
-  color: dbd81a
-  description: "P2: Should be resolved"
-- name: P3
-  color: 9fea8f
-  description: "P3: Might get resolved"
-
-###
-### Hints
-#
-#- name: hint/good-first-issue
-#  color: 7057ff
-#  description: "Hint: Good First Issue"
-#- name: hint/help-wanted
-#  color: 008672
-#  description: "Hint: Help Wanted"
-- name: hint/needs-decision
-  color: 33B9A5
-  description: "Hint: Needs Decision"
-- name: hint/needs-triage
-  color: 1AA08C
-  description: "Hint: Needs Triage"
-- name: hint/needs-analysis
-  color: 26AC98
-  description: "Hint: Needs Analysis"
-- name: hint/needs-author-input
-  color: 33B9A5
-  description: "Hint: Needs Author Input"
-- name: hint/needs-team-input
-  color: 40C6B2
-  description: "Hint: Needs Team Input"
-- name: hint/needs-community-input
-  color: 4DD3BF
-  description: "Hint: Needs Community Input"
-- name: hint/needs-review
-  color: 5AE0CC
-  description: "Hint: Needs Review"
-
-###
-### Statuses
-#
-- name: status/done
-  color: edb3a6
-  description: "Status: Done"
-- name: status/deferred
-  color: E0A699
-  description: "Status: Deferred"
-- name: status/in-progress
-  color: D49A8D
-  description: "Status: In Progress"
-- name: status/blocked
-  color: C78D80
-  description: "Status: Blocked"
-- name: status/inactive
-  color: BA8073
-  description: "Status: Inactive"
-- name: status/waiting
-  color: AD7366
-  description: "Status: Waiting"
-- name: status/rotten
-  color: 7A4033
-  description: "Status: Rotten"
-- name: status/discarded
-  color: 6D3326
-  description: "Status: Discarded / Won't fix"

.github/workflows/label-syncer.yml

@@ -1,17 +0,0 @@
-
-name: Label syncer
-on:
-  push:
-    paths:
-      - '.github/labels.yml'
-    branches:
-      - master
-jobs:
-  build:
-    name: Sync labels
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@1.0.0
-      - uses: micnncim/action-label-syncer@v1.0.0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitmodules

@@ -5,6 +5,3 @@
 [submodule "extern/serialization-vectors"]
 	path = extern/serialization-vectors
 	url = https://github.com/filecoin-project/serialization-vectors
-[submodule "extern/test-vectors"]
-	path = extern/test-vectors
-	url = https://github.com/filecoin-project/test-vectors.git

.golangci.yml

@@ -23,14 +23,6 @@ issues:
     - "Potential file inclusion via variable"
     - "should have( a package)? comment"
     - "Error return value of `logging.SetLogLevel` is not checked"
-    - "comment on exported"
-    - "(func|method) \\w+ should be \\w+"
-    - "(type|var|struct field|(method|func) parameter) `\\w+` should be `\\w+`"
-    - "(G306|G301|G307|G108|G302|G204|G104)"
-    - "don't use ALL_CAPS in Go names"
-    - "string .* has .* occurrences, make it a constant"
-    - "a blank import should be only in a main or test package, or have a comment justifying it"
-    - "package comment should be of the form"
 
   exclude-use-default: false
   exclude-rules:
@@ -54,19 +46,6 @@ issues:
       linters:
         - gosec
 
-    - path: chain/vectors/gen/.*
-      linters:
-        - gosec
-
-    - path: cmd/lotus-bench/.*
-      linters:
-        - gosec
-
-    - path: api/test/.*
-      text: "context.Context should be the first parameter"
-      linters:
-        - golint
-
 linters-settings:
   goconst:
     min-occurrences: 6

CHANGELOG.md

@@ -1,360 +1,7 @@
-# Lotus changelog
+# lotus changelog
 
-# 0.6.1 / 2020-09-08
+## 0.1.0 / 2019-12-11
 
-This optional release introduces a minor improvement to the sync process, ensuring nodes don't fall behind and then resync.
+We are very excited to release **lotus** 0.1.0. This is our testnet release. To install lotus and join the testnet, please visit [docs.lotu.sh](docs.lotu.sh). Please file bug reports as [issues](https://github.com/filecoin-project/lotus/issues).
 
-## Changes
-
-- Update `test-vectors` (https://github.com/filecoin-project/lotus/pull/3645)
-- Revert "only subscribe to pubsub topics once we are synced" (https://github.com/filecoin-project/lotus/pull/3643)
-
-# 0.6.0 / 2020-09-07
-
-This consensus-breaking release of Lotus is designed to test a network upgrade on the space race testnet. The changes that break consensus are:
-
-- Tweaking of some cryptoecon parameters in specs-actors 0.9.7 (https://github.com/filecoin-project/specs-actors/releases/tag/v0.9.7)
-- Rebalancing FIL distribution to make testnet FIL scarce, which prevents base fee spikes and sets better expectations for mainnet
-
-This release also introduces many improvements to Lotus! Among them are a new version of go-fil-markets that supports non-blocking retrieval, various spam reduction measures in the messagepool and p2p logic, and UX improvements to payment channels, dealmaking, and state inspection.
-
-## Changes
-
-#### Core Lotus and dependencies
-
-- Implement faucet funds reallocation logic (https://github.com/filecoin-project/lotus/pull/3632)
-- Network upgrade: Upgrade to correct fork threshold (https://github.com/filecoin-project/lotus/pull/3628)
-- Update to specs 0.9.7 and markets 0.6.0 (https://github.com/filecoin-project/lotus/pull/3627)
-- Network upgrade: Perform base fee tamping (https://github.com/filecoin-project/lotus/pull/3623)
-- Chain events: if cache best() is nil, return chain head (https://github.com/filecoin-project/lotus/pull/3611)
-- Update to specs actors v0.9.6 (https://github.com/filecoin-project/lotus/pull/3603)
-
-#### Messagepool
-
-- Temporarily allow negative chains (https://github.com/filecoin-project/lotus/pull/3625)
-- Improve publish/republish logic (https://github.com/filecoin-project/lotus/pull/3592)
-- Fix selection bug; priority messages were not included if other chains were negative (https://github.com/filecoin-project/lotus/pull/3580)
-- Add defensive check for minimum GasFeeCap for inclusion within the next 20 blocks (https://github.com/filecoin-project/lotus/pull/3579)
-- Add additional info about gas premium (https://github.com/filecoin-project/lotus/pull/3578)
-- Fix GasPremium capping logic  (https://github.com/filecoin-project/lotus/pull/3552)
-
-#### Payment channels 
-
-- Get available funds by address or by from/to (https://github.com/filecoin-project/lotus/pull/3547)
-- Create `lotus paych status` command (https://github.com/filecoin-project/lotus/pull/3523)
-- Rename CLI command from "paych get" to "paych add-funds" (https://github.com/filecoin-project/lotus/pull/3520)
-
-#### Peer-to-peer
-
-- Only subscribe to pubsub topics once we are synced (https://github.com/filecoin-project/lotus/pull/3602)
-- Reduce mpool add failure log spam (https://github.com/filecoin-project/lotus/pull/3562)
-- Republish messages even if the chains have negative performance(https://github.com/filecoin-project/lotus/pull/3557)
-- Adjust gossipsub gossip factor (https://github.com/filecoin-project/lotus/pull/3556)
-- Integrate pubsub Random Early Drop (https://github.com/filecoin-project/lotus/pull/3518)
-
-#### Miscellaneous
-
-- Fix panic in OnDealExpiredSlashed (https://github.com/filecoin-project/lotus/pull/3553)
-- Robustify state manager against holes in actor method numbers (https://github.com/filecoin-project/lotus/pull/3538)
-
-#### UX
-
-- VM: Fix an error message (https://github.com/filecoin-project/lotus/pull/3608)
-- Documentation: Batch replacement,update lotus-storage-miner to lotus-miner (https://github.com/filecoin-project/lotus/pull/3571)
-- CLI: Robust actor lookup (https://github.com/filecoin-project/lotus/pull/3535)
-- Add agent flag to net peers (https://github.com/filecoin-project/lotus/pull/3534)
-- Add watch option to storage-deals list (https://github.com/filecoin-project/lotus/pull/3527)
-
-#### Testing & tooling
-
-- Decommission chain-validation (https://github.com/filecoin-project/lotus/pull/3606)
-- Metrics: add expected height metric (https://github.com/filecoin-project/lotus/pull/3586)
-- PCR: Use current tipset during refund (https://github.com/filecoin-project/lotus/pull/3570)
-- Lotus-shed: Add math command (https://github.com/filecoin-project/lotus/pull/3568)
-- PCR: Add tipset aggergation (https://github.com/filecoin-project/lotus/pull/3565)- Fix broken paych tests (https://github.com/filecoin-project/lotus/pull/3551)
-- Make chain export ~1000x times faster (https://github.com/filecoin-project/lotus/pull/3533)
-- Chainwatch: Stop SyncIncomingBlocks from leaking into chainwatch processing; No panics during processing (https://github.com/filecoin-project/lotus/pull/3526)
-- Conformance: various changes (https://github.com/filecoin-project/lotus/pull/3521)
-
-# 0.5.10 / 2020-09-03
-
-This patch includes a crucial fix to the message pool selection logic, strongly disfavouring messages that might cause a miner penalty.
-
-## Changes
-
-- Fix calculation of GasReward in messagepool (https://github.com/filecoin-project/lotus/pull/3528)
-
-# 0.5.9 / 2020-09-03
-
-This patch includes a hotfix to the `GasEstimateFeeCap` method, capping the estimated fee to a reasonable level by default.
-
-## Changes 
-
-- Added target height to sync wait (https://github.com/filecoin-project/lotus/pull/3502)
-- Disable codecov annotations (https://github.com/filecoin-project/lotus/pull/3514)
-- Cap fees to reasonable level by default (https://github.com/filecoin-project/lotus/pull/3516)
-- Add APIs and command to inspect bandwidth usage (https://github.com/filecoin-project/lotus/pull/3497)
-- Track expected nonce in mpool, ignore messages with large nonce gaps (https://github.com/filecoin-project/lotus/pull/3450)
-
-# 0.5.8 / 2020-09-02
-
-This patch includes some bugfixes to the sector sealing process, and updates go-fil-markets. It also improves the performance of blocksync, adds a method to export chain state trees, and improves chainwatch.
-
-## Changes
-
-- Upgrade markets to v0.5.9 (https://github.com/filecoin-project/lotus/pull/3496)
-- Improve blocksync to load fewer messages: (https://github.com/filecoin-project/lotus/pull/3494)
-- Fix a panic in the ffi-wrapper's `ReadPiece` (https://github.com/filecoin-project/lotus/pull/3492/files)
-- Fix a deadlock in the sealing scheduler (https://github.com/filecoin-project/lotus/pull/3489)
-- Add test vectors for tipset tests (https://github.com/filecoin-project/lotus/pull/3485/files)
-- Improve the advance-block debug command (https://github.com/filecoin-project/lotus/pull/3476)
-- Add toggle for message processing to Lotus PCR (https://github.com/filecoin-project/lotus/pull/3470)
-- Allow exporting recent chain state trees (https://github.com/filecoin-project/lotus/pull/3463)
-- Remove height from chain rand (https://github.com/filecoin-project/lotus/pull/3458)
-- Disable GC on chain badger datastore (https://github.com/filecoin-project/lotus/pull/3457)
-- Account for `GasPremium` in `GasEstimateFeeCap` (https://github.com/filecoin-project/lotus/pull/3456)
-- Update go-libp2p-pubsub to `master` (https://github.com/filecoin-project/lotus/pull/3455)
-- Chainwatch improvements (https://github.com/filecoin-project/lotus/pull/3442)
-
-# 0.5.7 / 2020-08-31
-
-This patch release includes some bugfixes and enhancements to the sector lifecycle and message pool logic. 
-
-## Changes
-
-- Rebuild unsealed infos on miner restart (https://github.com/filecoin-project/lotus/pull/3401)
-- CLI to attach storage paths to workers (https://github.com/filecoin-project/lotus/pull/3405)
-- Do not select negative performing message chains for inclusion (https://github.com/filecoin-project/lotus/pull/3392)
-- Remove a redundant error-check (https://github.com/filecoin-project/lotus/pull/3421)
-- Correctly move unsealed sectors in `FinalizeSectors` (https://github.com/filecoin-project/lotus/pull/3424)
-- Improve worker selection logic (https://github.com/filecoin-project/lotus/pull/3425)
-- Don't use context to close bitswap (https://github.com/filecoin-project/lotus/pull/3430)
-- Correctly estimate gas premium when there is only one message on chain (https://github.com/filecoin-project/lotus/pull/3428)
-
-# 0.5.6 / 2020-08-29
-
-Hotfix release that fixes a panic in the sealing scheduler (https://github.com/filecoin-project/lotus/pull/3389).
-
-# 0.5.5
-
-This patch release introduces a large number of improvements to the sealing process.
-It also updates go-fil-markets to 
-[version 0.5.8](https://github.com/filecoin-project/go-fil-markets/releases/tag/v0.5.8),
-and go-libp2p-pubsub to [v0.3.5](https://github.com/libp2p/go-libp2p-pubsub/releases/tag/v0.3.5).
-
-#### Downstream upgrades
-
-- Upgrades markets to v0.5.8 (https://github.com/filecoin-project/lotus/pull/3384)
-- Upgrades go-libp2p-pubsub to v0.3.5 (https://github.com/filecoin-project/lotus/pull/3305)
-
-#### Sector sealing
-
-- The following improvements were introduced in https://github.com/filecoin-project/lotus/pull/3350.
-
-    - Allow `lotus-miner sectors remove` to remove a sector in any state.
-    - Create a separate state in the storage FSM dedicated to submitting the Commit message.
-    - Recovery for when the Deal IDs of deals in a sector get changed in a reorg.
-    - Auto-retry sending Precommit and Commit messages if they run out of gas
-    - Auto-retry sector remove tasks when they fail
-    - Compact worker windows, and allow their tasks to be executed in any order
-
-- Don't simply skip PoSt for bad sectors (https://github.com/filecoin-project/lotus/pull/3323)
-
-#### Message Pool 
-
-- Spam Protection: Track required funds for pending messages (https://github.com/filecoin-project/lotus/pull/3313)
-
-#### Chainwatch
-
-- Add more power and reward metrics (https://github.com/filecoin-project/lotus/pull/3367)
-- Fix raciness in sector deal table (https://github.com/filecoin-project/lotus/pull/3275)
-- Parallelize miner processing (https://github.com/filecoin-project/lotus/pull/3380)
-- Accept Lotus API and token (https://github.com/filecoin-project/lotus/pull/3337)
-
-# 0.5.4
-
-A patch release, containing a few nice bugfixes and improvements:
-
-- Fix parsing of peer ID in `lotus-miner actor set-peer-id` (@whyrusleeping)
-- Update dependencies, fixing several bugs (@Stebalien)
-- Fix remaining linter warnings (@Stebalien)
-- Use safe string truncation (@Ingar)
-- Allow tweaking of blocksync message window size (@whyrusleeping)
-- Add some additional gas stats to metrics (@Kubuxu)
-- Fix an edge case bug in message selection, add many tests (@vyzo)
-
-# 0.5.3
-
-Yet another hotfix release. 
-A lesson for readers, having people who have been awake for 12+ hours review
-your hotfix PR is not a good idea. Find someone who has enough slept recently
-enough to give you good code review, otherwise you'll end up quickly bumping
-versions again.
-
-- Fixed a bug in the mempool that was introduced in v0.5.2
-
-# 0.5.2 / 2020-08-24
-
-This is a hotfix release.
-
-- Fix message selection to not include messages that are invalid for block
-  inclusion.
-- Improve SelectMessage handling of the case where the message pools tipset
-  differs from our mining base.
-
-# 0.5.1 / 2020-08-24
-
-The Space Race release! 
-This release contains the genesis car file and bootstrap peers for the space
-race network. 
-
-Additionally, we included two small fixes to genesis creation:
-- Randomize ticket value in genesis generation
-- Correctly set t099 (burnt funds actor) to have valid account actor state
-
-# 0.5.0 / 2020-08-20
-
-This version of Lotus will be used for the incentivized testnet Space Race competition,
-and can be considered mainnet-ready code. It includes some protocol
-changes, upgrades of core dependencies, and various bugfixes and UX/performance improvements.
-
-## Highlights
-
-Among the highlights included in this release are:
-
-- Gas changes: We implemented EIP-1559 and introduced real gas values.
-- Deal-making: We now support "Committed Capacity" sectors, "fast-retrieval" deals,
-and the packing of multiple deals into a single sector.
-- Renamed features: We renamed some of the binaries, environment variables, and default
-paths associated with a Lotus node.
-
-### Gas changes
-
-We made some significant changes to the mechanics of gas in this release.
-
-#### Network fee
-
-We implemented something similar to 
-[Ethereum's EIP-1559](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-1559.md).
-The `Message` structure had three changes:
-- The `GasPrice` field has been removed
-- A new `GasFeeCap` field has been added, which controls the maximum cost
-the sender incurs for the message
-- A new `GasPremium` field has been added, which controls the reward a miner
-earns for including the message
-
-A sender will never be charged more than `GasFeeCap * GasLimit`. 
-A miner will typically earn `GasPremium * GasLimit` as a reward.
-
-The `Blockheader` structure has one new field, called `ParentBaseFee`. 
-Informally speaking,the `ParentBaseFee`
-is increased when blocks are densely packed with messages, and decreased otherwise.
-
-The `ParentBaseFee` is used when calculating how much a sender burns when executing a message. _Burning_ simply refers to sending attoFIL to a dedicated, unreachable account.
-A message causes `ParentBaseFee * GasUsed` attoFIL to be burnt.
-
-#### Real gas values
-
-This release also includes our first "real" gas costs for primitive operations.
-The costs were designed to account for both the _time_ that message execution takes,
-as well as the _space_ a message adds to the state tree.
-
-## Deal-making changes
-
-There are three key changes to the deal-making process.
-
-#### Committed Capacity sectors
-
-Miners can now pledge "Committed Capacity" (CC) sectors, which are explicitly
-stated as containing junk data, and must not include any deals. Miners can do this
-to increase their storage power, and win block rewards from this pledged storage.
-
-They can mark these sectors as "upgradable" with `lotus-miner sectors mark-for-upgrade`.
-If the miner receives and accepts one or more storage deals, the sector that includes
-those deals will _replace_ the CC sector. This is intended to maximize the amount of useful
-storage on the Filecoin network.
-
-#### Fast-retrieval deals
-
-Clients can now include a `fast-retrieval` flag when proposing deals with storage miners.
-If set to true, the miner will include an extra copy of the deal data. This
-data can be quickly served in a retrieval deal, since it will not need to be unsealed.
-
-#### Multiple deals per sector
-
-Miners can now pack multiple deals into a single sector, so long as all the deals
-fit into the sector capacity. This should increase the packing efficiency of miners.
-
-### Renamed features
-
-To improve the user experience, we updated several names to mainatin
-standard prefixing, and to better reflect the meaning of the features being referenced.
-
-In particular, the Lotus miner binary is now called `lotus-miner`, the default
-path for miner data is now `~/.lotusminer`, and the environment variable
-that sets the path for miner data is now `$LOTUS_MINER_PATH`. A full list of renamed
-features can be found [here](https://github.com/filecoin-project/lotus/issues/2304).
-
-## Changelog
-
-#### Downstream upgrades
-- Upgrades markets to v0.5.6 (https://github.com/filecoin-project/lotus/pull/3058)
-- Upgrades specs-actors to v0.9.3 (https://github.com/filecoin-project/lotus/pull/3151)
-
-#### Core protocol
-- Introduces gas values, replacing placeholders (https://github.com/filecoin-project/lotus/pull/2343)
-- Implements EIP-1559, introducing a network base fee, message gas fee cap, and message gas fee premium (https://github.com/filecoin-project/lotus/pull/2874)
-- Implements Poisson Sortition for elections (https://github.com/filecoin-project/lotus/pull/2084)
-
-#### Deal-making lifecycle
-- Introduces "Committed Capacity" sectors (https://github.com/filecoin-project/lotus/pull/2220)
-- Introduces "fast-retrieval" flag for deals (https://github.com/filecoin-project/lotus/pull/2323
-- Supports packing multiple deals into one sector (https://github.com/filecoin-project/storage-fsm/pull/38)
-
-#### Enhancements
-
-- Optimized message pool selection logic (https://github.com/filecoin-project/lotus/pull/2838)
-- Window-based scheduling of sealing tasks (https://github.com/filecoin-project/sector-storage/pull/67)
-- Faster window PoSt (https://github.com/filecoin-project/lotus/pull/2209/files)
-- Refactors the payment channel manager (https://github.com/filecoin-project/lotus/pull/2640)
-- Refactors blocksync (https://github.com/filecoin-project/lotus/pull/2715/files)
-
-#### UX
-
-- Provide status updates for data-transfer (https://github.com/filecoin-project/lotus/pull/3162, https://github.com/filecoin-project/lotus/pull/3191)
-- Miners can customise asks (https://github.com/filecoin-project/lotus/pull/2046)
-- Miners can toggle auto-acceptance of deals (https://github.com/filecoin-project/lotus/pull/1994)
-- Miners can maintain a blocklist of piece CIDs (https://github.com/filecoin-project/lotus/pull/2069)
-
-## Contributors
-
-The following contributors had 10 or more commits go into this release.
-We are grateful for every contribution!
-
-| Contributor        | Commits | Lines ±       |
-|--------------------|---------|---------------|
-| magik6k            | 361     | +13197/-6136  |
-| Kubuxu             | 227     | +5670/-2587   |
-| arajasek           | 120     | +2916/-1264   |
-| whyrusleeping      | 112     | +3979/-1089   |
-| vyzo               | 99      | +3343/-1305   |
-| dirkmc             | 68      | +8732/-3621   |
-| laser              | 45      | +1489/-501    |
-| hannahhoward       | 43      | +2654/-990    |
-| frrist             | 37      | +6630/-4338   |
-| schomatis          | 28      | +3016/-1368   |
-| placer14           | 27      | +824/-350     |
-| raulk              | 25      | +28718/-29849 |
-| mrsmkl             | 22      | +560/-368     |
-| travisperson       | 18      | +1354/-314    |
-| nonsense           | 16      | +2956/-2842   |
-| ingar              | 13      | +331/-123     |
-| daviddias          | 11      | +311/-11      |
-| Stebalien          | 11      | +1204/-980    |
-| RobQuistNL         | 10      | +69/-74       |
-
-# 0.1.0 / 2019-12-11
-
-We are very excited to release **lotus** 0.1.0. This is our testnet release. To install lotus and join the testnet, please visit [lotu.sh](lotu.sh). Please file bug reports as [issues](https://github.com/filecoin-project/lotus/issues).
-
-A huge thank you to all contributors for this testnet release!
+A huge thank you to all contributors for this testnet release!

Makefile

@@ -98,17 +98,26 @@ an existing lotus binary in your PATH. This may cause problems if you don't run
 
 .PHONY: build
 
-install: install-daemon install-miner install-worker
-
-install-daemon:
+install:
 	install -C ./lotus /usr/local/bin/lotus
-
-install-miner:
 	install -C ./lotus-miner /usr/local/bin/lotus-miner
-
-install-worker:
 	install -C ./lotus-worker /usr/local/bin/lotus-worker
 
+install-services: install
+	mkdir -p /usr/local/lib/systemd/system
+	mkdir -p /var/log/lotus
+	install -C -m 0644 ./scripts/lotus-daemon.service /usr/local/lib/systemd/system/lotus-daemon.service
+	install -C -m 0644 ./scripts/lotus-miner.service /usr/local/lib/systemd/system/lotus-miner.service
+	systemctl daemon-reload
+	@echo
+	@echo "lotus-daemon and lotus-miner services installed. Don't forget to 'systemctl enable lotus-daemon|lotus-miner' for it to be enabled on startup."
+
+clean-services:
+	rm -f /usr/local/lib/systemd/system/lotus-daemon.service
+	rm -f /usr/local/lib/systemd/system/lotus-miner.service
+	rm -f /usr/local/lib/systemd/system/lotus-chainwatch.service
+	systemctl daemon-reload
+
 # TOOLS
 
 lotus-seed: $(BUILD_DEPS)
@@ -148,10 +157,18 @@ BINS+=lotus-fountain
 
 lotus-chainwatch:
 	rm -f lotus-chainwatch
-	go build $(GOFLAGS) -o lotus-chainwatch ./cmd/lotus-chainwatch
+	go build -o lotus-chainwatch ./cmd/lotus-chainwatch
 .PHONY: lotus-chainwatch
 BINS+=lotus-chainwatch
 
+install-chainwatch-service: chainwatch
+	mkdir -p /etc/lotus
+	install -C ./lotus-chainwatch /usr/local/bin/lotus-chainwatch
+	install -C -m 0644 ./scripts/lotus-chainwatch.service /usr/local/lib/systemd/system/lotus-chainwatch.service
+	systemctl daemon-reload
+	@echo
+	@echo "chainwatch installed. Don't forget to 'systemctl enable chainwatch' for it to be enabled on startup."
+
 lotus-bench:
 	rm -f lotus-bench
 	go build -o lotus-bench ./cmd/lotus-bench
@@ -166,13 +183,6 @@ lotus-stats:
 .PHONY: lotus-stats
 BINS+=lotus-stats
 
-lotus-pcr:
-	rm -f lotus-pcr
-	go build $(GOFLAGS) -o lotus-pcr ./cmd/lotus-pcr
-	go run github.com/GeertJohan/go.rice/rice append --exec lotus-pcr -i ./build
-.PHONY: lotus-pcr
-BINS+=lotus-pcr
-
 lotus-health:
 	rm -f lotus-health
 	go build -o lotus-health ./cmd/lotus-health
@@ -185,65 +195,6 @@ testground:
 .PHONY: testground
 BINS+=testground
 
-install-chainwatch: lotus-chainwatch
-	install -C ./lotus-chainwatch /usr/local/bin/lotus-chainwatch
-
-# SYSTEMD
-
-install-daemon-service: install-daemon
-	mkdir -p /etc/systemd/system
-	mkdir -p /var/log/lotus
-	install -C -m 0644 ./scripts/lotus-daemon.service /etc/systemd/system/lotus-daemon.service
-	systemctl daemon-reload
-	@echo
-	@echo "lotus-daemon service installed. Don't forget to run 'sudo systemctl start lotus-daemon' to start it and 'sudo systemctl enable lotus-daemon' for it to be enabled on startup."
-
-install-miner-service: install-miner install-daemon-service
-	mkdir -p /etc/systemd/system
-	mkdir -p /var/log/lotus
-	install -C -m 0644 ./scripts/lotus-miner.service /etc/systemd/system/lotus-miner.service
-	systemctl daemon-reload
-	@echo
-	@echo "lotus-miner service installed. Don't forget to run 'sudo systemctl start lotus-miner' to start it and 'sudo systemctl enable lotus-miner' for it to be enabled on startup."
-
-install-chainwatch-service: install-chainwatch install-daemon-service
-	mkdir -p /etc/systemd/system
-	mkdir -p /var/log/lotus
-	install -C -m 0644 ./scripts/lotus-chainwatch.service /etc/systemd/system/lotus-chainwatch.service
-	systemctl daemon-reload
-	@echo
-	@echo "chainwatch service installed. Don't forget to run 'sudo systemctl start lotus-chainwatch' to start it and 'sudo systemctl enable lotus-chainwatch' for it to be enabled on startup."
-
-install-main-services: install-miner-service
-
-install-all-services: install-main-services install-chainwatch-service
-
-install-services: install-main-services
-
-clean-daemon-service: clean-miner-service clean-chainwatch-service
-	-systemctl stop lotus-daemon
-	-systemctl disable lotus-daemon
-	rm -f /etc/systemd/system/lotus-daemon.service
-	systemctl daemon-reload
-
-clean-miner-service:
-	-systemctl stop lotus-miner
-	-systemctl disable lotus-miner
-	rm -f /etc/systemd/system/lotus-miner.service
-	systemctl daemon-reload
-
-clean-chainwatch-service:
-	-systemctl stop lotus-chainwatch
-	-systemctl disable lotus-chainwatch
-	rm -f /etc/systemd/system/lotus-chainwatch.service
-	systemctl daemon-reload
-
-clean-main-services: clean-daemon-service
-
-clean-all-services: clean-main-services
-
-clean-services: clean-all-services
-
 # MISC
 
 buildall: $(BINS)
@@ -272,15 +223,11 @@ dist-clean:
 
 type-gen:
 	go run ./gen/main.go
-	go generate ./...
 
 method-gen:
 	(cd ./lotuspond/front/src/chain && go run ./methodgen.go)
 
 gen: type-gen method-gen
 
-docsgen:
-	go run ./api/docgen > documentation/en/api-methods.md
-
 print-%:
 	@echo $*=$($*)

README.md

@@ -1,24 +1,12 @@
-<p align="center">
-  <a href="https://lotu.sh/" title="Lotus Docs">
-    <img src="documentation/images/lotus_logo_h.png" alt="Project Lotus Logo" width="244" />
-  </a>
-</p>
+![Lotus](documentation/images/lotus_logo_h.png)
 
-<h1 align="center">Project Lotus - 莲</h1>
+# Project Lotus - 莲
 
-<p align="center">
-  <a href="https://circleci.com/gh/filecoin-project/lotus"><img src="https://circleci.com/gh/filecoin-project/lotus.svg?style=svg"></a>
-  <a href="https://codecov.io/gh/filecoin-project/lotus"><img src="https://codecov.io/gh/filecoin-project/lotus/branch/master/graph/badge.svg"></a>
-  <a href="https://goreportcard.com/report/github.com/filecoin-project/lotus"><img src="https://goreportcard.com/badge/github.com/filecoin-project/lotus" /></a>  
-  <a href=""><img src="https://img.shields.io/badge/golang-%3E%3D1.14.7-blue.svg" /></a>
-  <br>
-</p>
-
-Lotus is an implementation of the Filecoin Distributed Storage Network. For more details about Filecoin, check out the [Filecoin Spec](https://spec.filecoin.io).
+Lotus is an implementation of the Filecoin Distributed Storage Network. For more details about Filecoin, check out the [Filecoin Spec](https://github.com/filecoin-project/specs).
 
 ## Building & Documentation
 
-For instructions on how to build lotus from source, please visit [https://lotu.sh](https://lotu.sh) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation).
+For instructions on how to build lotus from source, please visit [https://docs.lotu.sh](https://docs.lotu.sh) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation).
 
 ## Reporting a Vulnerability
 
@@ -26,22 +14,12 @@ Please send an email to security@filecoin.org. See our [security policy](SECURIT
 
 ## Development
 
+All work is tracked via issues. An attempt at keeping an up-to-date view on remaining work is in the [lotus testnet github project board](https://github.com/filecoin-project/lotus/projects/1).
+
 The main branches under development at the moment are:
 * [`master`](https://github.com/filecoin-project/lotus): current testnet.
 * [`next`](https://github.com/filecoin-project/lotus/tree/next): working branch with chain-breaking changes.
-* [`ntwk-calibration`](https://github.com/filecoin-project/lotus/tree/ntwk-calibration): devnet running one of `next` commits.
-
-### Tracker
-
-All work is tracked via issues. An attempt at keeping an up-to-date view on remaining work towards Mainnet launch can be seen at the [lotus github project board](https://github.com/orgs/filecoin-project/projects/8). The issues labeled with `incentives` are there to identify the issues needed for Space Race launch.
-
-### Packages
-
-The lotus Filecoin implementation unfolds into the following packages:
-
-- [This repo](https://github.com/filecoin-project/lotus)
-- [go-fil-markets](https://github.com/filecoin-project/go-fil-markets) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/markets-shared-components-5daa144a7046a60001c6e253/board)
-- [spec-actors](https://github.com/filecoin-project/specs-actors) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/actors-5ee6f3aa87591f0016c05685/board)
+* [`interopnet`](https://github.com/filecoin-project/lotus/tree/interopnet): devnet running one of `next` commits.
 
 ## License
 

api/api_common.go

@@ -4,11 +4,10 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/filecoin-project/go-jsonrpc/auth"
-	metrics "github.com/libp2p/go-libp2p-core/metrics"
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
-	protocol "github.com/libp2p/go-libp2p-core/protocol"
+
+	"github.com/filecoin-project/go-jsonrpc/auth"
 
 	"github.com/filecoin-project/lotus/build"
 )
@@ -29,20 +28,6 @@ type Common interface {
 	NetDisconnect(context.Context, peer.ID) error
 	NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error)
 	NetPubsubScores(context.Context) ([]PubsubScore, error)
-	NetAutoNatStatus(context.Context) (NatInfo, error)
-	NetAgentVersion(ctx context.Context, p peer.ID) (string, error)
-
-	// NetBandwidthStats returns statistics about the nodes total bandwidth
-	// usage and current rate across all peers and protocols.
-	NetBandwidthStats(ctx context.Context) (metrics.Stats, error)
-
-	// NetBandwidthStatsByPeer returns statistics about the nodes bandwidth
-	// usage and current rate per peer
-	NetBandwidthStatsByPeer(ctx context.Context) (map[string]metrics.Stats, error)
-
-	// NetBandwidthStatsByProtocol returns statistics about the nodes bandwidth
-	// usage and current rate per protocol
-	NetBandwidthStatsByProtocol(ctx context.Context) (map[protocol.ID]metrics.Stats, error)
 
 	// MethodGroup: Common
 
@@ -80,8 +65,3 @@ type Version struct {
 func (v Version) String() string {
 	return fmt.Sprintf("%s+api%s", v.Version, v.APIVersion.String())
 }
-
-type NatInfo struct {
-	Reachability network.Reachability
-	PublicAddr   string
-}

api/api_full.go

@@ -4,27 +4,21 @@ import (
 	"context"
 	"time"
 
-	"github.com/filecoin-project/specs-actors/actors/runtime/proof"
-
 	"github.com/ipfs/go-cid"
 	"github.com/libp2p/go-libp2p-core/peer"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-bitfield"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
-	"github.com/filecoin-project/go-multistore"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin/market"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-actors/actors/builtin/paych"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
 
 	"github.com/filecoin-project/lotus/chain/types"
-	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 )
 
@@ -43,11 +37,8 @@ type FullNode interface {
 	// ChainHead returns the current head of the chain.
 	ChainHead(context.Context) (*types.TipSet, error)
 
-	// ChainGetRandomnessFromTickets is used to sample the chain for randomness.
-	ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error)
-
-	// ChainGetRandomnessFromBeacon is used to sample the beacon for randomness.
-	ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error)
+	// ChainGetRandomness is used to sample the chain for randomness.
+	ChainGetRandomness(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error)
 
 	// ChainGetBlock returns the block specified by the given CID.
 	ChainGetBlock(context.Context, cid.Cid) (*types.BlockHeader, error)
@@ -66,7 +57,7 @@ type FullNode interface {
 	ChainGetParentMessages(ctx context.Context, blockCid cid.Cid) ([]Message, error)
 
 	// ChainGetTipSetByHeight looks back for a tipset at the specified epoch.
-	// If there are no blocks at the specified epoch, a tipset at an earlier epoch
+	// If there are no blocks at the specified epoch, a tipset at higher epoch
 	// will be returned.
 	ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)
 
@@ -76,11 +67,7 @@ type FullNode interface {
 
 	// ChainHasObj checks if a given CID exists in the chain blockstore.
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
-
-	// ChainStatObj returns statistics about the graph referenced by 'obj'.
-	// If 'base' is also specified, then the returned stat will be a diff
-	// between the two objects.
-	ChainStatObj(ctx context.Context, obj cid.Cid, base cid.Cid) (ObjStat, error)
+	ChainStatObj(context.Context, cid.Cid, cid.Cid) (ObjStat, error)
 
 	// ChainSetHead forcefully sets current chain head. Use with caution.
 	ChainSetHead(context.Context, types.TipSetKey) error
@@ -112,34 +99,7 @@ type FullNode interface {
 	ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*HeadChange, error)
 
 	// ChainExport returns a stream of bytes with CAR dump of chain data.
-	// The exported chain data includes the header chain from the given tipset
-	// back to genesis, the entire genesis state, and the most recent 'nroots'
-	// state trees.
-	ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk types.TipSetKey) (<-chan []byte, error)
-
-	// MethodGroup: Beacon
-	// The Beacon method group contains methods for interacting with the random beacon (DRAND)
-
-	// BeaconGetEntry returns the beacon entry for the given filecoin epoch. If
-	// the entry has not yet been produced, the call will block until the entry
-	// becomes available
-	BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error)
-
-	// GasEstimateFeeCap estimates gas fee cap
-	GasEstimateFeeCap(context.Context, *types.Message, int64, types.TipSetKey) (types.BigInt, error)
-
-	// GasEstimateGasLimit estimates gas used by the message and returns it.
-	// It fails if message fails to execute.
-	GasEstimateGasLimit(context.Context, *types.Message, types.TipSetKey) (int64, error)
-
-	// GasEstimateGasPremium estimates what gas price should be used for a
-	// message to have high likelihood of inclusion in `nblocksincl` epochs.
-
-	GasEstimateGasPremium(_ context.Context, nblocksincl uint64,
-		sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error)
-
-	// GasEstimateMessageGas estimates gas values for unset message gas fields
-	GasEstimateMessageGas(context.Context, *types.Message, *MessageSendSpec, types.TipSetKey) (*types.Message, error)
+	ChainExport(context.Context, types.TipSetKey) (<-chan []byte, error)
 
 	// MethodGroup: Sync
 	// The Sync method group contains methods for interacting with and
@@ -171,32 +131,21 @@ type FullNode interface {
 	// MpoolPending returns pending mempool messages.
 	MpoolPending(context.Context, types.TipSetKey) ([]*types.SignedMessage, error)
 
-	// MpoolSelect returns a list of pending messages for inclusion in the next block
-	MpoolSelect(context.Context, types.TipSetKey, float64) ([]*types.SignedMessage, error)
-
 	// MpoolPush pushes a signed message to mempool.
 	MpoolPush(context.Context, *types.SignedMessage) (cid.Cid, error)
 
 	// MpoolPushMessage atomically assigns a nonce, signs, and pushes a message
 	// to mempool.
-	// maxFee is only used when GasFeeCap/GasPremium fields aren't specified
-	//
-	// When maxFee is set to 0, MpoolPushMessage will guess appropriate fee
-	// based on current chain conditions
-	MpoolPushMessage(ctx context.Context, msg *types.Message, spec *MessageSendSpec) (*types.SignedMessage, error)
+	MpoolPushMessage(context.Context, *types.Message) (*types.SignedMessage, error)
 
 	// MpoolGetNonce gets next nonce for the specified sender.
 	// Note that this method may not be atomic. Use MpoolPushMessage instead.
 	MpoolGetNonce(context.Context, address.Address) (uint64, error)
 	MpoolSub(context.Context) (<-chan MpoolUpdate, error)
 
-	// MpoolClear clears pending messages from the mpool
-	MpoolClear(context.Context, bool) error
-
-	// MpoolGetConfig returns (a copy of) the current mpool config
-	MpoolGetConfig(context.Context) (*types.MpoolConfig, error)
-	// MpoolSetConfig sets the mpool config to (a copy of) the supplied config
-	MpoolSetConfig(context.Context, *types.MpoolConfig) error
+	// MpoolEstimateGasPrice estimates what gas price should be used for a
+	// message to have high likelihood of inclusion in `nblocksincl` epochs.
+	MpoolEstimateGasPrice(ctx context.Context, nblocksincl uint64, sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error)
 
 	// MethodGroup: Miner
 
@@ -242,15 +191,13 @@ type FullNode interface {
 	// ClientImport imports file under the specified path into filestore.
 	ClientImport(ctx context.Context, ref FileRef) (*ImportRes, error)
 	// ClientRemoveImport removes file import
-	ClientRemoveImport(ctx context.Context, importID multistore.StoreID) error
+	ClientRemoveImport(ctx context.Context, importID int) error
 	// ClientStartDeal proposes a deal with a miner.
 	ClientStartDeal(ctx context.Context, params *StartDealParams) (*cid.Cid, error)
 	// ClientGetDealInfo returns the latest information about a given deal.
 	ClientGetDealInfo(context.Context, cid.Cid) (*DealInfo, error)
 	// ClientListDeals returns information about the deals made by the local client.
 	ClientListDeals(ctx context.Context) ([]DealInfo, error)
-	// ClientGetDealUpdates returns the status of updated deals
-	ClientGetDealUpdates(ctx context.Context) (<-chan DealInfo, error)
 	// ClientHasLocal indicates whether a certain CID is locally stored.
 	ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error)
 	// ClientFindData identifies peers that have a certain file, and returns QueryOffers (one per peer).
@@ -259,23 +206,12 @@ type FullNode interface {
 	ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (QueryOffer, error)
 	// ClientRetrieve initiates the retrieval of a file, as specified in the order.
 	ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *FileRef) error
-	// ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel
-	// of status updates.
-	ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *FileRef) (<-chan marketevents.RetrievalEvent, error)
 	// ClientQueryAsk returns a signed StorageAsk from the specified miner.
 	ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error)
-	// ClientCalcCommP calculates the CommP for a specified file
-	ClientCalcCommP(ctx context.Context, inpath string) (*CommPRet, error)
+	// ClientCalcCommP calculates the CommP for a specified file, based on the sector size of the provided miner.
+	ClientCalcCommP(ctx context.Context, inpath string, miner address.Address) (*CommPRet, error)
 	// ClientGenCar generates a CAR file for the specified file.
 	ClientGenCar(ctx context.Context, ref FileRef, outpath string) error
-	// ClientDealSize calculates real deal data size
-	ClientDealSize(ctx context.Context, root cid.Cid) (DataSize, error)
-	// ClientListTransfers returns the status of all ongoing transfers of data
-	ClientListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)
-	ClientDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)
-	// ClientRetrieveTryRestartInsufficientFunds attempts to restart stalled retrievals on a given payment channel
-	// which are stuck due to insufficient funds
-	ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error
 
 	// ClientUnimport removes references to the specified file from filestore
 	//ClientUnimport(path string)
@@ -306,7 +242,7 @@ type FullNode interface {
 	// StateMinerSectors returns info about the given miner's sectors. If the filter bitfield is nil, all sectors are included.
 	// If the filterOut boolean is set to true, any sectors in the filter are excluded.
 	// If false, only those sectors in the filter are included.
-	StateMinerSectors(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*ChainSectorInfo, error)
+	StateMinerSectors(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*ChainSectorInfo, error)
 	// StateMinerActiveSectors returns info about sectors that a given miner is actively proving.
 	StateMinerActiveSectors(context.Context, address.Address, types.TipSetKey) ([]*ChainSectorInfo, error)
 	// StateMinerProvingDeadline calculates the deadline at some epoch for a proving period
@@ -321,20 +257,18 @@ type FullNode interface {
 	// StateMinerPartitions loads miner partitions for the specified miner/deadline
 	StateMinerPartitions(context.Context, address.Address, uint64, types.TipSetKey) ([]*miner.Partition, error)
 	// StateMinerFaults returns a bitfield indicating the faulty sectors of the given miner
-	StateMinerFaults(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error)
+	StateMinerFaults(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)
 	// StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset
 	StateAllMinerFaults(ctx context.Context, lookback abi.ChainEpoch, ts types.TipSetKey) ([]*Fault, error)
 	// StateMinerRecoveries returns a bitfield indicating the recovering sectors of the given miner
-	StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error)
-	// StateMinerInitialPledgeCollateral returns the precommit deposit for the specified miner's sector
-	StateMinerPreCommitDepositForPower(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error)
+	StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)
 	// StateMinerInitialPledgeCollateral returns the initial pledge collateral for the specified miner's sector
 	StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error)
 	// StateMinerAvailableBalance returns the portion of a miner's balance that can be withdrawn or spent
 	StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)
 	// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector
 	StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error)
-	// StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found
+	// StateSectorGetInfo returns the on-chain info for the specified miner's sector
 	// NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate
 	// expiration epoch
 	StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error)
@@ -342,6 +276,7 @@ type FullNode interface {
 	StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*SectorExpiration, error)
 	// StateSectorPartition finds deadline/partition with the specified sector
 	StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*SectorLocation, error)
+	StatePledgeCollateral(context.Context, types.TipSetKey) (types.BigInt, error)
 	// StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed
 	StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error)
 	// StateWaitMsg looks back in the chain for a message. If not found, it blocks until the
@@ -377,12 +312,6 @@ type FullNode interface {
 	// Returns nil if there is no entry in the data cap table for the
 	// address.
 	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*verifreg.DataCap, error)
-	// StateDealProviderCollateralBounds returns the min and max collateral a storage provider
-	// can issue. It takes the deal size and verified status as parameters.
-	StateDealProviderCollateralBounds(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (DealCollateralBounds, error)
-
-	// StateCirculatingSupply returns the circulating supply of Filecoin at the given tipset
-	StateCirculatingSupply(context.Context, types.TipSetKey) (CirculatingSupply, error)
 
 	// MethodGroup: Msig
 	// The Msig methods are used to interact with multisig wallets on the
@@ -425,22 +354,18 @@ type FullNode interface {
 	// MethodGroup: Paych
 	// The Paych methods are for interacting with and managing payment channels
 
-	PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*ChannelInfo, error)
-	PaychGetWaitReady(context.Context, cid.Cid) (address.Address, error)
-	PaychAvailableFunds(ch address.Address) (*ChannelAvailableFunds, error)
-	PaychAvailableFundsByFromTo(from, to address.Address) (*ChannelAvailableFunds, error)
+	PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error)
 	PaychList(context.Context) ([]address.Address, error)
 	PaychStatus(context.Context, address.Address) (*PaychStatus, error)
-	PaychSettle(context.Context, address.Address) (cid.Cid, error)
-	PaychCollect(context.Context, address.Address) (cid.Cid, error)
+	PaychClose(context.Context, address.Address) (cid.Cid, error)
 	PaychAllocateLane(ctx context.Context, ch address.Address) (uint64, error)
 	PaychNewPayment(ctx context.Context, from, to address.Address, vouchers []VoucherSpec) (*PaymentInfo, error)
 	PaychVoucherCheckValid(context.Context, address.Address, *paych.SignedVoucher) error
 	PaychVoucherCheckSpendable(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (bool, error)
-	PaychVoucherCreate(context.Context, address.Address, types.BigInt, uint64) (*VoucherCreateResult, error)
+	PaychVoucherCreate(context.Context, address.Address, types.BigInt, uint64) (*paych.SignedVoucher, error)
 	PaychVoucherAdd(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error)
 	PaychVoucherList(context.Context, address.Address) ([]*paych.SignedVoucher, error)
-	PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error)
+	PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher) (cid.Cid, error)
 }
 
 type FileRef struct {
@@ -468,11 +393,11 @@ type SectorLocation struct {
 
 type ImportRes struct {
 	Root     cid.Cid
-	ImportID multistore.StoreID
+	ImportID int
 }
 
 type Import struct {
-	Key multistore.StoreID
+	Key int
 	Err string
 
 	Root     *cid.Cid
@@ -486,7 +411,6 @@ type DealInfo struct {
 	Message     string // more information about deal state, particularly errors
 	Provider    address.Address
 
-	DataRef  *storagemarket.DataRef
 	PieceCID cid.Cid
 	Size     uint64
 
@@ -494,12 +418,9 @@ type DealInfo struct {
 	Duration      uint64
 
 	DealID abi.DealID
-
-	CreationTime time.Time
 }
 
 type MsgLookup struct {
-	Message   cid.Cid // Can be different than requested, in case it was replaced, but only gas values changed
 	Receipt   types.MessageReceipt
 	ReturnDec interface{}
 	TipSet    types.TipSetKey
@@ -542,36 +463,14 @@ type PaychStatus struct {
 }
 
 type ChannelInfo struct {
-	Channel      address.Address
-	WaitSentinel cid.Cid
-}
-
-type ChannelAvailableFunds struct {
-	// Channel is the address of the channel
-	Channel *address.Address
-	// From is the from address of the channel (channel creator)
-	From address.Address
-	// To is the to address of the channel
-	To address.Address
-	// ConfirmedAmt is the amount of funds that have been confirmed on-chain
-	// for the channel
-	ConfirmedAmt types.BigInt
-	// PendingAmt is the amount of funds that are pending confirmation on-chain
-	PendingAmt types.BigInt
-	// PendingWaitSentinel can be used with PaychGetWaitReady to wait for
-	// confirmation of pending funds
-	PendingWaitSentinel *cid.Cid
-	// QueuedAmt is the amount that is queued up behind a pending request
-	QueuedAmt types.BigInt
-	// VoucherRedeemedAmt is the amount that is redeemed by vouchers on-chain
-	// and in the local datastore
-	VoucherReedeemedAmt types.BigInt
+	Channel        address.Address
+	ChannelMessage cid.Cid
 }
 
 type PaymentInfo struct {
-	Channel      address.Address
-	WaitSentinel cid.Cid
-	Vouchers     []*paych.SignedVoucher
+	Channel        address.Address
+	ChannelMessage *cid.Cid
+	Vouchers       []*paych.SignedVoucher
 }
 
 type VoucherSpec struct {
@@ -583,16 +482,6 @@ type VoucherSpec struct {
 	Extra *paych.ModVerifyParams
 }
 
-// VoucherCreateResult is the response to calling PaychVoucherCreate
-type VoucherCreateResult struct {
-	// Voucher that was created, or nil if there was an error or if there
-	// were insufficient funds in the channel
-	Voucher *paych.SignedVoucher
-	// Shortfall is the additional amount that would be needed in the channel
-	// in order to be able to create the voucher
-	Shortfall types.BigInt
-}
-
 type MinerPower struct {
 	MinerPower power.Claim
 	TotalPower power.Claim
@@ -606,11 +495,10 @@ type QueryOffer struct {
 
 	Size                    uint64
 	MinPrice                types.BigInt
-	UnsealPrice             types.BigInt
 	PaymentInterval         uint64
 	PaymentIntervalIncrease uint64
 	Miner                   address.Address
-	MinerPeer               retrievalmarket.RetrievalPeer
+	MinerPeerID             peer.ID
 }
 
 func (o *QueryOffer) Order(client address.Address) RetrievalOrder {
@@ -619,13 +507,12 @@ func (o *QueryOffer) Order(client address.Address) RetrievalOrder {
 		Piece:                   o.Piece,
 		Size:                    o.Size,
 		Total:                   o.MinPrice,
-		UnsealPrice:             o.UnsealPrice,
 		PaymentInterval:         o.PaymentInterval,
 		PaymentIntervalIncrease: o.PaymentIntervalIncrease,
 		Client:                  client,
 
-		Miner:     o.Miner,
-		MinerPeer: o.MinerPeer,
+		Miner:       o.Miner,
+		MinerPeerID: o.MinerPeerID,
 	}
 }
 
@@ -646,12 +533,11 @@ type RetrievalOrder struct {
 	Size  uint64
 	// TODO: support offset
 	Total                   types.BigInt
-	UnsealPrice             types.BigInt
 	PaymentInterval         uint64
 	PaymentIntervalIncrease uint64
 	Client                  address.Address
 	Miner                   address.Address
-	MinerPeer               retrievalmarket.RetrievalPeer
+	MinerPeerID             peer.ID
 }
 
 type InvocResult struct {
@@ -668,15 +554,14 @@ type MethodCall struct {
 }
 
 type StartDealParams struct {
-	Data               *storagemarket.DataRef
-	Wallet             address.Address
-	Miner              address.Address
-	EpochPrice         types.BigInt
-	MinBlocksDuration  uint64
-	ProviderCollateral big.Int
-	DealStartEpoch     abi.ChainEpoch
-	FastRetrieval      bool
-	VerifiedDeal       bool
+	Data              *storagemarket.DataRef
+	Wallet            address.Address
+	Miner             address.Address
+	EpochPrice        types.BigInt
+	MinBlocksDuration uint64
+	DealStartEpoch    abi.ChainEpoch
+	FastRetrieval     bool
+	VerifiedDeal      bool
 }
 
 type IpldObject struct {
@@ -728,28 +613,14 @@ type ComputeStateOutput struct {
 	Trace []*InvocResult
 }
 
-type DealCollateralBounds struct {
-	Min abi.TokenAmount
-	Max abi.TokenAmount
-}
-
-type CirculatingSupply struct {
-	FilVested      abi.TokenAmount
-	FilMined       abi.TokenAmount
-	FilBurnt       abi.TokenAmount
-	FilLocked      abi.TokenAmount
-	FilCirculating abi.TokenAmount
-}
-
 type MiningBaseInfo struct {
 	MinerPower      types.BigInt
 	NetworkPower    types.BigInt
-	Sectors         []proof.SectorInfo
+	Sectors         []abi.SectorInfo
 	WorkerKey       address.Address
 	SectorSize      abi.SectorSize
 	PrevBeaconEntry types.BeaconEntry
 	BeaconEntries   []types.BeaconEntry
-	HasMinPower     bool
 }
 
 type BlockTemplate struct {
@@ -761,12 +632,7 @@ type BlockTemplate struct {
 	Messages         []*types.SignedMessage
 	Epoch            abi.ChainEpoch
 	Timestamp        uint64
-	WinningPoStProof []proof.PoStProof
-}
-
-type DataSize struct {
-	PayloadSize int64
-	PieceSize   abi.PaddedPieceSize
+	WinningPoStProof []abi.PoStProof
 }
 
 type CommPRet struct {

api/api_storage.go

@@ -8,14 +8,12 @@ import (
 	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-fil-markets/piecestore"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
-	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	"github.com/filecoin-project/sector-storage/fsutil"
+	"github.com/filecoin-project/sector-storage/stores"
+	"github.com/filecoin-project/sector-storage/storiface"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 )
 
 // StorageMiner is a low-level interface to the Filecoin network storage miner node
@@ -32,7 +30,7 @@ type StorageMiner interface {
 	PledgeSector(context.Context) error
 
 	// Get the status of a given sector by ID
-	SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (SectorInfo, error)
+	SectorsStatus(context.Context, abi.SectorNumber) (SectorInfo, error)
 
 	// List all staged sectors
 	SectorsList(context.Context) ([]abi.SectorNumber, error)
@@ -63,24 +61,14 @@ type StorageMiner interface {
 	// WorkerConnect tells the node to connect to workers RPC
 	WorkerConnect(context.Context, string) error
 	WorkerStats(context.Context) (map[uint64]storiface.WorkerStats, error)
-	WorkerJobs(context.Context) (map[uint64][]storiface.WorkerJob, error)
-
-	// SealingSchedDiag dumps internal sealing scheduler state
-	SealingSchedDiag(context.Context) (interface{}, error)
 
 	stores.SectorIndex
 
 	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error
 	MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error)
-	MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error)
-	MarketGetDealUpdates(ctx context.Context) (<-chan storagemarket.MinerDeal, error)
 	MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error)
-	MarketSetAsk(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error
+	MarketSetAsk(ctx context.Context, price types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error
 	MarketGetAsk(ctx context.Context) (*storagemarket.SignedStorageAsk, error)
-	MarketSetRetrievalAsk(ctx context.Context, rask *retrievalmarket.Ask) error
-	MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error)
-	MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)
-	MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)
 
 	DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error
 	DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error)
@@ -96,11 +84,6 @@ type StorageMiner interface {
 	DealsSetConsiderOfflineRetrievalDeals(context.Context, bool) error
 
 	StorageAddLocal(ctx context.Context, path string) error
-
-	PiecesListPieces(ctx context.Context) ([]cid.Cid, error)
-	PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error)
-	PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error)
-	PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error)
 }
 
 type SealRes struct {
@@ -120,40 +103,24 @@ type SectorLog struct {
 }
 
 type SectorInfo struct {
-	SectorID     abi.SectorNumber
-	State        SectorState
-	CommD        *cid.Cid
-	CommR        *cid.Cid
-	Proof        []byte
-	Deals        []abi.DealID
-	Ticket       SealTicket
-	Seed         SealSeed
-	PreCommitMsg *cid.Cid
-	CommitMsg    *cid.Cid
-	Retries      uint64
-	ToUpgrade    bool
+	SectorID abi.SectorNumber
+	State    SectorState
+	CommD    *cid.Cid
+	CommR    *cid.Cid
+	Proof    []byte
+	Deals    []abi.DealID
+	Ticket   SealTicket
+	Seed     SealSeed
+	Retries  uint64
 
 	LastErr string
 
 	Log []SectorLog
-
-	// On Chain Info
-	SealProof          abi.RegisteredSealProof // The seal proof type implies the PoSt proof/s
-	Activation         abi.ChainEpoch          // Epoch during which the sector proof was accepted
-	Expiration         abi.ChainEpoch          // Epoch during which the sector expires
-	DealWeight         abi.DealWeight          // Integral of active deals over sector lifetime
-	VerifiedDealWeight abi.DealWeight          // Integral of active verified deals over sector lifetime
-	InitialPledge      abi.TokenAmount         // Pledge collected to commit this sector
-	// Expiration Info
-	OnTime abi.ChainEpoch
-	// non-zero if sector is faulty, epoch at which it will be permanently
-	// removed if it doesn't recover
-	Early abi.ChainEpoch
 }
 
 type SealedRef struct {
 	SectorID abi.SectorNumber
-	Offset   abi.PaddedPieceSize
+	Offset   uint64
 	Size     abi.UnpaddedPieceSize
 }
 

api/api_worker.go

@@ -6,10 +6,10 @@ import (
 
 	"github.com/ipfs/go-cid"
 
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	"github.com/filecoin-project/sector-storage/sealtasks"
+	"github.com/filecoin-project/sector-storage/stores"
+	"github.com/filecoin-project/sector-storage/storiface"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-storage/storage"
 
 	"github.com/filecoin-project/lotus/build"
@@ -23,16 +23,12 @@ type WorkerAPI interface {
 	Paths(context.Context) ([]stores.StoragePath, error)
 	Info(context.Context) (storiface.WorkerInfo, error)
 
-	AddPiece(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error)
-
 	storage.Sealer
 
-	MoveStorage(ctx context.Context, sector abi.SectorID, types stores.SectorFileType) error
+	MoveStorage(ctx context.Context, sector abi.SectorID) error
 
 	UnsealPiece(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error
-	ReadPiece(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) (bool, error)
-
-	StorageAddLocal(ctx context.Context, path string) error
+	ReadPiece(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) error
 
 	Fetch(context.Context, abi.SectorID, stores.SectorFileType, stores.PathType, stores.AcquireMode) error
 

api/apibstore/apibstore.go

@@ -5,9 +5,8 @@ import (
 
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 type ChainIO interface {

api/apistruct/struct.go

@@ -6,29 +6,22 @@ import (
 	"time"
 
 	"github.com/ipfs/go-cid"
-	metrics "github.com/libp2p/go-libp2p-core/metrics"
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
-	protocol "github.com/libp2p/go-libp2p-core/protocol"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-bitfield"
-	"github.com/filecoin-project/go-fil-markets/piecestore"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-jsonrpc/auth"
-	"github.com/filecoin-project/go-multistore"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/go-state-types/crypto"
-	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
-	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-	marketevents "github.com/filecoin-project/lotus/markets/loggers"
+	"github.com/filecoin-project/sector-storage/fsutil"
+	"github.com/filecoin-project/sector-storage/sealtasks"
+	"github.com/filecoin-project/sector-storage/stores"
+	"github.com/filecoin-project/sector-storage/storiface"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-actors/actors/builtin/paych"
 	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
 	"github.com/filecoin-project/specs-storage/storage"
 
 	"github.com/filecoin-project/lotus/api"
@@ -45,18 +38,13 @@ type CommonStruct struct {
 		AuthVerify func(ctx context.Context, token string) ([]auth.Permission, error) `perm:"read"`
 		AuthNew    func(ctx context.Context, perms []auth.Permission) ([]byte, error) `perm:"admin"`
 
-		NetConnectedness            func(context.Context, peer.ID) (network.Connectedness, error)    `perm:"read"`
-		NetPeers                    func(context.Context) ([]peer.AddrInfo, error)                   `perm:"read"`
-		NetConnect                  func(context.Context, peer.AddrInfo) error                       `perm:"write"`
-		NetAddrsListen              func(context.Context) (peer.AddrInfo, error)                     `perm:"read"`
-		NetDisconnect               func(context.Context, peer.ID) error                             `perm:"write"`
-		NetFindPeer                 func(context.Context, peer.ID) (peer.AddrInfo, error)            `perm:"read"`
-		NetPubsubScores             func(context.Context) ([]api.PubsubScore, error)                 `perm:"read"`
-		NetAutoNatStatus            func(context.Context) (api.NatInfo, error)                       `perm:"read"`
-		NetBandwidthStats           func(ctx context.Context) (metrics.Stats, error)                 `perm:"read"`
-		NetBandwidthStatsByPeer     func(ctx context.Context) (map[string]metrics.Stats, error)      `perm:"read"`
-		NetBandwidthStatsByProtocol func(ctx context.Context) (map[protocol.ID]metrics.Stats, error) `perm:"read"`
-		NetAgentVersion             func(ctx context.Context, p peer.ID) (string, error)             `perm:"read"`
+		NetConnectedness func(context.Context, peer.ID) (network.Connectedness, error) `perm:"read"`
+		NetPeers         func(context.Context) ([]peer.AddrInfo, error)                `perm:"read"`
+		NetConnect       func(context.Context, peer.AddrInfo) error                    `perm:"write"`
+		NetAddrsListen   func(context.Context) (peer.AddrInfo, error)                  `perm:"read"`
+		NetDisconnect    func(context.Context, peer.ID) error                          `perm:"write"`
+		NetFindPeer      func(context.Context, peer.ID) (peer.AddrInfo, error)         `perm:"read"`
+		NetPubsubScores  func(context.Context) ([]api.PubsubScore, error)              `perm:"read"`
 
 		ID      func(context.Context) (peer.ID, error)     `perm:"read"`
 		Version func(context.Context) (api.Version, error) `perm:"read"`
@@ -74,33 +62,25 @@ type FullNodeStruct struct {
 	CommonStruct
 
 	Internal struct {
-		ChainNotify                   func(context.Context) (<-chan []*api.HeadChange, error)                                                            `perm:"read"`
-		ChainHead                     func(context.Context) (*types.TipSet, error)                                                                       `perm:"read"`
-		ChainGetRandomnessFromTickets func(context.Context, types.TipSetKey, crypto.DomainSeparationTag, abi.ChainEpoch, []byte) (abi.Randomness, error) `perm:"read"`
-		ChainGetRandomnessFromBeacon  func(context.Context, types.TipSetKey, crypto.DomainSeparationTag, abi.ChainEpoch, []byte) (abi.Randomness, error) `perm:"read"`
-		ChainGetBlock                 func(context.Context, cid.Cid) (*types.BlockHeader, error)                                                         `perm:"read"`
-		ChainGetTipSet                func(context.Context, types.TipSetKey) (*types.TipSet, error)                                                      `perm:"read"`
-		ChainGetBlockMessages         func(context.Context, cid.Cid) (*api.BlockMessages, error)                                                         `perm:"read"`
-		ChainGetParentReceipts        func(context.Context, cid.Cid) ([]*types.MessageReceipt, error)                                                    `perm:"read"`
-		ChainGetParentMessages        func(context.Context, cid.Cid) ([]api.Message, error)                                                              `perm:"read"`
-		ChainGetTipSetByHeight        func(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)                                      `perm:"read"`
-		ChainReadObj                  func(context.Context, cid.Cid) ([]byte, error)                                                                     `perm:"read"`
-		ChainHasObj                   func(context.Context, cid.Cid) (bool, error)                                                                       `perm:"read"`
-		ChainStatObj                  func(context.Context, cid.Cid, cid.Cid) (api.ObjStat, error)                                                       `perm:"read"`
-		ChainSetHead                  func(context.Context, types.TipSetKey) error                                                                       `perm:"admin"`
-		ChainGetGenesis               func(context.Context) (*types.TipSet, error)                                                                       `perm:"read"`
-		ChainTipSetWeight             func(context.Context, types.TipSetKey) (types.BigInt, error)                                                       `perm:"read"`
-		ChainGetNode                  func(ctx context.Context, p string) (*api.IpldObject, error)                                                       `perm:"read"`
-		ChainGetMessage               func(context.Context, cid.Cid) (*types.Message, error)                                                             `perm:"read"`
-		ChainGetPath                  func(context.Context, types.TipSetKey, types.TipSetKey) ([]*api.HeadChange, error)                                 `perm:"read"`
-		ChainExport                   func(context.Context, abi.ChainEpoch, types.TipSetKey) (<-chan []byte, error)                                      `perm:"read"`
-
-		BeaconGetEntry func(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) `perm:"read"`
-
-		GasEstimateGasPremium func(context.Context, uint64, address.Address, int64, types.TipSetKey) (types.BigInt, error)         `perm:"read"`
-		GasEstimateGasLimit   func(context.Context, *types.Message, types.TipSetKey) (int64, error)                                `perm:"read"`
-		GasEstimateFeeCap     func(context.Context, *types.Message, int64, types.TipSetKey) (types.BigInt, error)                  `perm:"read"`
-		GasEstimateMessageGas func(context.Context, *types.Message, *api.MessageSendSpec, types.TipSetKey) (*types.Message, error) `perm:"read"`
+		ChainNotify            func(context.Context) (<-chan []*api.HeadChange, error)                                                            `perm:"read"`
+		ChainHead              func(context.Context) (*types.TipSet, error)                                                                       `perm:"read"`
+		ChainGetRandomness     func(context.Context, types.TipSetKey, crypto.DomainSeparationTag, abi.ChainEpoch, []byte) (abi.Randomness, error) `perm:"read"`
+		ChainGetBlock          func(context.Context, cid.Cid) (*types.BlockHeader, error)                                                         `perm:"read"`
+		ChainGetTipSet         func(context.Context, types.TipSetKey) (*types.TipSet, error)                                                      `perm:"read"`
+		ChainGetBlockMessages  func(context.Context, cid.Cid) (*api.BlockMessages, error)                                                         `perm:"read"`
+		ChainGetParentReceipts func(context.Context, cid.Cid) ([]*types.MessageReceipt, error)                                                    `perm:"read"`
+		ChainGetParentMessages func(context.Context, cid.Cid) ([]api.Message, error)                                                              `perm:"read"`
+		ChainGetTipSetByHeight func(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)                                      `perm:"read"`
+		ChainReadObj           func(context.Context, cid.Cid) ([]byte, error)                                                                     `perm:"read"`
+		ChainHasObj            func(context.Context, cid.Cid) (bool, error)                                                                       `perm:"read"`
+		ChainStatObj           func(context.Context, cid.Cid, cid.Cid) (api.ObjStat, error)                                                       `perm:"read"`
+		ChainSetHead           func(context.Context, types.TipSetKey) error                                                                       `perm:"admin"`
+		ChainGetGenesis        func(context.Context) (*types.TipSet, error)                                                                       `perm:"read"`
+		ChainTipSetWeight      func(context.Context, types.TipSetKey) (types.BigInt, error)                                                       `perm:"read"`
+		ChainGetNode           func(ctx context.Context, p string) (*api.IpldObject, error)                                                       `perm:"read"`
+		ChainGetMessage        func(context.Context, cid.Cid) (*types.Message, error)                                                             `perm:"read"`
+		ChainGetPath           func(context.Context, types.TipSetKey, types.TipSetKey) ([]*api.HeadChange, error)                                 `perm:"read"`
+		ChainExport            func(context.Context, types.TipSetKey) (<-chan []byte, error)                                                      `perm:"read"`
 
 		SyncState          func(context.Context) (*api.SyncState, error)                `perm:"read"`
 		SyncSubmitBlock    func(ctx context.Context, blk *types.BlockMsg) error         `perm:"write"`
@@ -108,18 +88,12 @@ type FullNodeStruct struct {
 		SyncMarkBad        func(ctx context.Context, bcid cid.Cid) error                `perm:"admin"`
 		SyncCheckBad       func(ctx context.Context, bcid cid.Cid) (string, error)      `perm:"read"`
 
-		MpoolGetConfig func(context.Context) (*types.MpoolConfig, error) `perm:"read"`
-		MpoolSetConfig func(context.Context, *types.MpoolConfig) error   `perm:"write"`
-
-		MpoolSelect func(context.Context, types.TipSetKey, float64) ([]*types.SignedMessage, error) `perm:"read"`
-
-		MpoolPending func(context.Context, types.TipSetKey) ([]*types.SignedMessage, error) `perm:"read"`
-		MpoolClear   func(context.Context, bool) error                                      `perm:"write"`
-
-		MpoolPush        func(context.Context, *types.SignedMessage) (cid.Cid, error)                              `perm:"write"`
-		MpoolPushMessage func(context.Context, *types.Message, *api.MessageSendSpec) (*types.SignedMessage, error) `perm:"sign"`
-		MpoolGetNonce    func(context.Context, address.Address) (uint64, error)                                    `perm:"read"`
-		MpoolSub         func(context.Context) (<-chan api.MpoolUpdate, error)                                     `perm:"read"`
+		MpoolPending          func(context.Context, types.TipSetKey) ([]*types.SignedMessage, error)                       `perm:"read"`
+		MpoolPush             func(context.Context, *types.SignedMessage) (cid.Cid, error)                                 `perm:"write"`
+		MpoolPushMessage      func(context.Context, *types.Message) (*types.SignedMessage, error)                          `perm:"sign"`
+		MpoolGetNonce         func(context.Context, address.Address) (uint64, error)                                       `perm:"read"`
+		MpoolSub              func(context.Context) (<-chan api.MpoolUpdate, error)                                        `perm:"read"`
+		MpoolEstimateGasPrice func(context.Context, uint64, address.Address, int64, types.TipSetKey) (types.BigInt, error) `perm:"read"`
 
 		MinerGetBaseInfo func(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*api.MiningBaseInfo, error) `perm:"read"`
 		MinerCreateBlock func(context.Context, *api.BlockTemplate) (*types.BlockMsg, error)                                   `perm:"write"`
@@ -137,66 +111,58 @@ type FullNodeStruct struct {
 		WalletImport         func(context.Context, *types.KeyInfo) (address.Address, error)                       `perm:"admin"`
 		WalletDelete         func(context.Context, address.Address) error                                         `perm:"write"`
 
-		ClientImport                              func(ctx context.Context, ref api.FileRef) (*api.ImportRes, error)                                                `perm:"admin"`
-		ClientListImports                         func(ctx context.Context) ([]api.Import, error)                                                                   `perm:"write"`
-		ClientRemoveImport                        func(ctx context.Context, importID multistore.StoreID) error                                                      `perm:"admin"`
-		ClientHasLocal                            func(ctx context.Context, root cid.Cid) (bool, error)                                                             `perm:"write"`
-		ClientFindData                            func(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]api.QueryOffer, error)                                 `perm:"read"`
-		ClientMinerQueryOffer                     func(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error)            `perm:"read"`
-		ClientStartDeal                           func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error)                                          `perm:"admin"`
-		ClientGetDealInfo                         func(context.Context, cid.Cid) (*api.DealInfo, error)                                                             `perm:"read"`
-		ClientListDeals                           func(ctx context.Context) ([]api.DealInfo, error)                                                                 `perm:"write"`
-		ClientGetDealUpdates                      func(ctx context.Context) (<-chan api.DealInfo, error)                                                            `perm:"read"`
-		ClientRetrieve                            func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error                                       `perm:"admin"`
-		ClientRetrieveWithEvents                  func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) `perm:"admin"`
-		ClientQueryAsk                            func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error)              `perm:"read"`
-		ClientCalcCommP                           func(ctx context.Context, inpath string) (*api.CommPRet, error)                                                   `perm:"read"`
-		ClientGenCar                              func(ctx context.Context, ref api.FileRef, outpath string) error                                                  `perm:"write"`
-		ClientDealSize                            func(ctx context.Context, root cid.Cid) (api.DataSize, error)                                                     `perm:"read"`
-		ClientListDataTransfers                   func(ctx context.Context) ([]api.DataTransferChannel, error)                                                      `perm:"write"`
-		ClientDataTransferUpdates                 func(ctx context.Context) (<-chan api.DataTransferChannel, error)                                                 `perm:"write"`
-		ClientRetrieveTryRestartInsufficientFunds func(ctx context.Context, paymentChannel address.Address) error                                                   `perm:"write"`
+		ClientImport          func(ctx context.Context, ref api.FileRef) (*api.ImportRes, error)                                     `perm:"admin"`
+		ClientListImports     func(ctx context.Context) ([]api.Import, error)                                                        `perm:"write"`
+		ClientRemoveImport    func(ctx context.Context, importID int) error                                                          `perm:"admin"`
+		ClientHasLocal        func(ctx context.Context, root cid.Cid) (bool, error)                                                  `perm:"write"`
+		ClientFindData        func(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]api.QueryOffer, error)                      `perm:"read"`
+		ClientMinerQueryOffer func(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error) `perm:"read"`
+		ClientStartDeal       func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error)                               `perm:"admin"`
+		ClientGetDealInfo     func(context.Context, cid.Cid) (*api.DealInfo, error)                                                  `perm:"read"`
+		ClientListDeals       func(ctx context.Context) ([]api.DealInfo, error)                                                      `perm:"write"`
+		ClientRetrieve        func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error                            `perm:"admin"`
+		ClientQueryAsk        func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error)   `perm:"read"`
+		ClientCalcCommP       func(ctx context.Context, inpath string, miner address.Address) (*api.CommPRet, error)                 `perm:"read"`
+		ClientGenCar          func(ctx context.Context, ref api.FileRef, outpath string) error                                       `perm:"write"`
 
-		StateNetworkName                   func(context.Context) (dtypes.NetworkName, error)                                                                   `perm:"read"`
-		StateMinerSectors                  func(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error)   `perm:"read"`
-		StateMinerActiveSectors            func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error)                             `perm:"read"`
-		StateMinerProvingDeadline          func(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error)                                `perm:"read"`
-		StateMinerPower                    func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)                                    `perm:"read"`
-		StateMinerInfo                     func(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error)                                      `perm:"read"`
-		StateMinerDeadlines                func(context.Context, address.Address, types.TipSetKey) ([]*miner.Deadline, error)                                  `perm:"read"`
-		StateMinerPartitions               func(context.Context, address.Address, uint64, types.TipSetKey) ([]*miner.Partition, error)                         `perm:"read"`
-		StateMinerFaults                   func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error)                                  `perm:"read"`
-		StateAllMinerFaults                func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error)                                        `perm:"read"`
-		StateMinerRecoveries               func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error)                                  `perm:"read"`
-		StateMinerPreCommitDepositForPower func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error)            `perm:"read"`
-		StateMinerInitialPledgeCollateral  func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error)            `perm:"read"`
-		StateMinerAvailableBalance         func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)                                       `perm:"read"`
-		StateSectorPreCommitInfo           func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) `perm:"read"`
-		StateSectorGetInfo                 func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error)         `perm:"read"`
-		StateSectorExpiration              func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*api.SectorExpiration, error)            `perm:"read"`
-		StateSectorPartition               func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*api.SectorLocation, error)              `perm:"read"`
-		StateCall                          func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error)                                    `perm:"read"`
-		StateReplay                        func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error)                                           `perm:"read"`
-		StateGetActor                      func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error)                                       `perm:"read"`
-		StateReadState                     func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error)                                    `perm:"read"`
-		StateWaitMsg                       func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error)                                   `perm:"read"`
-		StateSearchMsg                     func(context.Context, cid.Cid) (*api.MsgLookup, error)                                                              `perm:"read"`
-		StateListMiners                    func(context.Context, types.TipSetKey) ([]address.Address, error)                                                   `perm:"read"`
-		StateListActors                    func(context.Context, types.TipSetKey) ([]address.Address, error)                                                   `perm:"read"`
-		StateMarketBalance                 func(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error)                                  `perm:"read"`
-		StateMarketParticipants            func(context.Context, types.TipSetKey) (map[string]api.MarketBalance, error)                                        `perm:"read"`
-		StateMarketDeals                   func(context.Context, types.TipSetKey) (map[string]api.MarketDeal, error)                                           `perm:"read"`
-		StateMarketStorageDeal             func(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error)                                         `perm:"read"`
-		StateLookupID                      func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)                       `perm:"read"`
-		StateAccountKey                    func(context.Context, address.Address, types.TipSetKey) (address.Address, error)                                    `perm:"read"`
-		StateChangedActors                 func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error)                                             `perm:"read"`
-		StateGetReceipt                    func(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)                                      `perm:"read"`
-		StateMinerSectorCount              func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error)                                   `perm:"read"`
-		StateListMessages                  func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error)        `perm:"read"`
-		StateCompute                       func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error)           `perm:"read"`
-		StateVerifiedClientStatus          func(context.Context, address.Address, types.TipSetKey) (*verifreg.DataCap, error)                                  `perm:"read"`
-		StateDealProviderCollateralBounds  func(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error)                 `perm:"read"`
-		StateCirculatingSupply             func(context.Context, types.TipSetKey) (api.CirculatingSupply, error)                                               `perm:"read"`
+		StateNetworkName                  func(context.Context) (dtypes.NetworkName, error)                                                                   `perm:"read"`
+		StateMinerSectors                 func(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error)        `perm:"read"`
+		StateMinerActiveSectors           func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error)                             `perm:"read"`
+		StateMinerProvingDeadline         func(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error)                                `perm:"read"`
+		StateMinerPower                   func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)                                    `perm:"read"`
+		StateMinerInfo                    func(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error)                                      `perm:"read"`
+		StateMinerDeadlines               func(context.Context, address.Address, types.TipSetKey) ([]*miner.Deadline, error)                                  `perm:"read"`
+		StateMinerPartitions              func(context.Context, address.Address, uint64, types.TipSetKey) ([]*miner.Partition, error)                         `perm:"read"`
+		StateMinerFaults                  func(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)                                      `perm:"read"`
+		StateAllMinerFaults               func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error)                                        `perm:"read"`
+		StateMinerRecoveries              func(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)                                      `perm:"read"`
+		StateMinerInitialPledgeCollateral func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error)            `perm:"read"`
+		StateMinerAvailableBalance        func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)                                       `perm:"read"`
+		StateSectorPreCommitInfo          func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) `perm:"read"`
+		StateSectorGetInfo                func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error)         `perm:"read"`
+		StateSectorExpiration             func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*api.SectorExpiration, error)            `perm:"read"`
+		StateSectorPartition              func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*api.SectorLocation, error)              `perm:"read"`
+		StateCall                         func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error)                                    `perm:"read"`
+		StateReplay                       func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error)                                           `perm:"read"`
+		StateGetActor                     func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error)                                       `perm:"read"`
+		StateReadState                    func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error)                                    `perm:"read"`
+		StatePledgeCollateral             func(context.Context, types.TipSetKey) (types.BigInt, error)                                                        `perm:"read"`
+		StateWaitMsg                      func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error)                                   `perm:"read"`
+		StateSearchMsg                    func(context.Context, cid.Cid) (*api.MsgLookup, error)                                                              `perm:"read"`
+		StateListMiners                   func(context.Context, types.TipSetKey) ([]address.Address, error)                                                   `perm:"read"`
+		StateListActors                   func(context.Context, types.TipSetKey) ([]address.Address, error)                                                   `perm:"read"`
+		StateMarketBalance                func(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error)                                  `perm:"read"`
+		StateMarketParticipants           func(context.Context, types.TipSetKey) (map[string]api.MarketBalance, error)                                        `perm:"read"`
+		StateMarketDeals                  func(context.Context, types.TipSetKey) (map[string]api.MarketDeal, error)                                           `perm:"read"`
+		StateMarketStorageDeal            func(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error)                                         `perm:"read"`
+		StateLookupID                     func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)                       `perm:"read"`
+		StateAccountKey                   func(context.Context, address.Address, types.TipSetKey) (address.Address, error)                                    `perm:"read"`
+		StateChangedActors                func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error)                                             `perm:"read"`
+		StateGetReceipt                   func(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)                                      `perm:"read"`
+		StateMinerSectorCount             func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error)                                   `perm:"read"`
+		StateListMessages                 func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error)        `perm:"read"`
+		StateCompute                      func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error)           `perm:"read"`
+		StateVerifiedClientStatus         func(context.Context, address.Address, types.TipSetKey) (*verifreg.DataCap, error)                                  `perm:"read"`
 
 		MsigGetAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)                                                                    `perm:"read"`
 		MsigCreate              func(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error)                   `perm:"sign"`
@@ -209,23 +175,19 @@ type FullNodeStruct struct {
 
 		MarketEnsureAvailable func(context.Context, address.Address, address.Address, types.BigInt) (cid.Cid, error) `perm:"sign"`
 
-		PaychGet                    func(ctx context.Context, from, to address.Address, amt types.BigInt) (*api.ChannelInfo, error)           `perm:"sign"`
-		PaychGetWaitReady           func(context.Context, cid.Cid) (address.Address, error)                                                   `perm:"sign"`
-		PaychAvailableFunds         func(address.Address) (*api.ChannelAvailableFunds, error)                                                 `perm:"sign"`
-		PaychAvailableFundsByFromTo func(address.Address, address.Address) (*api.ChannelAvailableFunds, error)                                `perm:"sign"`
-		PaychList                   func(context.Context) ([]address.Address, error)                                                          `perm:"read"`
-		PaychStatus                 func(context.Context, address.Address) (*api.PaychStatus, error)                                          `perm:"read"`
-		PaychSettle                 func(context.Context, address.Address) (cid.Cid, error)                                                   `perm:"sign"`
-		PaychCollect                func(context.Context, address.Address) (cid.Cid, error)                                                   `perm:"sign"`
-		PaychAllocateLane           func(context.Context, address.Address) (uint64, error)                                                    `perm:"sign"`
-		PaychNewPayment             func(ctx context.Context, from, to address.Address, vouchers []api.VoucherSpec) (*api.PaymentInfo, error) `perm:"sign"`
-		PaychVoucherCheck           func(context.Context, *paych.SignedVoucher) error                                                         `perm:"read"`
-		PaychVoucherCheckValid      func(context.Context, address.Address, *paych.SignedVoucher) error                                        `perm:"read"`
-		PaychVoucherCheckSpendable  func(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (bool, error)                `perm:"read"`
-		PaychVoucherAdd             func(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error)  `perm:"write"`
-		PaychVoucherCreate          func(context.Context, address.Address, big.Int, uint64) (*api.VoucherCreateResult, error)                 `perm:"sign"`
-		PaychVoucherList            func(context.Context, address.Address) ([]*paych.SignedVoucher, error)                                    `perm:"write"`
-		PaychVoucherSubmit          func(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error)             `perm:"sign"`
+		PaychGet                   func(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*api.ChannelInfo, error)   `perm:"sign"`
+		PaychList                  func(context.Context) ([]address.Address, error)                                                          `perm:"read"`
+		PaychStatus                func(context.Context, address.Address) (*api.PaychStatus, error)                                          `perm:"read"`
+		PaychClose                 func(context.Context, address.Address) (cid.Cid, error)                                                   `perm:"sign"`
+		PaychAllocateLane          func(context.Context, address.Address) (uint64, error)                                                    `perm:"sign"`
+		PaychNewPayment            func(ctx context.Context, from, to address.Address, vouchers []api.VoucherSpec) (*api.PaymentInfo, error) `perm:"sign"`
+		PaychVoucherCheck          func(context.Context, *paych.SignedVoucher) error                                                         `perm:"read"`
+		PaychVoucherCheckValid     func(context.Context, address.Address, *paych.SignedVoucher) error                                        `perm:"read"`
+		PaychVoucherCheckSpendable func(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (bool, error)                `perm:"read"`
+		PaychVoucherAdd            func(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error)  `perm:"write"`
+		PaychVoucherCreate         func(context.Context, address.Address, big.Int, uint64) (*paych.SignedVoucher, error)                     `perm:"sign"`
+		PaychVoucherList           func(context.Context, address.Address) ([]*paych.SignedVoucher, error)                                    `perm:"write"`
+		PaychVoucherSubmit         func(context.Context, address.Address, *paych.SignedVoucher) (cid.Cid, error)                             `perm:"sign"`
 	}
 }
 
@@ -242,37 +204,28 @@ type StorageMinerStruct struct {
 
 		MiningBase func(context.Context) (*types.TipSet, error) `perm:"read"`
 
-		MarketImportDealData      func(context.Context, cid.Cid, string) error                                                                                                                                 `perm:"write"`
-		MarketListDeals           func(ctx context.Context) ([]storagemarket.StorageDeal, error)                                                                                                               `perm:"read"`
-		MarketListRetrievalDeals  func(ctx context.Context) ([]retrievalmarket.ProviderDealState, error)                                                                                                       `perm:"read"`
-		MarketGetDealUpdates      func(ctx context.Context) (<-chan storagemarket.MinerDeal, error)                                                                                                            `perm:"read"`
-		MarketListIncompleteDeals func(ctx context.Context) ([]storagemarket.MinerDeal, error)                                                                                                                 `perm:"read"`
-		MarketSetAsk              func(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error `perm:"admin"`
-		MarketGetAsk              func(ctx context.Context) (*storagemarket.SignedStorageAsk, error)                                                                                                           `perm:"read"`
-		MarketSetRetrievalAsk     func(ctx context.Context, rask *retrievalmarket.Ask) error                                                                                                                   `perm:"admin"`
-		MarketGetRetrievalAsk     func(ctx context.Context) (*retrievalmarket.Ask, error)                                                                                                                      `perm:"read"`
-		MarketListDataTransfers   func(ctx context.Context) ([]api.DataTransferChannel, error)                                                                                                                 `perm:"write"`
-		MarketDataTransferUpdates func(ctx context.Context) (<-chan api.DataTransferChannel, error)                                                                                                            `perm:"write"`
+		MarketImportDealData      func(context.Context, cid.Cid, string) error                                                                                                     `perm:"write"`
+		MarketListDeals           func(ctx context.Context) ([]storagemarket.StorageDeal, error)                                                                                   `perm:"read"`
+		MarketListIncompleteDeals func(ctx context.Context) ([]storagemarket.MinerDeal, error)                                                                                     `perm:"read"`
+		MarketSetAsk              func(ctx context.Context, price types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error `perm:"admin"`
+		MarketGetAsk              func(ctx context.Context) (*storagemarket.SignedStorageAsk, error)                                                                               `perm:"read"`
 
 		PledgeSector func(context.Context) error `perm:"write"`
 
-		SectorsStatus                 func(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) `perm:"read"`
-		SectorsList                   func(context.Context) ([]abi.SectorNumber, error)                                             `perm:"read"`
-		SectorsRefs                   func(context.Context) (map[string][]api.SealedRef, error)                                     `perm:"read"`
-		SectorStartSealing            func(context.Context, abi.SectorNumber) error                                                 `perm:"write"`
-		SectorSetSealDelay            func(context.Context, time.Duration) error                                                    `perm:"write"`
-		SectorGetSealDelay            func(context.Context) (time.Duration, error)                                                  `perm:"read"`
-		SectorSetExpectedSealDuration func(context.Context, time.Duration) error                                                    `perm:"write"`
-		SectorGetExpectedSealDuration func(context.Context) (time.Duration, error)                                                  `perm:"read"`
-		SectorsUpdate                 func(context.Context, abi.SectorNumber, api.SectorState) error                                `perm:"admin"`
-		SectorRemove                  func(context.Context, abi.SectorNumber) error                                                 `perm:"admin"`
-		SectorMarkForUpgrade          func(ctx context.Context, id abi.SectorNumber) error                                          `perm:"admin"`
+		SectorsStatus                 func(context.Context, abi.SectorNumber) (api.SectorInfo, error) `perm:"read"`
+		SectorsList                   func(context.Context) ([]abi.SectorNumber, error)               `perm:"read"`
+		SectorsRefs                   func(context.Context) (map[string][]api.SealedRef, error)       `perm:"read"`
+		SectorStartSealing            func(context.Context, abi.SectorNumber) error                   `perm:"write"`
+		SectorSetSealDelay            func(context.Context, time.Duration) error                      `perm:"write"`
+		SectorGetSealDelay            func(context.Context) (time.Duration, error)                    `perm:"read"`
+		SectorSetExpectedSealDuration func(context.Context, time.Duration) error                      `perm:"write"`
+		SectorGetExpectedSealDuration func(context.Context) (time.Duration, error)                    `perm:"read"`
+		SectorsUpdate                 func(context.Context, abi.SectorNumber, api.SectorState) error  `perm:"admin"`
+		SectorRemove                  func(context.Context, abi.SectorNumber) error                   `perm:"admin"`
+		SectorMarkForUpgrade          func(ctx context.Context, id abi.SectorNumber) error            `perm:"admin"`
 
 		WorkerConnect func(context.Context, string) error                             `perm:"admin"` // TODO: worker perm
 		WorkerStats   func(context.Context) (map[uint64]storiface.WorkerStats, error) `perm:"admin"`
-		WorkerJobs    func(context.Context) (map[uint64][]storiface.WorkerJob, error) `perm:"admin"`
-
-		SealingSchedDiag func(context.Context) (interface{}, error) `perm:"admin"`
 
 		StorageList          func(context.Context) (map[stores.ID][]stores.Decl, error)                                                                                    `perm:"admin"`
 		StorageLocal         func(context.Context) (map[stores.ID]string, error)                                                                                           `perm:"admin"`
@@ -280,7 +233,7 @@ type StorageMinerStruct struct {
 		StorageAttach        func(context.Context, stores.StorageInfo, fsutil.FsStat) error                                                                                `perm:"admin"`
 		StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType, bool) error                                                             `perm:"admin"`
 		StorageDropSector    func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType) error                                                                   `perm:"admin"`
-		StorageFindSector    func(context.Context, abi.SectorID, stores.SectorFileType, abi.RegisteredSealProof, bool) ([]stores.SectorStorageInfo, error)                 `perm:"admin"`
+		StorageFindSector    func(context.Context, abi.SectorID, stores.SectorFileType, bool) ([]stores.SectorStorageInfo, error)                                          `perm:"admin"`
 		StorageInfo          func(context.Context, stores.ID) (stores.StorageInfo, error)                                                                                  `perm:"admin"`
 		StorageBestAlloc     func(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredSealProof, sealing stores.PathType) ([]stores.StorageInfo, error) `perm:"admin"`
 		StorageReportHealth  func(ctx context.Context, id stores.ID, report stores.HealthReport) error                                                                     `perm:"admin"`
@@ -301,11 +254,6 @@ type StorageMinerStruct struct {
 		DealsSetPieceCidBlocklist             func(context.Context, []cid.Cid) error                            `perm:"admin"`
 
 		StorageAddLocal func(ctx context.Context, path string) error `perm:"admin"`
-
-		PiecesListPieces   func(ctx context.Context) ([]cid.Cid, error)                               `perm:"read"`
-		PiecesListCidInfos func(ctx context.Context) ([]cid.Cid, error)                               `perm:"read"`
-		PiecesGetPieceInfo func(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) `perm:"read"`
-		PiecesGetCIDInfo   func(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) `perm:"read"`
 	}
 }
 
@@ -319,7 +267,6 @@ type WorkerStruct struct {
 		Paths     func(context.Context) ([]stores.StoragePath, error)            `perm:"admin"`
 		Info      func(context.Context) (storiface.WorkerInfo, error)            `perm:"admin"`
 
-		AddPiece        func(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error)                      `perm:"admin"`
 		SealPreCommit1  func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error)                                                           `perm:"admin"`
 		SealPreCommit2  func(context.Context, abi.SectorID, storage.PreCommit1Out) (cids storage.SectorCids, err error)                                                                                            `perm:"admin"`
 		SealCommit1     func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) `perm:"admin"`
@@ -327,11 +274,10 @@ type WorkerStruct struct {
 		FinalizeSector  func(context.Context, abi.SectorID, []storage.Range) error                                                                                                                                 `perm:"admin"`
 		ReleaseUnsealed func(ctx context.Context, sector abi.SectorID, safeToFree []storage.Range) error                                                                                                           `perm:"admin"`
 		Remove          func(ctx context.Context, sector abi.SectorID) error                                                                                                                                       `perm:"admin"`
-		MoveStorage     func(ctx context.Context, sector abi.SectorID, types stores.SectorFileType) error                                                                                                          `perm:"admin"`
-		StorageAddLocal func(ctx context.Context, path string) error                                                                                                                                               `perm:"admin"`
+		MoveStorage     func(ctx context.Context, sector abi.SectorID) error                                                                                                                                       `perm:"admin"`
 
 		UnsealPiece func(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error `perm:"admin"`
-		ReadPiece   func(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) (bool, error)           `perm:"admin"`
+		ReadPiece   func(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) error                   `perm:"admin"`
 
 		Fetch func(context.Context, abi.SectorID, stores.SectorFileType, stores.PathType, stores.AcquireMode) error `perm:"admin"`
 
@@ -352,7 +298,6 @@ func (c *CommonStruct) AuthNew(ctx context.Context, perms []auth.Permission) ([]
 func (c *CommonStruct) NetPubsubScores(ctx context.Context) ([]api.PubsubScore, error) {
 	return c.Internal.NetPubsubScores(ctx)
 }
-
 func (c *CommonStruct) NetConnectedness(ctx context.Context, pid peer.ID) (network.Connectedness, error) {
 	return c.Internal.NetConnectedness(ctx, pid)
 }
@@ -377,26 +322,6 @@ func (c *CommonStruct) NetFindPeer(ctx context.Context, p peer.ID) (peer.AddrInf
 	return c.Internal.NetFindPeer(ctx, p)
 }
 
-func (c *CommonStruct) NetAutoNatStatus(ctx context.Context) (api.NatInfo, error) {
-	return c.Internal.NetAutoNatStatus(ctx)
-}
-
-func (c *CommonStruct) NetBandwidthStats(ctx context.Context) (metrics.Stats, error) {
-	return c.Internal.NetBandwidthStats(ctx)
-}
-
-func (c *CommonStruct) NetBandwidthStatsByPeer(ctx context.Context) (map[string]metrics.Stats, error) {
-	return c.Internal.NetBandwidthStatsByPeer(ctx)
-}
-
-func (c *CommonStruct) NetBandwidthStatsByProtocol(ctx context.Context) (map[protocol.ID]metrics.Stats, error) {
-	return c.Internal.NetBandwidthStatsByProtocol(ctx)
-}
-
-func (c *CommonStruct) NetAgentVersion(ctx context.Context, p peer.ID) (string, error) {
-	return c.Internal.NetAgentVersion(ctx, p)
-}
-
 // ID implements API.ID
 func (c *CommonStruct) ID(ctx context.Context) (peer.ID, error) {
 	return c.Internal.ID(ctx)
@@ -429,7 +354,7 @@ func (c *FullNodeStruct) ClientListImports(ctx context.Context) ([]api.Import, e
 	return c.Internal.ClientListImports(ctx)
 }
 
-func (c *FullNodeStruct) ClientRemoveImport(ctx context.Context, importID multistore.StoreID) error {
+func (c *FullNodeStruct) ClientRemoveImport(ctx context.Context, importID int) error {
 	return c.Internal.ClientRemoveImport(ctx, importID)
 }
 
@@ -452,7 +377,6 @@ func (c *FullNodeStruct) ClientMinerQueryOffer(ctx context.Context, miner addres
 func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) {
 	return c.Internal.ClientStartDeal(ctx, params)
 }
-
 func (c *FullNodeStruct) ClientGetDealInfo(ctx context.Context, deal cid.Cid) (*api.DealInfo, error) {
 	return c.Internal.ClientGetDealInfo(ctx, deal)
 }
@@ -461,93 +385,41 @@ func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]api.DealInfo, e
 	return c.Internal.ClientListDeals(ctx)
 }
 
-func (c *FullNodeStruct) ClientGetDealUpdates(ctx context.Context) (<-chan api.DealInfo, error) {
-	return c.Internal.ClientGetDealUpdates(ctx)
-}
-
 func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error {
 	return c.Internal.ClientRetrieve(ctx, order, ref)
 }
 
-func (c *FullNodeStruct) ClientRetrieveWithEvents(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
-	return c.Internal.ClientRetrieveWithEvents(ctx, order, ref)
-}
-
 func (c *FullNodeStruct) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) {
 	return c.Internal.ClientQueryAsk(ctx, p, miner)
 }
-func (c *FullNodeStruct) ClientCalcCommP(ctx context.Context, inpath string) (*api.CommPRet, error) {
-	return c.Internal.ClientCalcCommP(ctx, inpath)
+func (c *FullNodeStruct) ClientCalcCommP(ctx context.Context, inpath string, miner address.Address) (*api.CommPRet, error) {
+	return c.Internal.ClientCalcCommP(ctx, inpath, miner)
 }
 
 func (c *FullNodeStruct) ClientGenCar(ctx context.Context, ref api.FileRef, outpath string) error {
 	return c.Internal.ClientGenCar(ctx, ref, outpath)
 }
 
-func (c *FullNodeStruct) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, error) {
-	return c.Internal.ClientDealSize(ctx, root)
-}
-
-func (c *FullNodeStruct) ClientListDataTransfers(ctx context.Context) ([]api.DataTransferChannel, error) {
-	return c.Internal.ClientListDataTransfers(ctx)
-}
-
-func (c *FullNodeStruct) ClientDataTransferUpdates(ctx context.Context) (<-chan api.DataTransferChannel, error) {
-	return c.Internal.ClientDataTransferUpdates(ctx)
-}
-
-func (c *FullNodeStruct) ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error {
-	return c.Internal.ClientRetrieveTryRestartInsufficientFunds(ctx, paymentChannel)
-}
-
-func (c *FullNodeStruct) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) {
-	return c.Internal.GasEstimateGasPremium(ctx, nblocksincl, sender, gaslimit, tsk)
-}
-
-func (c *FullNodeStruct) GasEstimateFeeCap(ctx context.Context, msg *types.Message, maxqueueblks int64, tsk types.TipSetKey) (types.BigInt, error) {
-	return c.Internal.GasEstimateFeeCap(ctx, msg, maxqueueblks, tsk)
-}
-
-func (c *FullNodeStruct) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) {
-	return c.Internal.GasEstimateMessageGas(ctx, msg, spec, tsk)
-}
-
-func (c *FullNodeStruct) GasEstimateGasLimit(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (int64, error) {
-	return c.Internal.GasEstimateGasLimit(ctx, msg, tsk)
-}
-
-func (c *FullNodeStruct) MpoolGetConfig(ctx context.Context) (*types.MpoolConfig, error) {
-	return c.Internal.MpoolGetConfig(ctx)
-}
-
-func (c *FullNodeStruct) MpoolSetConfig(ctx context.Context, cfg *types.MpoolConfig) error {
-	return c.Internal.MpoolSetConfig(ctx, cfg)
-}
-
-func (c *FullNodeStruct) MpoolSelect(ctx context.Context, tsk types.TipSetKey, tq float64) ([]*types.SignedMessage, error) {
-	return c.Internal.MpoolSelect(ctx, tsk, tq)
-}
-
 func (c *FullNodeStruct) MpoolPending(ctx context.Context, tsk types.TipSetKey) ([]*types.SignedMessage, error) {
 	return c.Internal.MpoolPending(ctx, tsk)
 }
 
-func (c *FullNodeStruct) MpoolClear(ctx context.Context, local bool) error {
-	return c.Internal.MpoolClear(ctx, local)
-}
-
 func (c *FullNodeStruct) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) {
 	return c.Internal.MpoolPush(ctx, smsg)
 }
 
-func (c *FullNodeStruct) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) {
-	return c.Internal.MpoolPushMessage(ctx, msg, spec)
+func (c *FullNodeStruct) MpoolPushMessage(ctx context.Context, msg *types.Message) (*types.SignedMessage, error) {
+	return c.Internal.MpoolPushMessage(ctx, msg)
 }
 
 func (c *FullNodeStruct) MpoolSub(ctx context.Context) (<-chan api.MpoolUpdate, error) {
 	return c.Internal.MpoolSub(ctx)
 }
 
+func (c *FullNodeStruct) MpoolEstimateGasPrice(ctx context.Context, nblocksincl uint64, sender address.Address, limit int64, tsk types.TipSetKey) (types.BigInt, error) {
+	return c.Internal.MpoolEstimateGasPrice(ctx, nblocksincl, sender, limit, tsk)
+}
+
 func (c *FullNodeStruct) MinerGetBaseInfo(ctx context.Context, maddr address.Address, epoch abi.ChainEpoch, tsk types.TipSetKey) (*api.MiningBaseInfo, error) {
 	return c.Internal.MinerGetBaseInfo(ctx, maddr, epoch, tsk)
 }
@@ -560,12 +432,8 @@ func (c *FullNodeStruct) ChainHead(ctx context.Context) (*types.TipSet, error) {
 	return c.Internal.ChainHead(ctx)
 }
 
-func (c *FullNodeStruct) ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
-	return c.Internal.ChainGetRandomnessFromTickets(ctx, tsk, personalization, randEpoch, entropy)
-}
-
-func (c *FullNodeStruct) ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
-	return c.Internal.ChainGetRandomnessFromBeacon(ctx, tsk, personalization, randEpoch, entropy)
+func (c *FullNodeStruct) ChainGetRandomness(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
+	return c.Internal.ChainGetRandomness(ctx, tsk, personalization, randEpoch, entropy)
 }
 
 func (c *FullNodeStruct) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) {
@@ -684,12 +552,8 @@ func (c *FullNodeStruct) ChainGetPath(ctx context.Context, from types.TipSetKey,
 	return c.Internal.ChainGetPath(ctx, from, to)
 }
 
-func (c *FullNodeStruct) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk types.TipSetKey) (<-chan []byte, error) {
-	return c.Internal.ChainExport(ctx, nroots, tsk)
-}
-
-func (c *FullNodeStruct) BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) {
-	return c.Internal.BeaconGetEntry(ctx, epoch)
+func (c *FullNodeStruct) ChainExport(ctx context.Context, tsk types.TipSetKey) (<-chan []byte, error) {
+	return c.Internal.ChainExport(ctx, tsk)
 }
 
 func (c *FullNodeStruct) SyncState(ctx context.Context) (*api.SyncState, error) {
@@ -716,7 +580,7 @@ func (c *FullNodeStruct) StateNetworkName(ctx context.Context) (dtypes.NetworkNa
 	return c.Internal.StateNetworkName(ctx)
 }
 
-func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, filterOut bool, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) {
+func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, filter *abi.BitField, filterOut bool, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) {
 	return c.Internal.StateMinerSectors(ctx, addr, filter, filterOut, tsk)
 }
 
@@ -744,7 +608,7 @@ func (c *FullNodeStruct) StateMinerPartitions(ctx context.Context, m address.Add
 	return c.Internal.StateMinerPartitions(ctx, m, dlIdx, tsk)
 }
 
-func (c *FullNodeStruct) StateMinerFaults(ctx context.Context, actor address.Address, tsk types.TipSetKey) (bitfield.BitField, error) {
+func (c *FullNodeStruct) StateMinerFaults(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*abi.BitField, error) {
 	return c.Internal.StateMinerFaults(ctx, actor, tsk)
 }
 
@@ -752,14 +616,10 @@ func (c *FullNodeStruct) StateAllMinerFaults(ctx context.Context, cutoff abi.Cha
 	return c.Internal.StateAllMinerFaults(ctx, cutoff, endTsk)
 }
 
-func (c *FullNodeStruct) StateMinerRecoveries(ctx context.Context, actor address.Address, tsk types.TipSetKey) (bitfield.BitField, error) {
+func (c *FullNodeStruct) StateMinerRecoveries(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*abi.BitField, error) {
 	return c.Internal.StateMinerRecoveries(ctx, actor, tsk)
 }
 
-func (c *FullNodeStruct) StateMinerPreCommitDepositForPower(ctx context.Context, maddr address.Address, pci miner.SectorPreCommitInfo, tsk types.TipSetKey) (types.BigInt, error) {
-	return c.Internal.StateMinerPreCommitDepositForPower(ctx, maddr, pci, tsk)
-}
-
 func (c *FullNodeStruct) StateMinerInitialPledgeCollateral(ctx context.Context, maddr address.Address, pci miner.SectorPreCommitInfo, tsk types.TipSetKey) (types.BigInt, error) {
 	return c.Internal.StateMinerInitialPledgeCollateral(ctx, maddr, pci, tsk)
 }
@@ -800,6 +660,10 @@ func (c *FullNodeStruct) StateReadState(ctx context.Context, addr address.Addres
 	return c.Internal.StateReadState(ctx, addr, tsk)
 }
 
+func (c *FullNodeStruct) StatePledgeCollateral(ctx context.Context, tsk types.TipSetKey) (types.BigInt, error) {
+	return c.Internal.StatePledgeCollateral(ctx, tsk)
+}
+
 func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confidence uint64) (*api.MsgLookup, error) {
 	return c.Internal.StateWaitMsg(ctx, msgc, confidence)
 }
@@ -860,14 +724,6 @@ func (c *FullNodeStruct) StateVerifiedClientStatus(ctx context.Context, addr add
 	return c.Internal.StateVerifiedClientStatus(ctx, addr, tsk)
 }
 
-func (c *FullNodeStruct) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) {
-	return c.Internal.StateDealProviderCollateralBounds(ctx, size, verified, tsk)
-}
-
-func (c *FullNodeStruct) StateCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) {
-	return c.Internal.StateCirculatingSupply(ctx, tsk)
-}
-
 func (c *FullNodeStruct) MsigGetAvailableBalance(ctx context.Context, a address.Address, tsk types.TipSetKey) (types.BigInt, error) {
 	return c.Internal.MsigGetAvailableBalance(ctx, a, tsk)
 }
@@ -904,20 +760,8 @@ func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr, wallet
 	return c.Internal.MarketEnsureAvailable(ctx, addr, wallet, amt)
 }
 
-func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*api.ChannelInfo, error) {
-	return c.Internal.PaychGet(ctx, from, to, amt)
-}
-
-func (c *FullNodeStruct) PaychGetWaitReady(ctx context.Context, sentinel cid.Cid) (address.Address, error) {
-	return c.Internal.PaychGetWaitReady(ctx, sentinel)
-}
-
-func (c *FullNodeStruct) PaychAvailableFunds(ch address.Address) (*api.ChannelAvailableFunds, error) {
-	return c.Internal.PaychAvailableFunds(ch)
-}
-
-func (c *FullNodeStruct) PaychAvailableFundsByFromTo(from, to address.Address) (*api.ChannelAvailableFunds, error) {
-	return c.Internal.PaychAvailableFundsByFromTo(from, to)
+func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*api.ChannelInfo, error) {
+	return c.Internal.PaychGet(ctx, from, to, ensureFunds)
 }
 
 func (c *FullNodeStruct) PaychList(ctx context.Context) ([]address.Address, error) {
@@ -940,7 +784,7 @@ func (c *FullNodeStruct) PaychVoucherAdd(ctx context.Context, addr address.Addre
 	return c.Internal.PaychVoucherAdd(ctx, addr, sv, proof, minDelta)
 }
 
-func (c *FullNodeStruct) PaychVoucherCreate(ctx context.Context, pch address.Address, amt types.BigInt, lane uint64) (*api.VoucherCreateResult, error) {
+func (c *FullNodeStruct) PaychVoucherCreate(ctx context.Context, pch address.Address, amt types.BigInt, lane uint64) (*paych.SignedVoucher, error) {
 	return c.Internal.PaychVoucherCreate(ctx, pch, amt, lane)
 }
 
@@ -948,12 +792,8 @@ func (c *FullNodeStruct) PaychVoucherList(ctx context.Context, pch address.Addre
 	return c.Internal.PaychVoucherList(ctx, pch)
 }
 
-func (c *FullNodeStruct) PaychSettle(ctx context.Context, a address.Address) (cid.Cid, error) {
-	return c.Internal.PaychSettle(ctx, a)
-}
-
-func (c *FullNodeStruct) PaychCollect(ctx context.Context, a address.Address) (cid.Cid, error) {
-	return c.Internal.PaychCollect(ctx, a)
+func (c *FullNodeStruct) PaychClose(ctx context.Context, a address.Address) (cid.Cid, error) {
+	return c.Internal.PaychClose(ctx, a)
 }
 
 func (c *FullNodeStruct) PaychAllocateLane(ctx context.Context, ch address.Address) (uint64, error) {
@@ -964,8 +804,8 @@ func (c *FullNodeStruct) PaychNewPayment(ctx context.Context, from, to address.A
 	return c.Internal.PaychNewPayment(ctx, from, to, vouchers)
 }
 
-func (c *FullNodeStruct) PaychVoucherSubmit(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) {
-	return c.Internal.PaychVoucherSubmit(ctx, ch, sv, secret, proof)
+func (c *FullNodeStruct) PaychVoucherSubmit(ctx context.Context, ch address.Address, sv *paych.SignedVoucher) (cid.Cid, error) {
+	return c.Internal.PaychVoucherSubmit(ctx, ch, sv)
 }
 
 // StorageMinerStruct
@@ -987,8 +827,8 @@ func (c *StorageMinerStruct) PledgeSector(ctx context.Context) error {
 }
 
 // Get the status of a given sector by ID
-func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) {
-	return c.Internal.SectorsStatus(ctx, sid, showOnChainInfo)
+func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid abi.SectorNumber) (api.SectorInfo, error) {
+	return c.Internal.SectorsStatus(ctx, sid)
 }
 
 // List all staged sectors
@@ -1040,14 +880,6 @@ func (c *StorageMinerStruct) WorkerStats(ctx context.Context) (map[uint64]storif
 	return c.Internal.WorkerStats(ctx)
 }
 
-func (c *StorageMinerStruct) WorkerJobs(ctx context.Context) (map[uint64][]storiface.WorkerJob, error) {
-	return c.Internal.WorkerJobs(ctx)
-}
-
-func (c *StorageMinerStruct) SealingSchedDiag(ctx context.Context) (interface{}, error) {
-	return c.Internal.SealingSchedDiag(ctx)
-}
-
 func (c *StorageMinerStruct) StorageAttach(ctx context.Context, si stores.StorageInfo, st fsutil.FsStat) error {
 	return c.Internal.StorageAttach(ctx, si, st)
 }
@@ -1060,8 +892,8 @@ func (c *StorageMinerStruct) StorageDropSector(ctx context.Context, storageId st
 	return c.Internal.StorageDropSector(ctx, storageId, s, ft)
 }
 
-func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types stores.SectorFileType, spt abi.RegisteredSealProof, allowFetch bool) ([]stores.SectorStorageInfo, error) {
-	return c.Internal.StorageFindSector(ctx, si, types, spt, allowFetch)
+func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types stores.SectorFileType, allowFetch bool) ([]stores.SectorStorageInfo, error) {
+	return c.Internal.StorageFindSector(ctx, si, types, allowFetch)
 }
 
 func (c *StorageMinerStruct) StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) {
@@ -1104,42 +936,18 @@ func (c *StorageMinerStruct) MarketListDeals(ctx context.Context) ([]storagemark
 	return c.Internal.MarketListDeals(ctx)
 }
 
-func (c *StorageMinerStruct) MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error) {
-	return c.Internal.MarketListRetrievalDeals(ctx)
-}
-
-func (c *StorageMinerStruct) MarketGetDealUpdates(ctx context.Context) (<-chan storagemarket.MinerDeal, error) {
-	return c.Internal.MarketGetDealUpdates(ctx)
-}
-
 func (c *StorageMinerStruct) MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error) {
 	return c.Internal.MarketListIncompleteDeals(ctx)
 }
 
-func (c *StorageMinerStruct) MarketSetAsk(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error {
-	return c.Internal.MarketSetAsk(ctx, price, verifiedPrice, duration, minPieceSize, maxPieceSize)
+func (c *StorageMinerStruct) MarketSetAsk(ctx context.Context, price types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error {
+	return c.Internal.MarketSetAsk(ctx, price, duration, minPieceSize, maxPieceSize)
 }
 
 func (c *StorageMinerStruct) MarketGetAsk(ctx context.Context) (*storagemarket.SignedStorageAsk, error) {
 	return c.Internal.MarketGetAsk(ctx)
 }
 
-func (c *StorageMinerStruct) MarketSetRetrievalAsk(ctx context.Context, rask *retrievalmarket.Ask) error {
-	return c.Internal.MarketSetRetrievalAsk(ctx, rask)
-}
-
-func (c *StorageMinerStruct) MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error) {
-	return c.Internal.MarketGetRetrievalAsk(ctx)
-}
-
-func (c *StorageMinerStruct) MarketListDataTransfers(ctx context.Context) ([]api.DataTransferChannel, error) {
-	return c.Internal.MarketListDataTransfers(ctx)
-}
-
-func (c *StorageMinerStruct) MarketDataTransferUpdates(ctx context.Context) (<-chan api.DataTransferChannel, error) {
-	return c.Internal.MarketDataTransferUpdates(ctx)
-}
-
 func (c *StorageMinerStruct) DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error {
 	return c.Internal.DealsImportData(ctx, dealPropCid, file)
 }
@@ -1192,22 +1000,6 @@ func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) e
 	return c.Internal.StorageAddLocal(ctx, path)
 }
 
-func (c *StorageMinerStruct) PiecesListPieces(ctx context.Context) ([]cid.Cid, error) {
-	return c.Internal.PiecesListPieces(ctx)
-}
-
-func (c *StorageMinerStruct) PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error) {
-	return c.Internal.PiecesListCidInfos(ctx)
-}
-
-func (c *StorageMinerStruct) PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) {
-	return c.Internal.PiecesGetPieceInfo(ctx, pieceCid)
-}
-
-func (c *StorageMinerStruct) PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) {
-	return c.Internal.PiecesGetCIDInfo(ctx, payloadCid)
-}
-
 // WorkerStruct
 
 func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) {
@@ -1226,10 +1018,6 @@ func (w *WorkerStruct) Info(ctx context.Context) (storiface.WorkerInfo, error) {
 	return w.Internal.Info(ctx)
 }
 
-func (w *WorkerStruct) AddPiece(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) {
-	return w.Internal.AddPiece(ctx, sector, pieceSizes, newPieceSize, pieceData)
-}
-
 func (w *WorkerStruct) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) {
 	return w.Internal.SealPreCommit1(ctx, sector, ticket, pieces)
 }
@@ -1258,19 +1046,15 @@ func (w *WorkerStruct) Remove(ctx context.Context, sector abi.SectorID) error {
 	return w.Internal.Remove(ctx, sector)
 }
 
-func (w *WorkerStruct) MoveStorage(ctx context.Context, sector abi.SectorID, types stores.SectorFileType) error {
-	return w.Internal.MoveStorage(ctx, sector, types)
-}
-
-func (w *WorkerStruct) StorageAddLocal(ctx context.Context, path string) error {
-	return w.Internal.StorageAddLocal(ctx, path)
+func (w *WorkerStruct) MoveStorage(ctx context.Context, sector abi.SectorID) error {
+	return w.Internal.MoveStorage(ctx, sector)
 }
 
 func (w *WorkerStruct) UnsealPiece(ctx context.Context, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, c cid.Cid) error {
 	return w.Internal.UnsealPiece(ctx, id, index, size, randomness, c)
 }
 
-func (w *WorkerStruct) ReadPiece(ctx context.Context, writer io.Writer, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) {
+func (w *WorkerStruct) ReadPiece(ctx context.Context, writer io.Writer, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) error {
 	return w.Internal.ReadPiece(ctx, writer, id, index, size)
 }
 

api/cbor_gen.go

@@ -6,8 +6,8 @@ import (
 	"fmt"
 	"io"
 
-	abi "github.com/filecoin-project/go-state-types/abi"
-	paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/builtin/paych"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	xerrors "golang.org/x/xerrors"
 )
@@ -33,7 +33,7 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Channel"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Channel")); err != nil {
+	if _, err := io.WriteString(w, "Channel"); err != nil {
 		return err
 	}
 
@@ -41,20 +41,26 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	// t.WaitSentinel (cid.Cid) (struct)
-	if len("WaitSentinel") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"WaitSentinel\" was too long")
+	// t.ChannelMessage (cid.Cid) (struct)
+	if len("ChannelMessage") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"ChannelMessage\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("WaitSentinel"))); err != nil {
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ChannelMessage"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("WaitSentinel")); err != nil {
+	if _, err := io.WriteString(w, "ChannelMessage"); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteCidBuf(scratch, w, t.WaitSentinel); err != nil {
-		return xerrors.Errorf("failed to write cid field t.WaitSentinel: %w", err)
+	if t.ChannelMessage == nil {
+		if _, err := w.Write(cbg.CborNull); err != nil {
+			return err
+		}
+	} else {
+		if err := cbg.WriteCidBuf(scratch, w, *t.ChannelMessage); err != nil {
+			return xerrors.Errorf("failed to write cid field t.ChannelMessage: %w", err)
+		}
 	}
 
 	// t.Vouchers ([]*paych.SignedVoucher) (slice)
@@ -65,7 +71,7 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Vouchers"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Vouchers")); err != nil {
+	if _, err := io.WriteString(w, "Vouchers"); err != nil {
 		return err
 	}
 
@@ -127,17 +133,29 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 				}
 
 			}
-			// t.WaitSentinel (cid.Cid) (struct)
-		case "WaitSentinel":
+			// t.ChannelMessage (cid.Cid) (struct)
+		case "ChannelMessage":
 
 			{
 
-				c, err := cbg.ReadCid(br)
+				pb, err := br.PeekByte()
 				if err != nil {
-					return xerrors.Errorf("failed to read cid field t.WaitSentinel: %w", err)
+					return err
 				}
+				if pb == cbg.CborNull[0] {
+					var nbuf [1]byte
+					if _, err := br.Read(nbuf[:]); err != nil {
+						return err
+					}
+				} else {
 
-				t.WaitSentinel = c
+					c, err := cbg.ReadCid(br)
+					if err != nil {
+						return xerrors.Errorf("failed to read cid field t.ChannelMessage: %w", err)
+					}
+
+					t.ChannelMessage = &c
+				}
 
 			}
 			// t.Vouchers ([]*paych.SignedVoucher) (slice)
@@ -196,7 +214,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SectorID"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("SectorID")); err != nil {
+	if _, err := io.WriteString(w, "SectorID"); err != nil {
 		return err
 	}
 
@@ -204,7 +222,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	// t.Offset (abi.PaddedPieceSize) (uint64)
+	// t.Offset (uint64) (uint64)
 	if len("Offset") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"Offset\" was too long")
 	}
@@ -212,7 +230,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Offset"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Offset")); err != nil {
+	if _, err := io.WriteString(w, "Offset"); err != nil {
 		return err
 	}
 
@@ -228,7 +246,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Size"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Size")); err != nil {
+	if _, err := io.WriteString(w, "Size"); err != nil {
 		return err
 	}
 
@@ -287,7 +305,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 				t.SectorID = abi.SectorNumber(extra)
 
 			}
-			// t.Offset (abi.PaddedPieceSize) (uint64)
+			// t.Offset (uint64) (uint64)
 		case "Offset":
 
 			{
@@ -299,7 +317,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 				if maj != cbg.MajUnsignedInt {
 					return fmt.Errorf("wrong type for uint64 field")
 				}
-				t.Offset = abi.PaddedPieceSize(extra)
+				t.Offset = uint64(extra)
 
 			}
 			// t.Size (abi.UnpaddedPieceSize) (uint64)
@@ -344,7 +362,7 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Refs"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Refs")); err != nil {
+	if _, err := io.WriteString(w, "Refs"); err != nil {
 		return err
 	}
 
@@ -452,7 +470,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Value")); err != nil {
+	if _, err := io.WriteString(w, "Value"); err != nil {
 		return err
 	}
 
@@ -464,7 +482,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	if _, err := w.Write(t.Value[:]); err != nil {
+	if _, err := w.Write(t.Value); err != nil {
 		return err
 	}
 
@@ -476,7 +494,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Epoch")); err != nil {
+	if _, err := io.WriteString(w, "Epoch"); err != nil {
 		return err
 	}
 
@@ -539,12 +557,8 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
 			if maj != cbg.MajByteString {
 				return fmt.Errorf("expected byte array")
 			}
-
-			if extra > 0 {
-				t.Value = make([]uint8, extra)
-			}
-
-			if _, err := io.ReadFull(br, t.Value[:]); err != nil {
+			t.Value = make([]byte, extra)
+			if _, err := io.ReadFull(br, t.Value); err != nil {
 				return err
 			}
 			// t.Epoch (abi.ChainEpoch) (int64)
@@ -600,7 +614,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Value")); err != nil {
+	if _, err := io.WriteString(w, "Value"); err != nil {
 		return err
 	}
 
@@ -612,7 +626,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	if _, err := w.Write(t.Value[:]); err != nil {
+	if _, err := w.Write(t.Value); err != nil {
 		return err
 	}
 
@@ -624,7 +638,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Epoch")); err != nil {
+	if _, err := io.WriteString(w, "Epoch"); err != nil {
 		return err
 	}
 
@@ -687,12 +701,8 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
 			if maj != cbg.MajByteString {
 				return fmt.Errorf("expected byte array")
 			}
-
-			if extra > 0 {
-				t.Value = make([]uint8, extra)
-			}
-
-			if _, err := io.ReadFull(br, t.Value[:]); err != nil {
+			t.Value = make([]byte, extra)
+			if _, err := io.ReadFull(br, t.Value); err != nil {
 				return err
 			}
 			// t.Epoch (abi.ChainEpoch) (int64)

api/client/client.go

@@ -1,23 +1,18 @@
 package client
 
 import (
-	"context"
 	"net/http"
-	"net/url"
-	"path"
-	"time"
 
 	"github.com/filecoin-project/go-jsonrpc"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/apistruct"
-	"github.com/filecoin-project/lotus/lib/rpcenc"
 )
 
 // NewCommonRPC creates a new http jsonrpc client.
-func NewCommonRPC(ctx context.Context, addr string, requestHeader http.Header) (api.Common, jsonrpc.ClientCloser, error) {
+func NewCommonRPC(addr string, requestHeader http.Header) (api.Common, jsonrpc.ClientCloser, error) {
 	var res apistruct.CommonStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
 		[]interface{}{
 			&res.Internal,
 		},
@@ -28,9 +23,9 @@ func NewCommonRPC(ctx context.Context, addr string, requestHeader http.Header) (
 }
 
 // NewFullNodeRPC creates a new http jsonrpc client.
-func NewFullNodeRPC(ctx context.Context, addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
+func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
 	var res apistruct.FullNodeStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
 		[]interface{}{
 			&res.CommonStruct.Internal,
 			&res.Internal,
@@ -40,44 +35,26 @@ func NewFullNodeRPC(ctx context.Context, addr string, requestHeader http.Header)
 }
 
 // NewStorageMinerRPC creates a new http jsonrpc client for miner
-func NewStorageMinerRPC(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.StorageMiner, jsonrpc.ClientCloser, error) {
+func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMiner, jsonrpc.ClientCloser, error) {
 	var res apistruct.StorageMinerStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
 		[]interface{}{
 			&res.CommonStruct.Internal,
 			&res.Internal,
 		},
 		requestHeader,
-		opts...,
 	)
 
 	return &res, closer, err
 }
 
-func NewWorkerRPC(ctx context.Context, addr string, requestHeader http.Header) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
-	u, err := url.Parse(addr)
-	if err != nil {
-		return nil, nil, err
-	}
-	switch u.Scheme {
-	case "ws":
-		u.Scheme = "http"
-	case "wss":
-		u.Scheme = "https"
-	}
-	///rpc/v0 -> /rpc/streams/v0/push
-
-	u.Path = path.Join(u.Path, "../streams/v0/push")
-
+func NewWorkerRPC(addr string, requestHeader http.Header) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
 	var res apistruct.WorkerStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
 		[]interface{}{
 			&res.Internal,
 		},
 		requestHeader,
-		rpcenc.ReaderParamEncoder(u.String()),
-		jsonrpc.WithNoReconnect(),
-		jsonrpc.WithTimeout(30*time.Second),
 	)
 
 	return &res, closer, err

api/docgen/docgen.go

@@ -12,31 +12,22 @@ import (
 	"time"
 	"unicode"
 
-	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-filestore"
-	metrics "github.com/libp2p/go-libp2p-core/metrics"
-	"github.com/libp2p/go-libp2p-core/network"
-	"github.com/libp2p/go-libp2p-core/peer"
-	protocol "github.com/libp2p/go-libp2p-core/protocol"
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
-	"github.com/multiformats/go-multiaddr"
-
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
-	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-jsonrpc/auth"
-	"github.com/filecoin-project/go-multistore"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/crypto"
-	"github.com/filecoin-project/go-state-types/exitcode"
-
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/apistruct"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
+	"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-filestore"
+	"github.com/libp2p/go-libp2p-core/network"
+	peer "github.com/libp2p/go-libp2p-peer"
+	"github.com/multiformats/go-multiaddr"
 )
 
 var ExampleValues = map[reflect.Type]interface{}{
@@ -75,12 +66,11 @@ func init() {
 
 	ExampleValues[reflect.TypeOf(addr)] = addr
 
-	pid, err := peer.Decode("12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf")
+	pid, err := peer.IDB58Decode("12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf")
 	if err != nil {
 		panic(err)
 	}
 	addExample(pid)
-	addExample(&pid)
 
 	addExample(bitfield.NewFromSet([]uint64{5}))
 	addExample(abi.RegisteredSealProof_StackedDrg32GiBV1)
@@ -108,12 +98,6 @@ func init() {
 	addExample(build.APIVersion)
 	addExample(api.PCHInbound)
 	addExample(time.Minute)
-	addExample(datatransfer.TransferID(3))
-	addExample(datatransfer.Ongoing)
-	addExample(multistore.StoreID(50))
-	addExample(retrievalmarket.ClientEventDealAccepted)
-	addExample(retrievalmarket.DealStatusNew)
-	addExample(network.ReachabilityPublic)
 	addExample(&types.ExecutionTrace{
 		Msg:    exampleValue(reflect.TypeOf(&types.Message{}), nil).(*types.Message),
 		MsgRct: exampleValue(reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt),
@@ -127,30 +111,6 @@ func init() {
 	addExample(map[string]api.MarketBalance{
 		"t026363": exampleValue(reflect.TypeOf(api.MarketBalance{}), nil).(api.MarketBalance),
 	})
-	addExample(map[string]*pubsub.TopicScoreSnapshot{
-		"/blocks": {
-			TimeInMesh:               time.Minute,
-			FirstMessageDeliveries:   122,
-			MeshMessageDeliveries:    1234,
-			InvalidMessageDeliveries: 3,
-		},
-	})
-	addExample(map[string]metrics.Stats{
-		"12D3KooWSXmXLJmBR1M7i9RW9GQPNUhZSzXKzxDHWtAgNuJAbyEJ": {
-			RateIn:   100,
-			RateOut:  50,
-			TotalIn:  174000,
-			TotalOut: 12500,
-		},
-	})
-	addExample(map[protocol.ID]metrics.Stats{
-		"/fil/hello/1.0.0": {
-			RateIn:   100,
-			RateOut:  50,
-			TotalIn:  174000,
-			TotalOut: 12500,
-		},
-	})
 
 	maddr, err := multiaddr.NewMultiaddr("/ip4/52.36.61.156/tcp/1347/p2p/12D3KooWFETiESTf1v4PGUvtnxMAcEFMzLZbJGg4tjWfGEimYior")
 	if err != nil {

api/test/blockminer.go

@@ -1,56 +0,0 @@
-package test
-
-import (
-	"context"
-	"fmt"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/miner"
-)
-
-type BlockMiner struct {
-	ctx       context.Context
-	t         *testing.T
-	miner     TestStorageNode
-	blocktime time.Duration
-	mine      int64
-	nulls     int64
-	done      chan struct{}
-}
-
-func NewBlockMiner(ctx context.Context, t *testing.T, miner TestStorageNode, blocktime time.Duration) *BlockMiner {
-	return &BlockMiner{
-		ctx:       ctx,
-		t:         t,
-		miner:     miner,
-		blocktime: blocktime,
-		mine:      int64(1),
-		done:      make(chan struct{}),
-	}
-}
-
-func (bm *BlockMiner) MineBlocks() {
-	time.Sleep(time.Second)
-	go func() {
-		defer close(bm.done)
-		for atomic.LoadInt64(&bm.mine) == 1 {
-			time.Sleep(bm.blocktime)
-			nulls := atomic.SwapInt64(&bm.nulls, 0)
-			if err := bm.miner.MineOne(bm.ctx, miner.MineReq{
-				InjectNulls: abi.ChainEpoch(nulls),
-				Done:        func(bool, abi.ChainEpoch, error) {},
-			}); err != nil {
-				bm.t.Error(err)
-			}
-		}
-	}()
-}
-
-func (bm *BlockMiner) Stop() {
-	atomic.AddInt64(&bm.mine, -1)
-	fmt.Println("shutting down mining")
-	<-bm.done
-}

api/test/ccupgrade.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
-	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/node/impl"
@@ -20,7 +20,7 @@ func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) {
 	_ = os.Setenv("BELLMAN_NO_GPU", "1")
 
 	ctx := context.Background()
-	n, sn := b(t, 1, OneMiner)
+	n, sn := b(t, 1, oneMiner)
 	client := n[0].FullNode.(*impl.FullNodeAPI)
 	miner := sn[0]
 
@@ -40,7 +40,7 @@ func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) {
 		defer close(done)
 		for atomic.LoadInt64(&mine) == 1 {
 			time.Sleep(blocktime)
-			if err := sn[0].MineOne(ctx, MineNext); err != nil {
+			if err := sn[0].MineOne(ctx, func(bool, error) {}); err != nil {
 				t.Error(err)
 			}
 		}
@@ -54,7 +54,7 @@ func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) {
 	CC := abi.SectorNumber(GenesisPreseals + 1)
 	Upgraded := CC + 1
 
-	pledgeSectors(t, ctx, miner, 1, 0, nil)
+	pledgeSectors(t, ctx, miner, 1)
 
 	sl, err := miner.SectorsList(ctx)
 	if err != nil {

api/test/deals.go

@@ -20,11 +20,9 @@ import (
 	"github.com/ipld/go-car"
 
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
-	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
-	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
-	"github.com/filecoin-project/lotus/miner"
+	sealing "github.com/filecoin-project/storage-fsm"
 	dag "github.com/ipfs/go-merkledag"
 	dstest "github.com/ipfs/go-merkledag/test"
 	unixfile "github.com/ipfs/go-unixfs/file"
@@ -34,11 +32,6 @@ import (
 	ipld "github.com/ipfs/go-ipld-format"
 )
 
-var MineNext = miner.MineReq{
-	InjectNulls: 0,
-	Done:        func(bool, abi.ChainEpoch, error) {},
-}
-
 func init() {
 	logging.SetAllLoggers(logging.LevelInfo)
 	build.InsecurePoStValidation = true
@@ -48,7 +41,7 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport
 	_ = os.Setenv("BELLMAN_NO_GPU", "1")
 
 	ctx := context.Background()
-	n, sn := b(t, 1, OneMiner)
+	n, sn := b(t, 1, oneMiner)
 	client := n[0].FullNode.(*impl.FullNodeAPI)
 	miner := sn[0]
 
@@ -68,7 +61,7 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport
 		defer close(done)
 		for atomic.LoadInt64(&mine) == 1 {
 			time.Sleep(blocktime)
-			if err := sn[0].MineOne(ctx, MineNext); err != nil {
+			if err := sn[0].MineOne(ctx, func(bool, error) {}); err != nil {
 				t.Error(err)
 			}
 		}
@@ -85,7 +78,7 @@ func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
 	_ = os.Setenv("BELLMAN_NO_GPU", "1")
 
 	ctx := context.Background()
-	n, sn := b(t, 1, OneMiner)
+	n, sn := b(t, 1, oneMiner)
 	client := n[0].FullNode.(*impl.FullNodeAPI)
 	miner := sn[0]
 
@@ -106,7 +99,7 @@ func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
 		defer close(done)
 		for atomic.LoadInt64(&mine) == 1 {
 			time.Sleep(blocktime)
-			if err := sn[0].MineOne(ctx, MineNext); err != nil {
+			if err := sn[0].MineOne(ctx, func(bool, error) {}); err != nil {
 				t.Error(err)
 			}
 		}
@@ -136,144 +129,13 @@ func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNod
 
 	// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
 	time.Sleep(time.Second)
-	waitDealSealed(t, ctx, miner, client, deal, false)
+	waitDealSealed(t, ctx, miner, client, deal)
 
 	// Retrieval
 	info, err := client.ClientGetDealInfo(ctx, *deal)
 	require.NoError(t, err)
 
-	testRetrieval(t, ctx, client, fcid, &info.PieceCID, carExport, data)
-}
-
-func TestFastRetrievalDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
-	_ = os.Setenv("BELLMAN_NO_GPU", "1")
-
-	ctx := context.Background()
-	n, sn := b(t, 1, OneMiner)
-	client := n[0].FullNode.(*impl.FullNodeAPI)
-	miner := sn[0]
-
-	addrinfo, err := client.NetAddrsListen(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err := miner.NetConnect(ctx, addrinfo); err != nil {
-		t.Fatal(err)
-	}
-	time.Sleep(time.Second)
-
-	mine := int64(1)
-	done := make(chan struct{})
-	go func() {
-		defer close(done)
-		for atomic.LoadInt64(&mine) == 1 {
-			time.Sleep(blocktime)
-			if err := sn[0].MineOne(ctx, MineNext); err != nil {
-				t.Error(err)
-			}
-		}
-	}()
-
-	data := make([]byte, 1600)
-	rand.New(rand.NewSource(int64(8))).Read(data)
-
-	r := bytes.NewReader(data)
-	fcid, err := client.ClientImportLocal(ctx, r)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	fmt.Println("FILE CID: ", fcid)
-
-	deal := startDeal(t, ctx, miner, client, fcid, true)
-
-	waitDealPublished(t, ctx, miner, deal)
-	fmt.Println("deal published, retrieving")
-	// Retrieval
-	info, err := client.ClientGetDealInfo(ctx, *deal)
-	require.NoError(t, err)
-
-	testRetrieval(t, ctx, client, fcid, &info.PieceCID, false, data)
-	atomic.AddInt64(&mine, -1)
-	fmt.Println("shutting down mining")
-	<-done
-}
-
-func TestSenondDealRetrieval(t *testing.T, b APIBuilder, blocktime time.Duration) {
-	_ = os.Setenv("BELLMAN_NO_GPU", "1")
-
-	ctx := context.Background()
-	n, sn := b(t, 1, OneMiner)
-	client := n[0].FullNode.(*impl.FullNodeAPI)
-	miner := sn[0]
-
-	addrinfo, err := client.NetAddrsListen(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err := miner.NetConnect(ctx, addrinfo); err != nil {
-		t.Fatal(err)
-	}
-	time.Sleep(time.Second)
-
-	mine := int64(1)
-	done := make(chan struct{})
-
-	go func() {
-		defer close(done)
-		for atomic.LoadInt64(&mine) == 1 {
-			time.Sleep(blocktime)
-			if err := sn[0].MineOne(ctx, MineNext); err != nil {
-				t.Error(err)
-			}
-		}
-	}()
-
-	{
-		data1 := make([]byte, 800)
-		rand.New(rand.NewSource(int64(3))).Read(data1)
-		r := bytes.NewReader(data1)
-
-		fcid1, err := client.ClientImportLocal(ctx, r)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		data2 := make([]byte, 800)
-		rand.New(rand.NewSource(int64(9))).Read(data2)
-		r2 := bytes.NewReader(data2)
-
-		fcid2, err := client.ClientImportLocal(ctx, r2)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		deal1 := startDeal(t, ctx, miner, client, fcid1, true)
-
-		// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
-		time.Sleep(time.Second)
-		waitDealSealed(t, ctx, miner, client, deal1, true)
-
-		deal2 := startDeal(t, ctx, miner, client, fcid2, true)
-
-		time.Sleep(time.Second)
-		waitDealSealed(t, ctx, miner, client, deal2, false)
-
-		// Retrieval
-		info, err := client.ClientGetDealInfo(ctx, *deal2)
-		require.NoError(t, err)
-
-		rf, _ := miner.SectorsRefs(ctx)
-		fmt.Printf("refs: %+v\n", rf)
-
-		testRetrieval(t, ctx, client, fcid2, &info.PieceCID, false, data2)
-	}
-
-	atomic.AddInt64(&mine, -1)
-	fmt.Println("shutting down mining")
-	<-done
+	testRetrieval(t, ctx, err, client, fcid, &info.PieceCID, carExport, data)
 }
 
 func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, fcid cid.Cid, fastRet bool) *cid.Cid {
@@ -287,14 +149,11 @@ func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client
 		t.Fatal(err)
 	}
 	deal, err := client.ClientStartDeal(ctx, &api.StartDealParams{
-		Data: &storagemarket.DataRef{
-			TransferType: storagemarket.TTGraphsync,
-			Root:         fcid,
-		},
+		Data:              &storagemarket.DataRef{Root: fcid},
 		Wallet:            addr,
 		Miner:             maddr,
 		EpochPrice:        types.NewInt(1000000),
-		MinBlocksDuration: uint64(build.MinDealDuration),
+		MinBlocksDuration: 100,
 		FastRetrieval:     fastRet,
 	})
 	if err != nil {
@@ -303,7 +162,7 @@ func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client
 	return deal
 }
 
-func waitDealSealed(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, deal *cid.Cid, noseal bool) {
+func waitDealSealed(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, deal *cid.Cid) {
 loop:
 	for {
 		di, err := client.ClientGetDealInfo(ctx, *deal)
@@ -312,9 +171,6 @@ loop:
 		}
 		switch di.State {
 		case storagemarket.StorageDealSealing:
-			if noseal {
-				return
-			}
 			startSealingWaiting(t, ctx, miner)
 		case storagemarket.StorageDealProposalRejected:
 			t.Fatal("deal rejected")
@@ -331,42 +187,12 @@ loop:
 	}
 }
 
-func waitDealPublished(t *testing.T, ctx context.Context, miner TestStorageNode, deal *cid.Cid) {
-	subCtx, cancel := context.WithCancel(ctx)
-	defer cancel()
-	updates, err := miner.MarketGetDealUpdates(subCtx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	for {
-		select {
-		case <-ctx.Done():
-			t.Fatal("context timeout")
-		case di := <-updates:
-			if deal.Equals(di.ProposalCid) {
-				switch di.State {
-				case storagemarket.StorageDealProposalRejected:
-					t.Fatal("deal rejected")
-				case storagemarket.StorageDealFailing:
-					t.Fatal("deal failed")
-				case storagemarket.StorageDealError:
-					t.Fatal("deal errored", di.Message)
-				case storagemarket.StorageDealFinalizing, storagemarket.StorageDealSealing, storagemarket.StorageDealActive:
-					fmt.Println("COMPLETE", di)
-					return
-				}
-				fmt.Println("Deal state: ", storagemarket.DealStates[di.State])
-			}
-		}
-	}
-}
-
 func startSealingWaiting(t *testing.T, ctx context.Context, miner TestStorageNode) {
 	snums, err := miner.SectorsList(ctx)
 	require.NoError(t, err)
 
 	for _, snum := range snums {
-		si, err := miner.SectorsStatus(ctx, snum, false)
+		si, err := miner.SectorsStatus(ctx, snum)
 		require.NoError(t, err)
 
 		t.Logf("Sector state: %s", si.State)
@@ -376,7 +202,7 @@ func startSealingWaiting(t *testing.T, ctx context.Context, miner TestStorageNod
 	}
 }
 
-func testRetrieval(t *testing.T, ctx context.Context, client *impl.FullNodeAPI, fcid cid.Cid, piece *cid.Cid, carExport bool, data []byte) {
+func testRetrieval(t *testing.T, ctx context.Context, err error, client *impl.FullNodeAPI, fcid cid.Cid, piece *cid.Cid, carExport bool, data []byte) {
 	offers, err := client.ClientFindData(ctx, fcid, piece)
 	if err != nil {
 		t.Fatal(err)
@@ -401,11 +227,9 @@ func testRetrieval(t *testing.T, ctx context.Context, client *impl.FullNodeAPI,
 		Path:  filepath.Join(rpath, "ret"),
 		IsCAR: carExport,
 	}
-	updates, err := client.ClientRetrieveWithEvents(ctx, offers[0].Order(caddr), ref)
-	for update := range updates {
-		if update.Err != "" {
-			t.Fatalf("%v", err)
-		}
+	err = client.ClientRetrieve(ctx, offers[0].Order(caddr), ref)
+	if err != nil {
+		t.Fatalf("%+v", err)
 	}
 
 	rdata, err := ioutil.ReadFile(filepath.Join(rpath, "ret"))

api/test/mining.go

@@ -12,39 +12,36 @@ import (
 
 	logging "github.com/ipfs/go-log/v2"
 
-	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/miner"
 	"github.com/filecoin-project/lotus/node/impl"
 )
 
-//nolint:deadcode,varcheck
 var log = logging.Logger("apitest")
 
 func (ts *testSuite) testMining(t *testing.T) {
 	ctx := context.Background()
-	apis, sn := ts.makeNodes(t, 1, OneMiner)
+	apis, sn := ts.makeNodes(t, 1, oneMiner)
 	api := apis[0]
 
-	newHeads, err := api.ChainNotify(ctx)
-	require.NoError(t, err)
-	initHead := (<-newHeads)[0]
-	baseHeight := initHead.Val.Height()
-
 	h1, err := api.ChainHead(ctx)
 	require.NoError(t, err)
-	require.Equal(t, int64(h1.Height()), int64(baseHeight))
+	require.Equal(t, abi.ChainEpoch(0), h1.Height())
 
-	MineUntilBlock(ctx, t, apis[0], sn[0], nil)
+	newHeads, err := api.ChainNotify(ctx)
+	require.NoError(t, err)
+	<-newHeads
+
+	err = sn[0].MineOne(ctx, func(bool, error) {})
 	require.NoError(t, err)
 
 	<-newHeads
 
 	h2, err := api.ChainHead(ctx)
 	require.NoError(t, err)
-	require.Greater(t, int64(h2.Height()), int64(h1.Height()))
+	require.Equal(t, abi.ChainEpoch(1), h2.Height())
 }
 
 func (ts *testSuite) testMiningReal(t *testing.T) {
@@ -54,37 +51,34 @@ func (ts *testSuite) testMiningReal(t *testing.T) {
 	}()
 
 	ctx := context.Background()
-	apis, sn := ts.makeNodes(t, 1, OneMiner)
+	apis, sn := ts.makeNodes(t, 1, oneMiner)
 	api := apis[0]
 
-	newHeads, err := api.ChainNotify(ctx)
-	require.NoError(t, err)
-	initHead := (<-newHeads)[0]
-	if initHead.Val.Height() != 2 {
-		<-newHeads
-	}
-
 	h1, err := api.ChainHead(ctx)
 	require.NoError(t, err)
-	require.Equal(t, abi.ChainEpoch(2), h1.Height())
+	require.Equal(t, abi.ChainEpoch(0), h1.Height())
 
-	MineUntilBlock(ctx, t, apis[0], sn[0], nil)
+	newHeads, err := api.ChainNotify(ctx)
+	require.NoError(t, err)
+	<-newHeads
+
+	err = sn[0].MineOne(ctx, func(bool, error) {})
 	require.NoError(t, err)
 
 	<-newHeads
 
 	h2, err := api.ChainHead(ctx)
 	require.NoError(t, err)
-	require.Equal(t, abi.ChainEpoch(3), h2.Height())
+	require.Equal(t, abi.ChainEpoch(1), h2.Height())
 
-	MineUntilBlock(ctx, t, apis[0], sn[0], nil)
+	err = sn[0].MineOne(ctx, func(bool, error) {})
 	require.NoError(t, err)
 
 	<-newHeads
 
 	h2, err = api.ChainHead(ctx)
 	require.NoError(t, err)
-	require.Equal(t, abi.ChainEpoch(4), h2.Height())
+	require.Equal(t, abi.ChainEpoch(2), h2.Height())
 }
 
 func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExport bool) {
@@ -131,30 +125,28 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo
 	done := make(chan struct{})
 	minedTwo := make(chan struct{})
 
-	m2addr, err := sn[1].ActorAddress(context.TODO())
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	go func() {
+		doneMinedTwo := false
 		defer close(done)
 
-		complChan := minedTwo
+		prevExpect := 0
 		for atomic.LoadInt32(&mine) != 0 {
-			wait := make(chan int)
-			mdone := func(mined bool, _ abi.ChainEpoch, err error) {
-				n := 0
-				if mined {
-					n = 1
-				}
-				wait <- n
+			wait := make(chan int, 2)
+			mdone := func(mined bool, err error) {
+				go func() {
+					n := 0
+					if mined {
+						n = 1
+					}
+					wait <- n
+				}()
 			}
 
-			if err := sn[0].MineOne(ctx, miner.MineReq{Done: mdone}); err != nil {
+			if err := sn[0].MineOne(ctx, mdone); err != nil {
 				t.Error(err)
 			}
 
-			if err := sn[1].MineOne(ctx, miner.MineReq{Done: mdone}); err != nil {
+			if err := sn[1].MineOne(ctx, mdone); err != nil {
 				t.Error(err)
 			}
 
@@ -167,28 +159,33 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo
 				continue
 			}
 
-			var nodeOneMined bool
-			for _, node := range sn {
-				mb, err := node.MiningBase(ctx)
-				if err != nil {
-					t.Error(err)
-					return
-				}
-
-				for _, b := range mb.Blocks() {
-					if b.Miner == m2addr {
-						nodeOneMined = true
-						break
+			for {
+				n := 0
+				for i, node := range sn {
+					mb, err := node.MiningBase(ctx)
+					if err != nil {
+						t.Error(err)
+						return
 					}
+
+					if len(mb.Cids()) != expect {
+						log.Warnf("node %d mining base not complete (%d, want %d)", i, len(mb.Cids()), expect)
+						continue
+					}
+					n++
 				}
-
+				if n == len(sn) {
+					break
+				}
+				time.Sleep(blocktime)
 			}
 
-			if nodeOneMined && complChan != nil {
-				close(complChan)
-				complChan = nil
+			if prevExpect == 2 && expect == 2 && !doneMinedTwo {
+				close(minedTwo)
+				doneMinedTwo = true
 			}
 
+			prevExpect = expect
 		}
 	}()
 
@@ -197,7 +194,7 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo
 	// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
 	time.Sleep(time.Second)
 
-	waitDealSealed(t, ctx, provider, client, deal, false)
+	waitDealSealed(t, ctx, provider, client, deal)
 
 	<-minedTwo
 

api/test/paych.go

@@ -1,259 +0,0 @@
-package test
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/filecoin-project/specs-actors/actors/builtin"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/specs-actors/actors/builtin/paych"
-	"github.com/ipfs/go-cid"
-
-	"github.com/filecoin-project/go-address"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/events"
-	"github.com/filecoin-project/lotus/chain/events/state"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/chain/wallet"
-)
-
-func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) {
-	_ = os.Setenv("BELLMAN_NO_GPU", "1")
-
-	ctx := context.Background()
-	n, sn := b(t, 2, OneMiner)
-
-	paymentCreator := n[0]
-	paymentReceiver := n[1]
-	miner := sn[0]
-
-	// get everyone connected
-	addrs, err := paymentCreator.NetAddrsListen(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err := paymentReceiver.NetConnect(ctx, addrs); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := miner.NetConnect(ctx, addrs); err != nil {
-		t.Fatal(err)
-	}
-
-	// start mining blocks
-	bm := NewBlockMiner(ctx, t, miner, blocktime)
-	bm.MineBlocks()
-
-	// send some funds to register the receiver
-	receiverAddr, err := paymentReceiver.WalletNew(ctx, wallet.ActSigType("secp256k1"))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	SendFunds(ctx, t, paymentCreator, receiverAddr, abi.NewTokenAmount(1e18))
-
-	// setup the payment channel
-	createrAddr, err := paymentCreator.WalletDefaultAddress(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	channelAmt := int64(100000)
-	channelInfo, err := paymentCreator.PaychGet(ctx, createrAddr, receiverAddr, abi.NewTokenAmount(channelAmt))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	channel, err := paymentCreator.PaychGetWaitReady(ctx, channelInfo.WaitSentinel)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// allocate three lanes
-	var lanes []uint64
-	for i := 0; i < 3; i++ {
-		lane, err := paymentCreator.PaychAllocateLane(ctx, channel)
-		if err != nil {
-			t.Fatal(err)
-		}
-		lanes = append(lanes, lane)
-	}
-
-	// Make two vouchers each for each lane, then save on the other side
-	// Note that the voucher with a value of 2000 has a higher nonce, so it
-	// supersedes the voucher with a value of 1000
-	for _, lane := range lanes {
-		vouch1, err := paymentCreator.PaychVoucherCreate(ctx, channel, abi.NewTokenAmount(1000), lane)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if vouch1.Voucher == nil {
-			t.Fatal(fmt.Errorf("Not enough funds to create voucher: missing %d", vouch1.Shortfall))
-		}
-		vouch2, err := paymentCreator.PaychVoucherCreate(ctx, channel, abi.NewTokenAmount(2000), lane)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if vouch2.Voucher == nil {
-			t.Fatal(fmt.Errorf("Not enough funds to create voucher: missing %d", vouch2.Shortfall))
-		}
-		delta1, err := paymentReceiver.PaychVoucherAdd(ctx, channel, vouch1.Voucher, nil, abi.NewTokenAmount(1000))
-		if err != nil {
-			t.Fatal(err)
-		}
-		if !delta1.Equals(abi.NewTokenAmount(1000)) {
-			t.Fatal("voucher didn't have the right amount")
-		}
-		delta2, err := paymentReceiver.PaychVoucherAdd(ctx, channel, vouch2.Voucher, nil, abi.NewTokenAmount(1000))
-		if err != nil {
-			t.Fatal(err)
-		}
-		if !delta2.Equals(abi.NewTokenAmount(1000)) {
-			t.Fatal("voucher didn't have the right amount")
-		}
-	}
-
-	// settle the payment channel
-	settleMsgCid, err := paymentCreator.PaychSettle(ctx, channel)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	res := waitForMessage(ctx, t, paymentCreator, settleMsgCid, time.Second*10, "settle")
-	if res.Receipt.ExitCode != 0 {
-		t.Fatal("Unable to settle payment channel")
-	}
-
-	// wait for the receiver to submit their vouchers
-	ev := events.NewEvents(ctx, paymentCreator)
-	preds := state.NewStatePredicates(paymentCreator)
-	finished := make(chan struct{})
-	err = ev.StateChanged(func(ts *types.TipSet) (done bool, more bool, err error) {
-		act, err := paymentCreator.StateReadState(ctx, channel, ts.Key())
-		if err != nil {
-			return false, false, err
-		}
-		state := act.State.(paych.State)
-		if state.ToSend.GreaterThanEqual(abi.NewTokenAmount(6000)) {
-			return true, false, nil
-		}
-		return false, true, nil
-	}, func(oldTs, newTs *types.TipSet, states events.StateChange, curH abi.ChainEpoch) (more bool, err error) {
-		toSendChange := states.(*state.PayChToSendChange)
-		if toSendChange.NewToSend.GreaterThanEqual(abi.NewTokenAmount(6000)) {
-			close(finished)
-			return false, nil
-		}
-		return true, nil
-	}, func(ctx context.Context, ts *types.TipSet) error {
-		return nil
-	}, int(build.MessageConfidence)+1, build.SealRandomnessLookbackLimit, func(oldTs, newTs *types.TipSet) (bool, events.StateChange, error) {
-		return preds.OnPaymentChannelActorChanged(channel, preds.OnToSendAmountChanges())(ctx, oldTs.Key(), newTs.Key())
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	select {
-	case <-finished:
-	case <-time.After(time.Second):
-		t.Fatal("Timed out waiting for receiver to submit vouchers")
-	}
-
-	// wait for the settlement period to pass before collecting
-	waitForBlocks(ctx, t, bm, paymentReceiver, receiverAddr, paych.SettleDelay)
-
-	creatorPreCollectBalance, err := paymentCreator.WalletBalance(ctx, createrAddr)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// collect funds (from receiver, though either party can do it)
-	collectMsg, err := paymentReceiver.PaychCollect(ctx, channel)
-	if err != nil {
-		t.Fatal(err)
-	}
-	res, err = paymentReceiver.StateWaitMsg(ctx, collectMsg, 3)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if res.Receipt.ExitCode != 0 {
-		t.Fatal("unable to collect on payment channel")
-	}
-
-	// Finally, check the balance for the creator
-	currentCreatorBalance, err := paymentCreator.WalletBalance(ctx, createrAddr)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// The highest nonce voucher that the creator sent on each lane is 2000
-	totalVouchers := int64(len(lanes) * 2000)
-
-	// When receiver submits the tokens to the chain, creator should get a
-	// refund on the remaining balance, which is
-	// channel amount - total voucher value
-	expectedRefund := channelAmt - totalVouchers
-	delta := big.Sub(currentCreatorBalance, creatorPreCollectBalance)
-	if !delta.Equals(abi.NewTokenAmount(expectedRefund)) {
-		t.Fatalf("did not send correct funds from creator: expected %d, got %d", expectedRefund, delta)
-	}
-
-	// shut down mining
-	bm.Stop()
-}
-
-func waitForBlocks(ctx context.Context, t *testing.T, bm *BlockMiner, paymentReceiver TestNode, receiverAddr address.Address, count int) {
-	// We need to add null blocks in batches, if we add too many the chain can't sync
-	batchSize := 60
-	for i := 0; i < count; i += batchSize {
-		size := batchSize
-		if i > count {
-			size = count - i
-		}
-
-		// Add a batch of null blocks
-		atomic.StoreInt64(&bm.nulls, int64(size-1))
-
-		// Add a real block
-		m, err := paymentReceiver.MpoolPushMessage(ctx, &types.Message{
-			To:    builtin.BurntFundsActorAddr,
-			From:  receiverAddr,
-			Value: types.NewInt(0),
-		}, nil)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		_, err = paymentReceiver.StateWaitMsg(ctx, m.Cid(), 1)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
-func waitForMessage(ctx context.Context, t *testing.T, paymentCreator TestNode, msgCid cid.Cid, duration time.Duration, desc string) *api.MsgLookup {
-	ctx, cancel := context.WithTimeout(ctx, duration)
-	defer cancel()
-
-	fmt.Println("Waiting for", desc)
-	res, err := paymentCreator.StateWaitMsg(ctx, msgCid, 1)
-	if err != nil {
-		fmt.Println("Error waiting for", desc, err)
-		t.Fatal(err)
-	}
-	if res.Receipt.ExitCode != 0 {
-		t.Fatalf("did not successfully send %s", desc)
-	}
-	fmt.Println("Confirmed", desc)
-	return res
-}

api/test/test.go

@@ -4,30 +4,21 @@ import (
 	"context"
 	"testing"
 
-	"github.com/multiformats/go-multiaddr"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/miner"
 )
 
 type TestNode struct {
 	api.FullNode
-	// ListenAddr is the address on which an API server is listening, if an
-	// API server is created for this Node
-	ListenAddr multiaddr.Multiaddr
 }
 
 type TestStorageNode struct {
 	api.StorageMiner
-	// ListenAddr is the address on which an API server is listening, if an
-	// API server is created for this Node
-	ListenAddr multiaddr.Multiaddr
 
-	MineOne func(context.Context, miner.MineReq) error
+	MineOne func(context.Context, func(bool, error)) error
 }
 
 var PresealGenesis = -1
@@ -62,11 +53,11 @@ func TestApis(t *testing.T, b APIBuilder) {
 	t.Run("testMiningReal", ts.testMiningReal)
 }
 
-var OneMiner = []StorageMiner{{Full: 0, Preseal: PresealGenesis}}
+var oneMiner = []StorageMiner{{Full: 0, Preseal: PresealGenesis}}
 
 func (ts *testSuite) testVersion(t *testing.T) {
 	ctx := context.Background()
-	apis, _ := ts.makeNodes(t, 1, OneMiner)
+	apis, _ := ts.makeNodes(t, 1, oneMiner)
 	api := apis[0]
 
 	v, err := api.Version(ctx)
@@ -78,7 +69,7 @@ func (ts *testSuite) testVersion(t *testing.T) {
 
 func (ts *testSuite) testID(t *testing.T) {
 	ctx := context.Background()
-	apis, _ := ts.makeNodes(t, 1, OneMiner)
+	apis, _ := ts.makeNodes(t, 1, oneMiner)
 	api := apis[0]
 
 	id, err := api.ID(ctx)
@@ -90,7 +81,7 @@ func (ts *testSuite) testID(t *testing.T) {
 
 func (ts *testSuite) testConnectTwo(t *testing.T) {
 	ctx := context.Background()
-	apis, _ := ts.makeNodes(t, 2, OneMiner)
+	apis, _ := ts.makeNodes(t, 2, oneMiner)
 
 	p, err := apis[0].NetPeers(ctx)
 	if err != nil {

api/test/util.go

@@ -1,86 +0,0 @@
-package test
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"github.com/filecoin-project/go-state-types/abi"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/miner"
-)
-
-func SendFunds(ctx context.Context, t *testing.T, sender TestNode, addr address.Address, amount abi.TokenAmount) {
-	senderAddr, err := sender.WalletDefaultAddress(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	msg := &types.Message{
-		From:  senderAddr,
-		To:    addr,
-		Value: amount,
-	}
-
-	sm, err := sender.MpoolPushMessage(ctx, msg, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	res, err := sender.StateWaitMsg(ctx, sm.Cid(), 1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if res.Receipt.ExitCode != 0 {
-		t.Fatal("did not successfully send money")
-	}
-}
-
-func MineUntilBlock(ctx context.Context, t *testing.T, fn TestNode, sn TestStorageNode, cb func(abi.ChainEpoch)) {
-	for i := 0; i < 1000; i++ {
-		var success bool
-		var err error
-		var epoch abi.ChainEpoch
-		wait := make(chan struct{})
-		mineErr := sn.MineOne(ctx, miner.MineReq{
-			Done: func(win bool, ep abi.ChainEpoch, e error) {
-				success = win
-				err = e
-				epoch = ep
-				wait <- struct{}{}
-			},
-		})
-		if mineErr != nil {
-			t.Fatal(mineErr)
-		}
-		<-wait
-		if err != nil {
-			t.Fatal(err)
-		}
-		if success {
-			// Wait until it shows up on the given full nodes ChainHead
-			nloops := 50
-			for i := 0; i < nloops; i++ {
-				ts, err := fn.ChainHead(ctx)
-				if err != nil {
-					t.Fatal(err)
-				}
-				if ts.Height() == epoch {
-					break
-				}
-				if i == nloops-1 {
-					t.Fatal("block never managed to sync to node")
-				}
-				time.Sleep(time.Millisecond * 10)
-			}
-
-			if cb != nil {
-				cb(epoch)
-			}
-			return
-		}
-		t.Log("did not mine block, trying again", i)
-	}
-	t.Fatal("failed to mine 1000 times in a row...")
-}

api/test/window_post.go

@@ -3,6 +3,8 @@ package test
 import (
 	"context"
 	"fmt"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
 
 	"os"
 	"strings"
@@ -11,29 +13,19 @@ import (
 
 	"github.com/stretchr/testify/require"
 
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/extern/sector-storage/mock"
-	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	miner2 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
+	sealing "github.com/filecoin-project/storage-fsm"
 
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
-	bminer "github.com/filecoin-project/lotus/miner"
 	"github.com/filecoin-project/lotus/node/impl"
 )
 
-func init() {
-	err := os.Setenv("BELLMAN_NO_GPU", "1")
-	if err != nil {
-		panic(fmt.Sprintf("failed to set BELLMAN_NO_GPU env variable: %s", err))
-	}
-}
-
 func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) {
+	os.Setenv("BELLMAN_NO_GPU", "1")
+
 	ctx := context.Background()
-	n, sn := b(t, 1, OneMiner)
+	n, sn := b(t, 1, oneMiner)
 	client := n[0].FullNode.(*impl.FullNodeAPI)
 	miner := sn[0]
 
@@ -53,34 +45,29 @@ func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSect
 		defer close(done)
 		for mine {
 			build.Clock.Sleep(blocktime)
-			if err := sn[0].MineOne(ctx, bminer.MineReq{Done: func(bool, abi.ChainEpoch, error) {
-
-			}}); err != nil {
+			if err := sn[0].MineOne(ctx, func(bool, error) {}); err != nil {
 				t.Error(err)
 			}
 		}
 	}()
 
-	pledgeSectors(t, ctx, miner, nSectors, 0, nil)
+	pledgeSectors(t, ctx, miner, nSectors)
 
 	mine = false
 	<-done
 }
 
-func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n, existing int, blockNotif <-chan struct{}) {
+func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n int) {
 	for i := 0; i < n; i++ {
 		err := miner.PledgeSector(ctx)
 		require.NoError(t, err)
-		if i%3 == 0 && blockNotif != nil {
-			<-blockNotif
-		}
 	}
 
 	for {
 		s, err := miner.SectorsList(ctx) // Note - the test builder doesn't import genesis sectors into FSM
 		require.NoError(t, err)
 		fmt.Printf("Sectors: %d\n", len(s))
-		if len(s) >= n+existing {
+		if len(s) >= n {
 			break
 		}
 
@@ -99,7 +86,7 @@ func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n,
 
 	for len(toCheck) > 0 {
 		for n := range toCheck {
-			st, err := miner.SectorsStatus(ctx, n, false)
+			st, err := miner.SectorsStatus(ctx, n)
 			require.NoError(t, err)
 			if st.State == api.SectorState(sealing.Proving) {
 				delete(toCheck, n)
@@ -115,8 +102,10 @@ func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n,
 }
 
 func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) {
+	os.Setenv("BELLMAN_NO_GPU", "1")
+
 	ctx := context.Background()
-	n, sn := b(t, 1, OneMiner)
+	n, sn := b(t, 1, oneMiner)
 	client := n[0].FullNode.(*impl.FullNodeAPI)
 	miner := sn[0]
 
@@ -136,13 +125,13 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 		defer close(done)
 		for mine {
 			build.Clock.Sleep(blocktime)
-			if err := sn[0].MineOne(ctx, MineNext); err != nil {
+			if err := sn[0].MineOne(ctx, func(bool, error) {}); err != nil {
 				t.Error(err)
 			}
 		}
 	}()
 
-	pledgeSectors(t, ctx, miner, nSectors, 0, nil)
+	pledgeSectors(t, ctx, miner, nSectors)
 
 	maddr, err := miner.ActorAddress(ctx)
 	require.NoError(t, err)
@@ -150,10 +139,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 	di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
 	require.NoError(t, err)
 
-	mid, err := address.IDFromAddress(maddr)
-	require.NoError(t, err)
-
-	fmt.Printf("Running one proving period\n")
+	fmt.Printf("Running one proving periods\n")
 
 	for {
 		head, err := client.ChainHead(ctx)
@@ -178,143 +164,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 	require.Equal(t, p.MinerPower, p.TotalPower)
 	require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*uint64(nSectors+GenesisPreseals)))
 
-	fmt.Printf("Drop some sectors\n")
-
-	// Drop 2 sectors from deadline 2 partition 0 (full partition / deadline)
-	{
-		parts, err := client.StateMinerPartitions(ctx, maddr, 2, types.EmptyTSK)
-		require.NoError(t, err)
-		require.Greater(t, len(parts), 0)
-
-		n, err := parts[0].Sectors.Count()
-		require.NoError(t, err)
-		require.Equal(t, uint64(2), n)
-
-		// Drop the partition
-		err = parts[0].Sectors.ForEach(func(sid uint64) error {
-			return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(abi.SectorID{
-				Miner:  abi.ActorID(mid),
-				Number: abi.SectorNumber(sid),
-			}, true)
-		})
-		require.NoError(t, err)
-	}
-
-	var s abi.SectorID
-
-	// Drop 1 sectors from deadline 3 partition 0
-	{
-		parts, err := client.StateMinerPartitions(ctx, maddr, 3, types.EmptyTSK)
-		require.NoError(t, err)
-		require.Greater(t, len(parts), 0)
-
-		n, err := parts[0].Sectors.Count()
-		require.NoError(t, err)
-		require.Equal(t, uint64(2), n)
-
-		// Drop the sector
-		sn, err := parts[0].Sectors.First()
-		require.NoError(t, err)
-
-		all, err := parts[0].Sectors.All(2)
-		require.NoError(t, err)
-		fmt.Println("the sectors", all)
-
-		s = abi.SectorID{
-			Miner:  abi.ActorID(mid),
-			Number: abi.SectorNumber(sn),
-		}
-
-		err = miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(s, true)
-		require.NoError(t, err)
-	}
-
-	di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
-	require.NoError(t, err)
-
-	fmt.Printf("Go through another PP, wait for sectors to become faulty\n")
-
-	for {
-		head, err := client.ChainHead(ctx)
-		require.NoError(t, err)
-
-		if head.Height() > di.PeriodStart+(miner2.WPoStProvingPeriod)+2 {
-			break
-		}
-
-		if head.Height()%100 == 0 {
-			fmt.Printf("@%d\n", head.Height())
-		}
-		build.Clock.Sleep(blocktime)
-	}
-
-	p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK)
-	require.NoError(t, err)
-
-	require.Equal(t, p.MinerPower, p.TotalPower)
-
-	sectors := p.MinerPower.RawBytePower.Uint64() / uint64(ssz)
-	require.Equal(t, nSectors+GenesisPreseals-3, int(sectors)) // -3 just removed sectors
-
-	fmt.Printf("Recover one sector\n")
-
-	err = miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(s, false)
-	require.NoError(t, err)
-
-	di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
-	require.NoError(t, err)
-
-	for {
-		head, err := client.ChainHead(ctx)
-		require.NoError(t, err)
-
-		if head.Height() > di.PeriodStart+(miner2.WPoStProvingPeriod)+2 {
-			break
-		}
-
-		if head.Height()%100 == 0 {
-			fmt.Printf("@%d\n", head.Height())
-		}
-		build.Clock.Sleep(blocktime)
-	}
-
-	p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK)
-	require.NoError(t, err)
-
-	require.Equal(t, p.MinerPower, p.TotalPower)
-
-	sectors = p.MinerPower.RawBytePower.Uint64() / uint64(ssz)
-	require.Equal(t, nSectors+GenesisPreseals-2, int(sectors)) // -2 not recovered sectors
-
-	// pledge a sector after recovery
-
-	pledgeSectors(t, ctx, miner, 1, nSectors, nil)
-
-	{
-		// wait a bit more
-
-		head, err := client.ChainHead(ctx)
-		require.NoError(t, err)
-
-		waitUntil := head.Height() + 10
-
-		for {
-			head, err := client.ChainHead(ctx)
-			require.NoError(t, err)
-
-			if head.Height() > waitUntil {
-				break
-			}
-		}
-	}
-
-	p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK)
-	require.NoError(t, err)
-
-	require.Equal(t, p.MinerPower, p.TotalPower)
-
-	sectors = p.MinerPower.RawBytePower.Uint64() / uint64(ssz)
-	require.Equal(t, nSectors+GenesisPreseals-2+1, int(sectors)) // -2 not recovered sectors + 1 just pledged
+	// TODO: Inject faults here
 
 	mine = false
 	<-done

api/types.go

@@ -2,17 +2,11 @@ package api
 
 import (
 	"encoding/json"
-	"fmt"
-
 	"github.com/filecoin-project/go-address"
-	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
-	"github.com/ipfs/go-cid"
 
 	"github.com/libp2p/go-libp2p-core/peer"
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	ma "github.com/multiformats/go-multiaddr"
 )
 
@@ -46,16 +40,15 @@ type ObjStat struct {
 
 type PubsubScore struct {
 	ID    peer.ID
-	Score *pubsub.PeerScoreSnapshot
+	Score float64
 }
 
 type MinerInfo struct {
-	Owner                      address.Address   // Must be an ID-address.
-	Worker                     address.Address   // Must be an ID-address.
-	NewWorker                  address.Address   // Must be an ID-address.
-	ControlAddresses           []address.Address // Must be an ID-addresses.
+	Owner                      address.Address // Must be an ID-address.
+	Worker                     address.Address // Must be an ID-address.
+	NewWorker                  address.Address // Must be an ID-address.
 	WorkerChangeEpoch          abi.ChainEpoch
-	PeerId                     *peer.ID
+	PeerId                     peer.ID
 	Multiaddrs                 []abi.Multiaddrs
 	SealProofType              abi.RegisteredSealProof
 	SectorSize                 abi.SectorSize
@@ -63,20 +56,12 @@ type MinerInfo struct {
 }
 
 func NewApiMinerInfo(info *miner.MinerInfo) MinerInfo {
-	var pid *peer.ID
-	if peerID, err := peer.IDFromBytes(info.PeerId); err == nil {
-		pid = &peerID
-	}
-
 	mi := MinerInfo{
-		Owner:            info.Owner,
-		Worker:           info.Worker,
-		ControlAddresses: info.ControlAddresses,
-
-		NewWorker:         address.Undef,
-		WorkerChangeEpoch: -1,
-
-		PeerId:                     pid,
+		Owner:                      info.Owner,
+		Worker:                     info.Worker,
+		NewWorker:                  address.Undef,
+		WorkerChangeEpoch:          -1,
+		PeerId:                     peer.ID(info.PeerId),
 		Multiaddrs:                 info.Multiaddrs,
 		SealProofType:              info.SealProofType,
 		SectorSize:                 info.SectorSize,
@@ -90,64 +75,3 @@ func NewApiMinerInfo(info *miner.MinerInfo) MinerInfo {
 
 	return mi
 }
-
-type MessageSendSpec struct {
-	MaxFee abi.TokenAmount
-}
-
-var DefaultMessageSendSpec = MessageSendSpec{
-	// MaxFee of 0.1FIL
-	MaxFee: abi.NewTokenAmount(int64(build.FilecoinPrecision) / 10),
-}
-
-func (ms *MessageSendSpec) Get() MessageSendSpec {
-	if ms == nil {
-		return DefaultMessageSendSpec
-	}
-
-	return *ms
-}
-
-type DataTransferChannel struct {
-	TransferID  datatransfer.TransferID
-	Status      datatransfer.Status
-	BaseCID     cid.Cid
-	IsInitiator bool
-	IsSender    bool
-	Voucher     string
-	Message     string
-	OtherPeer   peer.ID
-	Transferred uint64
-}
-
-// NewDataTransferChannel constructs an API DataTransferChannel type from full channel state snapshot and a host id
-func NewDataTransferChannel(hostID peer.ID, channelState datatransfer.ChannelState) DataTransferChannel {
-	channel := DataTransferChannel{
-		TransferID: channelState.TransferID(),
-		Status:     channelState.Status(),
-		BaseCID:    channelState.BaseCID(),
-		IsSender:   channelState.Sender() == hostID,
-		Message:    channelState.Message(),
-	}
-	stringer, ok := channelState.Voucher().(fmt.Stringer)
-	if ok {
-		channel.Voucher = stringer.String()
-	} else {
-		voucherJSON, err := json.Marshal(channelState.Voucher())
-		if err != nil {
-			channel.Voucher = fmt.Errorf("Voucher Serialization: %w", err).Error()
-		} else {
-			channel.Voucher = string(voucherJSON)
-		}
-	}
-	if channel.IsSender {
-		channel.IsInitiator = !channelState.IsPull()
-		channel.Transferred = channelState.Sent()
-		channel.OtherPeer = channelState.Recipient()
-	} else {
-		channel.IsInitiator = channelState.IsPull()
-		channel.Transferred = channelState.Received()
-		channel.OtherPeer = channelState.Sender()
-	}
-	return channel
-}

api/utils.go

@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
 )
 
 type SignFunc = func(context.Context, []byte) (*crypto.Signature, error)

build/bootstrap.go

@@ -38,3 +38,12 @@ func BuiltinBootstrap() ([]peer.AddrInfo, error) {
 	})
 	return out, err
 }
+
+func DrandBootstrap() ([]peer.AddrInfo, error) {
+	addrs := []string{
+		"/dnsaddr/pl-eu.testnet.drand.sh/",
+		"/dnsaddr/pl-us.testnet.drand.sh/",
+		"/dnsaddr/pl-sin.testnet.drand.sh/",
+	}
+	return addrutil.ParseAddresses(context.TODO(), addrs)
+}

build/bootstrap/bootstrappers.pi

@@ -1,6 +1,2 @@
-/dns4/bootstrap-0.testnet.fildev.network/tcp/1347/p2p/12D3KooWJTUBUjtzWJGWU1XSiY21CwmHaCNLNYn2E7jqHEHyZaP7
-/dns4/bootstrap-1.testnet.fildev.network/tcp/1347/p2p/12D3KooW9yeKXha4hdrJKq74zEo99T8DhriQdWNoojWnnQbsgB3v
-/dns4/bootstrap-2.testnet.fildev.network/tcp/1347/p2p/12D3KooWCrx8yVG9U9Kf7w8KLN3Edkj5ZKDhgCaeMqQbcQUoB6CT
-/dns4/bootstrap-4.testnet.fildev.network/tcp/1347/p2p/12D3KooWPkL9LrKRQgHtq7kn9ecNhGU9QaziG8R5tX8v9v7t3h34
-/dns4/bootstrap-3.testnet.fildev.network/tcp/1347/p2p/12D3KooWKYSsbpgZ3HAjax5M1BXCwXLa6gVkUARciz7uN3FNtr7T
-/dns4/bootstrap-5.testnet.fildev.network/tcp/1347/p2p/12D3KooWQYzqnLASJAabyMpPb1GcWZvNSe7JDcRuhdRqonFoiK9W
+/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWPUpyEM2R6Drs67N79WcVa4v5AymCR8xjgah2iJW2ww4E
+/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWSmQbP2FnYH8X2ZzR11X8SyffZeBmjGZ8GbEtUzxdpcHt

build/clock.go

@@ -6,5 +6,5 @@ import "github.com/raulk/clock"
 // we use a real-time clock, which maps to the `time` package.
 //
 // Tests that need control of time can replace this variable with
-// clock.NewMock(). Always use real time for socket/stream deadlines.
+// clock.NewMock().
 var Clock = clock.New()

build/drand.go

@@ -1,72 +0,0 @@
-package build
-
-import "github.com/filecoin-project/lotus/node/modules/dtypes"
-
-var DrandNetwork = DrandIncentinet
-
-func DrandConfig() dtypes.DrandConfig {
-	return DrandConfigs[DrandNetwork]
-}
-
-type DrandEnum int
-
-const (
-	DrandMainnet DrandEnum = iota + 1
-	DrandTestnet
-	DrandDevnet
-	DrandLocalnet
-	DrandIncentinet
-)
-
-var DrandConfigs = map[DrandEnum]dtypes.DrandConfig{
-	DrandMainnet: {
-		Servers: []string{
-			"https://api.drand.sh",
-			"https://api2.drand.sh",
-			"https://api3.drand.sh",
-		},
-		Relays: []string{
-			"/dnsaddr/api.drand.sh/",
-			"/dnsaddr/api2.drand.sh/",
-			"/dnsaddr/api3.drand.sh/",
-		},
-		ChainInfoJSON: `{"public_key":"868f005eb8e6e4ca0a47c8a77ceaa5309a47978a7c71bc5cce96366b5d7a569937c529eeda66c7293784a9402801af31","period":30,"genesis_time":1595431050,"hash":"8990e7a9aaed2ffed73dbd7092123d6f289930540d7651336225dc172e51b2ce","groupHash":"176f93498eac9ca337150b46d21dd58673ea4e3581185f869672e59fa4cb390a"}`,
-	},
-	DrandTestnet: {
-		Servers: []string{
-			"https://pl-eu.testnet.drand.sh",
-			"https://pl-us.testnet.drand.sh",
-			"https://pl-sin.testnet.drand.sh",
-		},
-		Relays: []string{
-			"/dnsaddr/pl-eu.testnet.drand.sh/",
-			"/dnsaddr/pl-us.testnet.drand.sh/",
-			"/dnsaddr/pl-sin.testnet.drand.sh/",
-		},
-		ChainInfoJSON: `{"public_key":"922a2e93828ff83345bae533f5172669a26c02dc76d6bf59c80892e12ab1455c229211886f35bb56af6d5bea981024df","period":25,"genesis_time":1590445175,"hash":"84b2234fb34e835dccd048255d7ad3194b81af7d978c3bf157e3469592ae4e02","groupHash":"4dd408e5fdff9323c76a9b6f087ba8fdc5a6da907bd9217d9d10f2287d081957"}`,
-	},
-	DrandDevnet: {
-		Servers: []string{
-			"https://dev1.drand.sh",
-			"https://dev2.drand.sh",
-		},
-		Relays: []string{
-			"/dnsaddr/dev1.drand.sh/",
-			"/dnsaddr/dev2.drand.sh/",
-		},
-		ChainInfoJSON: `{"public_key":"8cda589f88914aa728fd183f383980b35789ce81b274e5daee1f338b77d02566ef4d3fb0098af1f844f10f9c803c1827","period":25,"genesis_time":1595348225,"hash":"e73b7dc3c4f6a236378220c0dd6aa110eb16eed26c11259606e07ee122838d4f","groupHash":"567d4785122a5a3e75a9bc9911d7ea807dd85ff76b78dc4ff06b075712898607"}`,
-	},
-	DrandIncentinet: {
-		Servers: []string{
-			"https://pl-eu.incentinet.drand.sh",
-			"https://pl-us.incentinet.drand.sh",
-			"https://pl-sin.incentinet.drand.sh",
-		},
-		Relays: []string{
-			"/dnsaddr/pl-eu.incentinet.drand.sh/",
-			"/dnsaddr/pl-us.incentinet.drand.sh/",
-			"/dnsaddr/pl-sin.incentinet.drand.sh/",
-		},
-		ChainInfoJSON: `{"public_key":"8cad0c72c606ab27d36ee06de1d5b2db1faf92e447025ca37575ab3a8aac2eaae83192f846fc9e158bc738423753d000","period":30,"genesis_time":1595873820,"hash":"80c8b872c714f4c00fdd3daa465d5514049f457f01f85a4caf68cdcd394ba039","groupHash":"d9406aaed487f7af71851b4399448e311f2328923d454e971536c05398ce2d9b"}`,
-	},
-}

build/params_2k.go

@@ -3,16 +3,13 @@
 package build
 
 import (
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
 )
 
-const UpgradeBreezeHeight = 0
-const BreezeGasTampingDuration = 0
-
 func init() {
 	power.ConsensusMinerMinPower = big.NewInt(2048)
 	miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{
@@ -23,9 +20,9 @@ func init() {
 	BuildType |= Build2k
 }
 
-const BlockDelaySecs = uint64(4)
+const BlockDelaySecs = uint64(2)
 
-const PropagationDelaySecs = uint64(1)
+const PropagationDelaySecs = uint64(3)
 
 // SlashablePowerDelay is the number of epochs after ElectionPeriodStart, after
 // which the miner is slashed

build/params_shared_funcs.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/libp2p/go-libp2p-core/protocol"
 
-	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 
 	"github.com/filecoin-project/lotus/node/modules/dtypes"

build/params_shared_vals.go

@@ -5,9 +5,11 @@ package build
 import (
 	"math/big"
 
-	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
+
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
 )
 
 // /////
@@ -59,8 +61,8 @@ const WinningPoStSectorSetLookback = abi.ChainEpoch(10)
 // /////
 // Devnet settings
 
-const FilBase = uint64(2_000_000_000)
-const FilAllocStorageMining = uint64(1_100_000_000)
+const TotalFilecoin = uint64(2_000_000_000)
+const MiningRewardTotal = uint64(1_900_000_000)
 
 const FilecoinPrecision = uint64(1_000_000_000_000_000_000)
 
@@ -69,7 +71,7 @@ var InitialRewardBalance *big.Int
 // TODO: Move other important consts here
 
 func init() {
-	InitialRewardBalance = big.NewInt(int64(FilAllocStorageMining))
+	InitialRewardBalance = big.NewInt(int64(MiningRewardTotal))
 	InitialRewardBalance = InitialRewardBalance.Mul(InitialRewardBalance, big.NewInt(int64(FilecoinPrecision)))
 }
 
@@ -88,16 +90,14 @@ const VerifSigCacheSize = 32000
 // Limits
 
 // TODO: If this is gonna stay, it should move to specs-actors
-const BlockMessageLimit = 10000
+const BlockMessageLimit = 512
+const BlockGasLimit = 7_500_000_000
 
-const BlockGasLimit = 10_000_000_000
-const BlockGasTarget = BlockGasLimit / 2
-const BaseFeeMaxChangeDenom = 8 // 12.5%
-const InitialBaseFee = 100e6
-const MinimumBaseFee = 100
-const PackingEfficiencyNum = 4
-const PackingEfficiencyDenom = 5
-
-// Actor consts
-// TODO: Pull from actors when its made not private
-var MinDealDuration = abi.ChainEpoch(180 * builtin.EpochsInDay)
+var DrandConfig = dtypes.DrandConfig{
+	Servers: []string{
+		"https://pl-eu.testnet.drand.sh",
+		"https://pl-us.testnet.drand.sh",
+		"https://pl-sin.testnet.drand.sh",
+	},
+	ChainInfoJSON: `{"public_key":"922a2e93828ff83345bae533f5172669a26c02dc76d6bf59c80892e12ab1455c229211886f35bb56af6d5bea981024df","period":25,"genesis_time":1590445175,"hash":"84b2234fb34e835dccd048255d7ad3194b81af7d978c3bf157e3469592ae4e02","groupHash":"4dd408e5fdff9323c76a9b6f087ba8fdc5a6da907bd9217d9d10f2287d081957"}`,
+}

build/params_testground.go

@@ -10,7 +10,9 @@ package build
 import (
 	"math/big"
 
-	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 )
@@ -19,15 +21,11 @@ var (
 	UnixfsChunkSize     = uint64(1 << 20)
 	UnixfsLinksPerLevel = 1024
 
-	BlocksPerEpoch        = uint64(builtin.ExpectedLeadersPerEpoch)
-	BlockMessageLimit     = 512
-	BlockGasLimit         = int64(100_000_000_000)
-	BlockGasTarget        = int64(BlockGasLimit / 2)
-	BaseFeeMaxChangeDenom = int64(8) // 12.5%
-	InitialBaseFee        = int64(100e6)
-	MinimumBaseFee        = int64(100)
-	BlockDelaySecs        = uint64(builtin.EpochDurationSeconds)
-	PropagationDelaySecs  = uint64(6)
+	BlocksPerEpoch       = uint64(builtin.ExpectedLeadersPerEpoch)
+	BlockMessageLimit    = 512
+	BlockGasLimit        = int64(100_000_000_000)
+	BlockDelaySecs       = uint64(builtin.EpochDurationSeconds)
+	PropagationDelaySecs = uint64(6)
 
 	AllowableClockDriftSecs = uint64(1)
 
@@ -53,23 +51,23 @@ var (
 	TicketRandomnessLookback     = abi.ChainEpoch(1)
 	WinningPoStSectorSetLookback = abi.ChainEpoch(10)
 
-	FilBase               uint64 = 2_000_000_000
-	FilAllocStorageMining uint64 = 1_400_000_000
+	TotalFilecoin     uint64 = 2_000_000_000
+	MiningRewardTotal uint64 = 1_400_000_000
 
 	FilecoinPrecision uint64 = 1_000_000_000_000_000_000
 
 	InitialRewardBalance = func() *big.Int {
-		v := big.NewInt(int64(FilAllocStorageMining))
+		v := big.NewInt(int64(MiningRewardTotal))
 		v = v.Mul(v, big.NewInt(int64(FilecoinPrecision)))
 		return v
 	}()
-	// Actor consts
-	// TODO: Pull from actors when its made not private
-	MinDealDuration = abi.ChainEpoch(180 * builtin.EpochsInDay)
 
-	PackingEfficiencyNum   int64 = 4
-	PackingEfficiencyDenom int64 = 5
-
-	UpgradeBreezeHeight      abi.ChainEpoch = 0
-	BreezeGasTampingDuration abi.ChainEpoch = 0
+	DrandConfig = dtypes.DrandConfig{
+		Servers: []string{
+			"https://pl-eu.testnet.drand.sh",
+			"https://pl-us.testnet.drand.sh",
+			"https://pl-sin.testnet.drand.sh",
+		},
+		ChainInfoJSON: `{"public_key":"922a2e93828ff83345bae533f5172669a26c02dc76d6bf59c80892e12ab1455c229211886f35bb56af6d5bea981024df","period":25,"genesis_time":1590445175,"hash":"138a324aa6540f93d0dad002aa89454b1bec2b6e948682cde6bd4db40f4b7c9b"}`,
+	}
 )

build/params_testnet.go

@@ -5,21 +5,19 @@
 package build
 
 import (
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 )
 
-const UpgradeBreezeHeight = 41280
-const BreezeGasTampingDuration = 120
-
 func init() {
-	power.ConsensusMinerMinPower = big.NewInt(10 << 40)
+	power.ConsensusMinerMinPower = big.NewInt(1024 << 20)
 	miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{
-		abi.RegisteredSealProof_StackedDrg32GiBV1: {},
-		abi.RegisteredSealProof_StackedDrg64GiBV1: {},
+		abi.RegisteredSealProof_StackedDrg512MiBV1: {},
+		abi.RegisteredSealProof_StackedDrg32GiBV1:  {},
+		abi.RegisteredSealProof_StackedDrg64GiBV1:  {},
 	}
 }
 

build/proof-params/parameters.json

@@ -1,152 +1,152 @@
 {
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.params": {
-    "cid": "QmVxjFRyhmyQaZEtCh7nk2abc7LhFkzhnRX4rcHqCCpikR",
-    "digest": "7610b9f82bfc88405b7a832b651ce2f6",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.params": {
+    "cid": "QmeDRyxek34F1H6xJY6AkFdWvPsy5F6dKTrebV3ZtWT4ky",
+    "digest": "f5827f2d8801c62c831e0f972f6dc8bb",
     "sector_size": 2048
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.vk": {
-    "cid": "QmcS5JZs8X3TdtkEBpHAdUYjdNDqcL7fWQFtQz69mpnu2X",
-    "digest": "0e0958009936b9d5e515ec97b8cb792d",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.vk": {
+    "cid": "QmUw1ZmG4BBbX19MsbH3zAEGKUc42iFJc5ZAyomDHeJTsA",
+    "digest": "398fecdb4b2de445125852bc3c080b35",
     "sector_size": 2048
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0cfb4f178bbb71cf2ecfcd42accce558b27199ab4fb59cb78f2483fe21ef36d9.params": {
-    "cid": "QmUiRx71uxfmUE8V3H9sWAsAXoM88KR4eo1ByvvcFNeTLR",
-    "digest": "1a7d4a9c8a502a497ed92a54366af33f",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0cfb4f178bbb71cf2ecfcd42accce558b27199ab4fb59cb78f2483fe21ef36d9.params": {
+    "cid": "QmUeNKp9YZpiAFm81RV5KuxH1FDGJx2DuwcbU2XNSZLLSv",
+    "digest": "2b6d2972ac9e862e8134d98fb695b0c5",
     "sector_size": 536870912
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0cfb4f178bbb71cf2ecfcd42accce558b27199ab4fb59cb78f2483fe21ef36d9.vk": {
-    "cid": "QmfCeddjFpWtavzfEzZpJfzSajGNwfL4RjFXWAvA9TSnTV",
-    "digest": "4dae975de4f011f101f5a2f86d1daaba",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0cfb4f178bbb71cf2ecfcd42accce558b27199ab4fb59cb78f2483fe21ef36d9.vk": {
+    "cid": "QmQaQmTXX995Akd66ggtJY5bNx6Gkxk8P34JTdMMq8393G",
+    "digest": "3688c9eb256b7b17f411dad78d5ef74a",
     "sector_size": 536870912
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.params": {
-    "cid": "QmcSTqDcFVLGGVYz1njhUZ7B6fkKtBumsLUwx4nkh22TzS",
-    "digest": "82c88066be968bb550a05e30ff6c2413",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.params": {
+    "cid": "QmfEYTMSkwGJTumQx26iKXGNKiYh3mmAC4SkdybZpJCj5p",
+    "digest": "09bff16aed893349d94485cfae366a9c",
     "sector_size": 2048
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.vk": {
-    "cid": "QmSTCXF2ipGA3f6muVo6kHc2URSx6PzZxGUqu7uykaH5KU",
-    "digest": "ffd79788d614d27919ae5bd2d94eacb6",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.vk": {
+    "cid": "QmP4ThPieSUJyRanjibWpT5R5cCMzMAU4j8Y7kBn7CSW1Q",
+    "digest": "142f2f7e8f1b1779290315cabfd2c803",
     "sector_size": 2048
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-50c7368dea9593ed0989e70974d28024efa9d156d585b7eea1be22b2e753f331.params": {
-    "cid": "QmU9SBzJNrcjRFDiFc4GcApqdApN6z9X7MpUr66mJ2kAJP",
-    "digest": "700171ecf7334e3199437c930676af82",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-50c7368dea9593ed0989e70974d28024efa9d156d585b7eea1be22b2e753f331.params": {
+    "cid": "QmcAixrHsz29DgvtZiMc2kQjvPRvWxYUp36QYmRDZbmREm",
+    "digest": "8f987f64d434365562180b96ec12e299",
     "sector_size": 8388608
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-50c7368dea9593ed0989e70974d28024efa9d156d585b7eea1be22b2e753f331.vk": {
-    "cid": "QmbmUMa3TbbW3X5kFhExs6WgC4KeWT18YivaVmXDkB6ANG",
-    "digest": "79ebb55f56fda427743e35053edad8fc",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-50c7368dea9593ed0989e70974d28024efa9d156d585b7eea1be22b2e753f331.vk": {
+    "cid": "QmT4iFnbL6r4txS5PXsiV7NTzbhCxHy54PvdkJJGV2VFXb",
+    "digest": "94b6c24ac01924f4feeecedd16b5d77d",
     "sector_size": 8388608
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-5294475db5237a2e83c3e52fd6c2b03859a1831d45ed08c4f35dbf9a803165a9.params": {
-    "cid": "QmdNEL2RtqL52GQNuj8uz6mVj5Z34NVnbaJ1yMyh1oXtBx",
-    "digest": "c49499bb76a0762884896f9683403f55",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-5294475db5237a2e83c3e52fd6c2b03859a1831d45ed08c4f35dbf9a803165a9.params": {
+    "cid": "QmbjFst6SFCK1KsTQrfwPdxf3VTNa1raed574tEZZ9PoyQ",
+    "digest": "2c245fe8179839dd6c6cdea207c67ae8",
     "sector_size": 8388608
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-5294475db5237a2e83c3e52fd6c2b03859a1831d45ed08c4f35dbf9a803165a9.vk": {
-    "cid": "QmUiVYCQUgr6Y13pZFr8acWpSM4xvTXUdcvGmxyuHbKhsc",
-    "digest": "34d4feeacd9abf788d69ef1bb4d8fd00",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-5294475db5237a2e83c3e52fd6c2b03859a1831d45ed08c4f35dbf9a803165a9.vk": {
+    "cid": "QmQJKmvZN1a5cQ1Nw6CDyXs3nuRPzvyU5NvCFMUL2BfcZC",
+    "digest": "56ae47bfda53bb8d22981ed8d8d27d72",
     "sector_size": 8388608
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-7d739b8cf60f1b0709eeebee7730e297683552e4b69cab6984ec0285663c5781.params": {
-    "cid": "QmVgCsJFRXKLuuUhT3aMYwKVGNA9rDeR6DCrs7cAe8riBT",
-    "digest": "827359440349fe8f5a016e7598993b79",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-7d739b8cf60f1b0709eeebee7730e297683552e4b69cab6984ec0285663c5781.params": {
+    "cid": "QmQCABxeTpdvXTyjDyk7nPBxkQzCh7MXfGztWnSXEPKMLW",
+    "digest": "7e6b2eb5ecbb11ac651ad66ebbb2075a",
     "sector_size": 536870912
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-7d739b8cf60f1b0709eeebee7730e297683552e4b69cab6984ec0285663c5781.vk": {
-    "cid": "QmfA31fbCWojSmhSGvvfxmxaYCpMoXP95zEQ9sLvBGHNaN",
-    "digest": "bd2cd62f65c1ab84f19ca27e97b7c731",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-7d739b8cf60f1b0709eeebee7730e297683552e4b69cab6984ec0285663c5781.vk": {
+    "cid": "QmPBweyugh5Sx4umk8ULhgEGbjY8xmWLfU6M7EMpc8Mad6",
+    "digest": "94a8d9e25a9ab9674d339833664eba25",
     "sector_size": 536870912
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.params": {
-    "cid": "QmaUmfcJt6pozn8ndq1JVBzLRjRJdHMTPd4foa8iw5sjBZ",
-    "digest": "2cf49eb26f1fee94c85781a390ddb4c8",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.params": {
+    "cid": "QmY5yax1E9KymBnCeHksE9Zi8NieZbmwcpoDGoabkeeb9h",
+    "digest": "c909ea9e3fe25ab9b391a64593afdbba",
     "sector_size": 34359738368
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.vk": {
-    "cid": "QmR9i9KL3vhhAqTBGj1bPPC7LvkptxrH9RvxJxLN1vvsBE",
-    "digest": "0f8ec542485568fa3468c066e9fed82b",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.vk": {
+    "cid": "QmXnPo4yH5mwMguwrvqgRfduSttbmPrXtbBfbwU21wQWHt",
+    "digest": "caf900461e988bbf86dbcaca087b7864",
     "sector_size": 34359738368
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.params": {
-    "cid": "Qmdtczp7p4wrbDofmHdGhiixn9irAcN77mV9AEHZBaTt1i",
-    "digest": "d84f79a16fe40e9e25a36e2107bb1ba0",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.params": {
+    "cid": "QmZtzzPWwmZEgR7MSMvXRbt9KVK8k4XZ5RLWHybHJW9SdE",
+    "digest": "a2844f0703f186d143a06146a04577d8",
     "sector_size": 34359738368
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.vk": {
-    "cid": "QmZCvxKcKP97vDAk8Nxs9R1fWtqpjQrAhhfXPoCi1nkDoF",
-    "digest": "fc02943678dd119e69e7fab8420e8819",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.vk": {
+    "cid": "QmWxEA7EdQCUJTzjNpxg5XTF45D2uVyYnN1QRUb5TRYU8M",
+    "digest": "2306247a1e616dbe07f01b88196c2044",
     "sector_size": 34359738368
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-2627e4006b67f99cef990c0a47d5426cb7ab0a0ad58fc1061547bf2d28b09def.params": {
-    "cid": "QmeAN4vuANhXsF8xP2Lx5j2L6yMSdogLzpcvqCJThRGK1V",
-    "digest": "3810b7780ac0e299b22ae70f1f94c9bc",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-2627e4006b67f99cef990c0a47d5426cb7ab0a0ad58fc1061547bf2d28b09def.params": {
+    "cid": "QmP676KwuvyF9Y64uJnXvLtvD1xcuWQ6wD23RzYtQ6dd4f",
+    "digest": "215b1c667a4f46a1d0178338df568615",
     "sector_size": 68719476736
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-2627e4006b67f99cef990c0a47d5426cb7ab0a0ad58fc1061547bf2d28b09def.vk": {
-    "cid": "QmWV8rqZLxs1oQN9jxNWmnT1YdgLwCcscv94VARrhHf1T7",
-    "digest": "59d2bf1857adc59a4f08fcf2afaa916b",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-2627e4006b67f99cef990c0a47d5426cb7ab0a0ad58fc1061547bf2d28b09def.vk": {
+    "cid": "QmPvPwbJtcSGyqB1rQJhSF5yvFbX9ZBSsHVej5F8JUyHUJ",
+    "digest": "0c9c423b28b1455fcbc329a1045fd4dd",
     "sector_size": 68719476736
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-b62098629d07946e9028127e70295ed996fe3ed25b0f9f88eb610a0ab4385a3c.params": {
-    "cid": "QmVkrXc1SLcpgcudK5J25HH93QvR9tNsVhVTYHm5UymXAz",
-    "digest": "2170a91ad5bae22ea61f2ea766630322",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-b62098629d07946e9028127e70295ed996fe3ed25b0f9f88eb610a0ab4385a3c.params": {
+    "cid": "QmUxPQfvckzm1t6MFRdDZ1fDK5UJzAjK7pTZ97cwyachdr",
+    "digest": "965132f51ae445b0e6d32692b7561995",
     "sector_size": 68719476736
   },
-  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-b62098629d07946e9028127e70295ed996fe3ed25b0f9f88eb610a0ab4385a3c.vk": {
-    "cid": "QmbfQjPD7EpzjhWGmvWAsyN2mAZ4PcYhsf3ujuhU9CSuBm",
-    "digest": "6d3789148fb6466d07ee1e24d6292fd6",
+  "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-b62098629d07946e9028127e70295ed996fe3ed25b0f9f88eb610a0ab4385a3c.vk": {
+    "cid": "QmTxq2EBnQWb5R8tS4MHdchj4vNfLYGoSXxwJFvs5xgW4K",
+    "digest": "fc8c3d26e0e56373ad96cb41520d55a6",
     "sector_size": 68719476736
   },
-  "v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.params": {
-    "cid": "QmWceMgnWYLopMuM4AoGMvGEau7tNe5UK83XFjH5V9B17h",
-    "digest": "434fb1338ecfaf0f59256f30dde4968f",
+  "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.params": {
+    "cid": "QmRjgZHERgqGoRagR788Kh6ybi26csVYa8mqbqhmZm57Jx",
+    "digest": "cfc7b0897d1eee48c586f7beb89e67f7",
     "sector_size": 2048
   },
-  "v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.vk": {
-    "cid": "QmamahpFCstMUqHi2qGtVoDnRrsXhid86qsfvoyCTKJqHr",
-    "digest": "dc1ade9929ade1708238f155343044ac",
+  "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.vk": {
+    "cid": "QmNjvnvFP7KgovHUddULoB19fBHT81iz7NcUbzEHZUUPsm",
+    "digest": "fb59bd061c987eac7068008c44de346b",
     "sector_size": 2048
   },
-  "v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-6babf46ce344ae495d558e7770a585b2382d54f225af8ed0397b8be7c3fcd472.params": {
-    "cid": "QmYBpTt7LWNAWr1JXThV5VxX7wsQFLd1PHrGYVbrU1EZjC",
-    "digest": "6c77597eb91ab936c1cef4cf19eba1b3",
+  "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-6babf46ce344ae495d558e7770a585b2382d54f225af8ed0397b8be7c3fcd472.params": {
+    "cid": "QmTpRPBA4dt8fgGpcVzi4L1KA1U2eBHCE8WVmS2GUygMvT",
+    "digest": "36d465915b0afbf96bd08e7915e00952",
     "sector_size": 536870912
   },
-  "v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-6babf46ce344ae495d558e7770a585b2382d54f225af8ed0397b8be7c3fcd472.vk": {
-    "cid": "QmWionkqH2B6TXivzBSQeSyBxojaiAFbzhjtwYRrfwd8nH",
-    "digest": "065179da19fbe515507267677f02823e",
+  "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-6babf46ce344ae495d558e7770a585b2382d54f225af8ed0397b8be7c3fcd472.vk": {
+    "cid": "QmRzDyVfQCLsxspoVsed5bcQRsG6KiktngJfcNBL3TJPZe",
+    "digest": "99d16df0eb6a7e227a4f4570c4f6b6f1",
     "sector_size": 536870912
   },
-  "v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-ecd683648512ab1765faa2a5f14bab48f676e633467f0aa8aad4b55dcb0652bb.params": {
-    "cid": "QmPXAPPuQtuQz7Zz3MHMAMEtsYwqM1o9H1csPLeiMUQwZH",
-    "digest": "09e612e4eeb7a0eb95679a88404f960c",
+  "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-ecd683648512ab1765faa2a5f14bab48f676e633467f0aa8aad4b55dcb0652bb.params": {
+    "cid": "QmV8ZjTSGzDUWmFvsq9NSyPBR7eDDUcvCPNgj2yE7HMAFu",
+    "digest": "34f3ddf1d1c9f41c0cd73b91e8b4bc27",
     "sector_size": 8388608
   },
-  "v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-ecd683648512ab1765faa2a5f14bab48f676e633467f0aa8aad4b55dcb0652bb.vk": {
-    "cid": "QmYCuipFyvVW1GojdMrjK1JnMobXtT4zRCZs1CGxjizs99",
-    "digest": "b687beb9adbd9dabe265a7e3620813e4",
+  "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-ecd683648512ab1765faa2a5f14bab48f676e633467f0aa8aad4b55dcb0652bb.vk": {
+    "cid": "QmTa3VbjTiqJWU6r4WKayaQrUaaBsrpp5UDqYvPDd2C5hs",
+    "digest": "ec62d59651daa5631d3d1e9c782dd940",
     "sector_size": 8388608
   },
-  "v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.params": {
-    "cid": "QmengpM684XLQfG8754ToonszgEg2bQeAGUan5uXTHUQzJ",
-    "digest": "6a388072a518cf46ebd661f5cc46900a",
+  "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.params": {
+    "cid": "Qmf8ngfArxrv9tFWDqBcNegdBMymvuakwyHKd1pbW3pbsb",
+    "digest": "a16d6f4c6424fb280236739f84b24f97",
     "sector_size": 34359738368
   },
-  "v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.vk": {
-    "cid": "Qmf93EMrADXAK6CyiSfE8xx45fkMfR3uzKEPCvZC1n2kzb",
-    "digest": "0c7b4aac1c40fdb7eb82bc355b41addf",
+  "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.vk": {
+    "cid": "QmfQgVFerArJ6Jupwyc9tKjLD9n1J9ajLHBdpY465tRM7M",
+    "digest": "7a139d82b8a02e35279d657e197f5c1f",
     "sector_size": 34359738368
   },
-  "v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-2-sha256_hasher-96f1b4a04c5c51e4759bbf224bbc2ef5a42c7100f16ec0637123f16a845ddfb2.params": {
-    "cid": "QmS7ye6Ri2MfFzCkcUJ7FQ6zxDKuJ6J6B8k5PN7wzSR9sX",
-    "digest": "1801f8a6e1b00bceb00cc27314bb5ce3",
+  "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-2-sha256_hasher-96f1b4a04c5c51e4759bbf224bbc2ef5a42c7100f16ec0637123f16a845ddfb2.params": {
+    "cid": "QmfDha8271nXJn14Aq3qQeghjMBWbs6HNSGa6VuzCVk4TW",
+    "digest": "5d3cd3f107a3bea8a96d1189efd2965c",
     "sector_size": 68719476736
   },
-  "v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-2-sha256_hasher-96f1b4a04c5c51e4759bbf224bbc2ef5a42c7100f16ec0637123f16a845ddfb2.vk": {
-    "cid": "QmehSmC6BhrgRZakPDta2ewoH9nosNzdjCqQRXsNFNUkLN",
-    "digest": "a89884252c04c298d0b3c81bfd884164",
+  "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-2-sha256_hasher-96f1b4a04c5c51e4759bbf224bbc2ef5a42c7100f16ec0637123f16a845ddfb2.vk": {
+    "cid": "QmRVtTtiFzHJTHurYzaCvetGAchux9cktixT4aGHthN6Zt",
+    "digest": "62c366405404e60f171e661492740b1c",
     "sector_size": 68719476736
   }
-}
+}

build/version.go

@@ -25,7 +25,7 @@ func buildType() string {
 }
 
 // BuildVersion is the local build version, set by build system
-const BuildVersion = "0.6.1"
+const BuildVersion = "0.4.1"
 
 func UserVersion() string {
 	return BuildVersion + buildType() + CurrentCommit
@@ -53,7 +53,7 @@ func (ve Version) EqMajorMinor(v2 Version) bool {
 }
 
 // APIVersion is a semver version of the rpc api exposed
-var APIVersion Version = newVer(0, 14, 0)
+var APIVersion Version = newVer(0, 8, 0)
 
 //nolint:varcheck,deadcode
 const (

chain/actors/aerrors/error.go

@@ -3,7 +3,7 @@ package aerrors
 import (
 	"fmt"
 
-	"github.com/filecoin-project/go-state-types/exitcode"
+	"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
 	"golang.org/x/xerrors"
 )
 

chain/actors/aerrors/error_test.go

@@ -3,8 +3,8 @@ package aerrors_test
 import (
 	"testing"
 
-	"github.com/filecoin-project/go-state-types/exitcode"
 	. "github.com/filecoin-project/lotus/chain/actors/aerrors"
+	"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
 
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/xerrors"

chain/actors/aerrors/wrap.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/filecoin-project/go-state-types/exitcode"
+	"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
 	cbor "github.com/ipfs/go-ipld-cbor"
 	"golang.org/x/xerrors"
 )

chain/beacon/beacon.go

@@ -3,7 +3,7 @@ package beacon
 import (
 	"context"
 
-	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	logging "github.com/ipfs/go-log"
 	"golang.org/x/xerrors"
 
@@ -37,10 +37,6 @@ func ValidateBlockValues(b RandomBeacon, h *types.BlockHeader, prevEntry types.B
 		return nil
 	}
 
-	if len(h.BeaconEntries) == 0 {
-		return xerrors.Errorf("expected to have beacon entries in this block, but didn't find any")
-	}
-
 	last := h.BeaconEntries[len(h.BeaconEntries)-1]
 	if last.Round != maxRound {
 		return xerrors.Errorf("expected final beacon entry in block to be at round %d, got %d", maxRound, last.Round)

chain/beacon/drand/drand.go

@@ -19,7 +19,7 @@ import (
 	logging "github.com/ipfs/go-log"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 
-	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/beacon"
@@ -171,10 +171,6 @@ func (db *DrandBeacon) VerifyEntry(curr types.BeaconEntry, prev types.BeaconEntr
 		// TODO handle genesis better
 		return nil
 	}
-	if be := db.getCachedValue(curr.Round); be != nil {
-		// return no error if the value is in the cache already
-		return nil
-	}
 	b := &dchain.Beacon{
 		PreviousSig: prev.Data,
 		Round:       curr.Round,

chain/beacon/drand/drand_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestPrintGroupInfo(t *testing.T) {
-	server := build.DrandConfig().Servers[0]
+	server := build.DrandConfig.Servers[0]
 	c, err := hclient.New(server, nil, nil)
 	assert.NoError(t, err)
 	cg := c.(interface {

chain/beacon/mock.go

@@ -6,8 +6,8 @@ import (
 	"encoding/binary"
 	"time"
 
-	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/minio/blake2b-simd"
 	"golang.org/x/xerrors"
 )

chain/block_receipt_tracker.go

@@ -7,8 +7,8 @@ import (
 
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
-	lru "github.com/hashicorp/golang-lru"
-	"github.com/libp2p/go-libp2p-core/peer"
+	"github.com/hashicorp/golang-lru"
+	peer "github.com/libp2p/go-libp2p-core/peer"
 )
 
 type blockReceiptTracker struct {

chain/blocksync/blocksync.go

@@ -0,0 +1,276 @@
+package blocksync
+
+import (
+	"bufio"
+	"context"
+	"time"
+
+	"github.com/libp2p/go-libp2p-core/protocol"
+	"go.opencensus.io/trace"
+	"golang.org/x/xerrors"
+
+	cborutil "github.com/filecoin-project/go-cbor-util"
+
+	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/types"
+
+	"github.com/ipfs/go-cid"
+	logging "github.com/ipfs/go-log/v2"
+	inet "github.com/libp2p/go-libp2p-core/network"
+	"github.com/libp2p/go-libp2p-core/peer"
+)
+
+var log = logging.Logger("blocksync")
+
+type NewStreamFunc func(context.Context, peer.ID, ...protocol.ID) (inet.Stream, error)
+
+const BlockSyncProtocolID = "/fil/sync/blk/0.0.1"
+
+const BlockSyncMaxRequestLength = 800
+
+// BlockSyncService is the component that services BlockSync requests from
+// peers.
+//
+// BlockSync is the basic chain synchronization protocol of Filecoin. BlockSync
+// is an RPC-oriented protocol, with a single operation to request blocks.
+//
+// A request contains a start anchor block (referred to with a CID), and a
+// amount of blocks requested beyond the anchor (including the anchor itself).
+//
+// A client can also pass options, encoded as a 64-bit bitfield. Lotus supports
+// two options at the moment:
+//
+//  - include block contents
+//  - include block messages
+//
+// The response will include a status code, an optional message, and the
+// response payload in case of success. The payload is a slice of serialized
+// tipsets.
+type BlockSyncService struct {
+	cs *store.ChainStore
+}
+
+type BlockSyncRequest struct {
+	Start         []cid.Cid
+	RequestLength uint64
+
+	Options uint64
+}
+
+type BSOptions struct {
+	IncludeBlocks   bool
+	IncludeMessages bool
+}
+
+func ParseBSOptions(optfield uint64) *BSOptions {
+	return &BSOptions{
+		IncludeBlocks:   optfield&(BSOptBlocks) != 0,
+		IncludeMessages: optfield&(BSOptMessages) != 0,
+	}
+}
+
+const (
+	BSOptBlocks = 1 << iota
+	BSOptMessages
+)
+
+const (
+	StatusOK            = uint64(0)
+	StatusPartial       = uint64(101)
+	StatusNotFound      = uint64(201)
+	StatusGoAway        = uint64(202)
+	StatusInternalError = uint64(203)
+	StatusBadRequest    = uint64(204)
+)
+
+type BlockSyncResponse struct {
+	Chain []*BSTipSet
+
+	Status  uint64
+	Message string
+}
+
+type BSTipSet struct {
+	Blocks []*types.BlockHeader
+
+	BlsMessages    []*types.Message
+	BlsMsgIncludes [][]uint64
+
+	SecpkMessages    []*types.SignedMessage
+	SecpkMsgIncludes [][]uint64
+}
+
+func NewBlockSyncService(cs *store.ChainStore) *BlockSyncService {
+	return &BlockSyncService{
+		cs: cs,
+	}
+}
+
+func (bss *BlockSyncService) HandleStream(s inet.Stream) {
+	ctx, span := trace.StartSpan(context.Background(), "blocksync.HandleStream")
+	defer span.End()
+
+	defer s.Close() //nolint:errcheck
+
+	var req BlockSyncRequest
+	if err := cborutil.ReadCborRPC(bufio.NewReader(s), &req); err != nil {
+		log.Warnf("failed to read block sync request: %s", err)
+		return
+	}
+	log.Infow("block sync request", "start", req.Start, "len", req.RequestLength)
+
+	resp, err := bss.processRequest(ctx, s.Conn().RemotePeer(), &req)
+	if err != nil {
+		log.Warn("failed to process block sync request: ", err)
+		return
+	}
+
+	writeDeadline := 60 * time.Second
+	_ = s.SetDeadline(time.Now().Add(writeDeadline)) // always use real time for socket/stream deadlines.
+	if err := cborutil.WriteCborRPC(s, resp); err != nil {
+		log.Warnw("failed to write back response for handle stream", "err", err, "peer", s.Conn().RemotePeer())
+		return
+	}
+}
+
+func (bss *BlockSyncService) processRequest(ctx context.Context, p peer.ID, req *BlockSyncRequest) (*BlockSyncResponse, error) {
+	_, span := trace.StartSpan(ctx, "blocksync.ProcessRequest")
+	defer span.End()
+
+	opts := ParseBSOptions(req.Options)
+	if len(req.Start) == 0 {
+		return &BlockSyncResponse{
+			Status:  StatusBadRequest,
+			Message: "no cids given in blocksync request",
+		}, nil
+	}
+
+	span.AddAttributes(
+		trace.BoolAttribute("blocks", opts.IncludeBlocks),
+		trace.BoolAttribute("messages", opts.IncludeMessages),
+		trace.Int64Attribute("reqlen", int64(req.RequestLength)),
+	)
+
+	reqlen := req.RequestLength
+	if reqlen > BlockSyncMaxRequestLength {
+		log.Warnw("limiting blocksync request length", "orig", req.RequestLength, "peer", p)
+		reqlen = BlockSyncMaxRequestLength
+	}
+
+	chain, err := collectChainSegment(bss.cs, types.NewTipSetKey(req.Start...), reqlen, opts)
+	if err != nil {
+		log.Warn("encountered error while responding to block sync request: ", err)
+		return &BlockSyncResponse{
+			Status:  StatusInternalError,
+			Message: err.Error(),
+		}, nil
+	}
+
+	status := StatusOK
+	if reqlen < req.RequestLength {
+		status = StatusPartial
+	}
+
+	return &BlockSyncResponse{
+		Chain:  chain,
+		Status: status,
+	}, nil
+}
+
+func collectChainSegment(cs *store.ChainStore, start types.TipSetKey, length uint64, opts *BSOptions) ([]*BSTipSet, error) {
+	var bstips []*BSTipSet
+	cur := start
+	for {
+		var bst BSTipSet
+		ts, err := cs.LoadTipSet(cur)
+		if err != nil {
+			return nil, xerrors.Errorf("failed loading tipset %s: %w", cur, err)
+		}
+
+		if opts.IncludeMessages {
+			bmsgs, bmincl, smsgs, smincl, err := gatherMessages(cs, ts)
+			if err != nil {
+				return nil, xerrors.Errorf("gather messages failed: %w", err)
+			}
+
+			bst.BlsMessages = bmsgs
+			bst.BlsMsgIncludes = bmincl
+			bst.SecpkMessages = smsgs
+			bst.SecpkMsgIncludes = smincl
+		}
+
+		if opts.IncludeBlocks {
+			bst.Blocks = ts.Blocks()
+		}
+
+		bstips = append(bstips, &bst)
+
+		if uint64(len(bstips)) >= length || ts.Height() == 0 {
+			return bstips, nil
+		}
+
+		cur = ts.Parents()
+	}
+}
+
+func gatherMessages(cs *store.ChainStore, ts *types.TipSet) ([]*types.Message, [][]uint64, []*types.SignedMessage, [][]uint64, error) {
+	blsmsgmap := make(map[cid.Cid]uint64)
+	secpkmsgmap := make(map[cid.Cid]uint64)
+	var secpkmsgs []*types.SignedMessage
+	var blsmsgs []*types.Message
+	var secpkincl, blsincl [][]uint64
+
+	for _, b := range ts.Blocks() {
+		bmsgs, smsgs, err := cs.MessagesForBlock(b)
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+
+		bmi := make([]uint64, 0, len(bmsgs))
+		for _, m := range bmsgs {
+			i, ok := blsmsgmap[m.Cid()]
+			if !ok {
+				i = uint64(len(blsmsgs))
+				blsmsgs = append(blsmsgs, m)
+				blsmsgmap[m.Cid()] = i
+			}
+
+			bmi = append(bmi, i)
+		}
+		blsincl = append(blsincl, bmi)
+
+		smi := make([]uint64, 0, len(smsgs))
+		for _, m := range smsgs {
+			i, ok := secpkmsgmap[m.Cid()]
+			if !ok {
+				i = uint64(len(secpkmsgs))
+				secpkmsgs = append(secpkmsgs, m)
+				secpkmsgmap[m.Cid()] = i
+			}
+
+			smi = append(smi, i)
+		}
+		secpkincl = append(secpkincl, smi)
+	}
+
+	return blsmsgs, blsincl, secpkmsgs, secpkincl, nil
+}
+
+func bstsToFullTipSet(bts *BSTipSet) (*store.FullTipSet, error) {
+	fts := &store.FullTipSet{}
+	for i, b := range bts.Blocks {
+		fb := &types.FullBlock{
+			Header: b,
+		}
+		for _, mi := range bts.BlsMsgIncludes[i] {
+			fb.BlsMessages = append(fb.BlsMessages, bts.BlsMessages[mi])
+		}
+		for _, mi := range bts.SecpkMsgIncludes[i] {
+			fb.SecpkMessages = append(fb.SecpkMessages, bts.SecpkMessages[mi])
+		}
+
+		fts.Blocks = append(fts.Blocks, fb)
+	}
+
+	return fts, nil
+}

chain/blocksync/blocksync_client.go

@@ -0,0 +1,603 @@
+package blocksync
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"math/rand"
+	"sort"
+	"sync"
+	"time"
+
+	blocks "github.com/ipfs/go-block-format"
+	bserv "github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
+	graphsync "github.com/ipfs/go-graphsync"
+	gsnet "github.com/ipfs/go-graphsync/network"
+	host "github.com/libp2p/go-libp2p-core/host"
+	inet "github.com/libp2p/go-libp2p-core/network"
+	"github.com/libp2p/go-libp2p-core/peer"
+	"go.opencensus.io/trace"
+	"golang.org/x/xerrors"
+
+	cborutil "github.com/filecoin-project/go-cbor-util"
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/types"
+	incrt "github.com/filecoin-project/lotus/lib/increadtimeout"
+	"github.com/filecoin-project/lotus/lib/peermgr"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+)
+
+type BlockSync struct {
+	bserv bserv.BlockService
+	gsync graphsync.GraphExchange
+	host  host.Host
+
+	syncPeers *bsPeerTracker
+	peerMgr   *peermgr.PeerMgr
+}
+
+func NewBlockSyncClient(bserv dtypes.ChainBlockService, h host.Host, pmgr peermgr.MaybePeerMgr, gs dtypes.Graphsync) *BlockSync {
+	return &BlockSync{
+		bserv:     bserv,
+		host:      h,
+		syncPeers: newPeerTracker(pmgr.Mgr),
+		peerMgr:   pmgr.Mgr,
+		gsync:     gs,
+	}
+}
+
+func (bs *BlockSync) processStatus(req *BlockSyncRequest, res *BlockSyncResponse) error {
+	switch res.Status {
+	case StatusPartial: // Partial Response
+		return xerrors.Errorf("not handling partial blocksync responses yet")
+	case StatusNotFound: // req.Start not found
+		return xerrors.Errorf("not found")
+	case StatusGoAway: // Go Away
+		return xerrors.Errorf("not handling 'go away' blocksync responses yet")
+	case StatusInternalError: // Internal Error
+		return xerrors.Errorf("block sync peer errored: %s", res.Message)
+	case StatusBadRequest:
+		return xerrors.Errorf("block sync request invalid: %s", res.Message)
+	default:
+		return xerrors.Errorf("unrecognized response code: %d", res.Status)
+	}
+}
+
+// GetBlocks fetches count blocks from the network, from the provided tipset
+// *backwards*, returning as many tipsets as count.
+//
+// {hint/usage}: This is used by the Syncer during normal chain syncing and when
+// resolving forks.
+func (bs *BlockSync) GetBlocks(ctx context.Context, tsk types.TipSetKey, count int) ([]*types.TipSet, error) {
+	ctx, span := trace.StartSpan(ctx, "bsync.GetBlocks")
+	defer span.End()
+	if span.IsRecordingEvents() {
+		span.AddAttributes(
+			trace.StringAttribute("tipset", fmt.Sprint(tsk.Cids())),
+			trace.Int64Attribute("count", int64(count)),
+		)
+	}
+
+	req := &BlockSyncRequest{
+		Start:         tsk.Cids(),
+		RequestLength: uint64(count),
+		Options:       BSOptBlocks,
+	}
+
+	// this peerset is sorted by latency and failure counting.
+	peers := bs.getPeers()
+
+	// randomize the first few peers so we don't always pick the same peer
+	shufflePrefix(peers)
+
+	start := build.Clock.Now()
+	var oerr error
+
+	for _, p := range peers {
+		// TODO: doing this synchronously isnt great, but fetching in parallel
+		// may not be a good idea either. think about this more
+		select {
+		case <-ctx.Done():
+			return nil, xerrors.Errorf("blocksync getblocks failed: %w", ctx.Err())
+		default:
+		}
+
+		res, err := bs.sendRequestToPeer(ctx, p, req)
+		if err != nil {
+			oerr = err
+			if !xerrors.Is(err, inet.ErrNoConn) {
+				log.Warnf("BlockSync request failed for peer %s: %s", p.String(), err)
+			}
+			continue
+		}
+
+		if res.Status == StatusOK || res.Status == StatusPartial {
+			resp, err := bs.processBlocksResponse(req, res)
+			if err != nil {
+				return nil, xerrors.Errorf("success response from peer failed to process: %w", err)
+			}
+			bs.syncPeers.logGlobalSuccess(build.Clock.Since(start))
+			bs.host.ConnManager().TagPeer(p, "bsync", 25)
+			return resp, nil
+		}
+
+		oerr = bs.processStatus(req, res)
+		if oerr != nil {
+			log.Warnf("BlockSync peer %s response was an error: %s", p.String(), oerr)
+		}
+	}
+	return nil, xerrors.Errorf("GetBlocks failed with all peers: %w", oerr)
+}
+
+func (bs *BlockSync) GetFullTipSet(ctx context.Context, p peer.ID, tsk types.TipSetKey) (*store.FullTipSet, error) {
+	// TODO: round robin through these peers on error
+
+	req := &BlockSyncRequest{
+		Start:         tsk.Cids(),
+		RequestLength: 1,
+		Options:       BSOptBlocks | BSOptMessages,
+	}
+
+	res, err := bs.sendRequestToPeer(ctx, p, req)
+	if err != nil {
+		return nil, err
+	}
+
+	switch res.Status {
+	case 0: // Success
+		if len(res.Chain) == 0 {
+			return nil, fmt.Errorf("got zero length chain response")
+		}
+		bts := res.Chain[0]
+
+		return bstsToFullTipSet(bts)
+	case 101: // Partial Response
+		return nil, xerrors.Errorf("partial responses are not handled for single tipset fetching")
+	case 201: // req.Start not found
+		return nil, fmt.Errorf("not found")
+	case 202: // Go Away
+		return nil, xerrors.Errorf("received 'go away' response peer")
+	case 203: // Internal Error
+		return nil, fmt.Errorf("block sync peer errored: %q", res.Message)
+	case 204: // Invalid Request
+		return nil, fmt.Errorf("block sync request invalid: %q", res.Message)
+	default:
+		return nil, fmt.Errorf("unrecognized response code")
+	}
+}
+
+func shufflePrefix(peers []peer.ID) {
+	pref := 5
+	if len(peers) < pref {
+		pref = len(peers)
+	}
+
+	buf := make([]peer.ID, pref)
+	perm := rand.Perm(pref)
+	for i, v := range perm {
+		buf[i] = peers[v]
+	}
+
+	copy(peers, buf)
+}
+
+func (bs *BlockSync) GetChainMessages(ctx context.Context, h *types.TipSet, count uint64) ([]*BSTipSet, error) {
+	ctx, span := trace.StartSpan(ctx, "GetChainMessages")
+	defer span.End()
+
+	peers := bs.getPeers()
+	// randomize the first few peers so we don't always pick the same peer
+	shufflePrefix(peers)
+
+	req := &BlockSyncRequest{
+		Start:         h.Cids(),
+		RequestLength: count,
+		Options:       BSOptMessages,
+	}
+
+	var err error
+	start := build.Clock.Now()
+
+	for _, p := range peers {
+		res, rerr := bs.sendRequestToPeer(ctx, p, req)
+		if rerr != nil {
+			err = rerr
+			log.Warnf("BlockSync request failed for peer %s: %s", p.String(), err)
+			continue
+		}
+
+		if res.Status == StatusOK {
+			bs.syncPeers.logGlobalSuccess(build.Clock.Since(start))
+			return res.Chain, nil
+		}
+
+		if res.Status == StatusPartial {
+			// TODO: track partial response sizes to ensure we don't overrequest too often
+			return res.Chain, nil
+		}
+
+		err = bs.processStatus(req, res)
+		if err != nil {
+			log.Warnf("BlockSync peer %s response was an error: %s", p.String(), err)
+		}
+	}
+
+	if err == nil {
+		return nil, xerrors.Errorf("GetChainMessages failed, no peers connected")
+	}
+
+	// TODO: What if we have no peers (and err is nil)?
+	return nil, xerrors.Errorf("GetChainMessages failed with all peers(%d): %w", len(peers), err)
+}
+
+func (bs *BlockSync) sendRequestToPeer(ctx context.Context, p peer.ID, req *BlockSyncRequest) (_ *BlockSyncResponse, err error) {
+	ctx, span := trace.StartSpan(ctx, "sendRequestToPeer")
+	defer span.End()
+
+	defer func() {
+		if err != nil {
+			if span.IsRecordingEvents() {
+				span.SetStatus(trace.Status{
+					Code:    5,
+					Message: err.Error(),
+				})
+			}
+		}
+	}()
+
+	if span.IsRecordingEvents() {
+		span.AddAttributes(
+			trace.StringAttribute("peer", p.Pretty()),
+		)
+	}
+
+	gsproto := string(gsnet.ProtocolGraphsync)
+	supp, err := bs.host.Peerstore().SupportsProtocols(p, BlockSyncProtocolID, gsproto)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to get protocols for peer: %w", err)
+	}
+
+	if len(supp) == 0 {
+		return nil, xerrors.Errorf("peer %s supports no known sync protocols", p)
+	}
+
+	switch supp[0] {
+	case BlockSyncProtocolID:
+		res, err := bs.fetchBlocksBlockSync(ctx, p, req)
+		if err != nil {
+			return nil, xerrors.Errorf("blocksync req failed: %w", err)
+		}
+		return res, nil
+	case gsproto:
+		res, err := bs.fetchBlocksGraphSync(ctx, p, req)
+		if err != nil {
+			return nil, xerrors.Errorf("graphsync req failed: %w", err)
+		}
+		return res, nil
+	default:
+		return nil, xerrors.Errorf("peerstore somehow returned unexpected protocols: %v", supp)
+	}
+
+}
+func (bs *BlockSync) fetchBlocksBlockSync(ctx context.Context, p peer.ID, req *BlockSyncRequest) (*BlockSyncResponse, error) {
+	ctx, span := trace.StartSpan(ctx, "blockSyncFetch")
+	defer span.End()
+
+	start := build.Clock.Now()
+	s, err := bs.host.NewStream(inet.WithNoDial(ctx, "should already have connection"), p, BlockSyncProtocolID)
+	if err != nil {
+		bs.RemovePeer(p)
+		return nil, xerrors.Errorf("failed to open stream to peer: %w", err)
+	}
+	_ = s.SetWriteDeadline(time.Now().Add(5 * time.Second)) // always use real time for socket/stream deadlines.
+
+	if err := cborutil.WriteCborRPC(s, req); err != nil {
+		_ = s.SetWriteDeadline(time.Time{})
+		bs.syncPeers.logFailure(p, build.Clock.Since(start))
+		return nil, err
+	}
+	_ = s.SetWriteDeadline(time.Time{})
+
+	var res BlockSyncResponse
+	r := incrt.New(s, 50<<10, 5*time.Second)
+	if err := cborutil.ReadCborRPC(bufio.NewReader(r), &res); err != nil {
+		bs.syncPeers.logFailure(p, build.Clock.Since(start))
+		return nil, err
+	}
+
+	if span.IsRecordingEvents() {
+		span.AddAttributes(
+			trace.Int64Attribute("resp_status", int64(res.Status)),
+			trace.StringAttribute("msg", res.Message),
+			trace.Int64Attribute("chain_len", int64(len(res.Chain))),
+		)
+	}
+
+	bs.syncPeers.logSuccess(p, build.Clock.Since(start))
+	return &res, nil
+}
+
+func (bs *BlockSync) processBlocksResponse(req *BlockSyncRequest, res *BlockSyncResponse) ([]*types.TipSet, error) {
+	if len(res.Chain) == 0 {
+		return nil, xerrors.Errorf("got no blocks in successful blocksync response")
+	}
+
+	cur, err := types.NewTipSet(res.Chain[0].Blocks)
+	if err != nil {
+		return nil, err
+	}
+
+	out := []*types.TipSet{cur}
+	for bi := 1; bi < len(res.Chain); bi++ {
+		next := res.Chain[bi].Blocks
+		nts, err := types.NewTipSet(next)
+		if err != nil {
+			return nil, err
+		}
+
+		if !types.CidArrsEqual(cur.Parents().Cids(), nts.Cids()) {
+			return nil, fmt.Errorf("parents of tipset[%d] were not tipset[%d]", bi-1, bi)
+		}
+
+		out = append(out, nts)
+		cur = nts
+	}
+	return out, nil
+}
+
+func (bs *BlockSync) GetBlock(ctx context.Context, c cid.Cid) (*types.BlockHeader, error) {
+	sb, err := bs.bserv.GetBlock(ctx, c)
+	if err != nil {
+		return nil, err
+	}
+
+	return types.DecodeBlock(sb.RawData())
+}
+
+func (bs *BlockSync) AddPeer(p peer.ID) {
+	bs.syncPeers.addPeer(p)
+}
+
+func (bs *BlockSync) RemovePeer(p peer.ID) {
+	bs.syncPeers.removePeer(p)
+}
+
+// getPeers returns a preference-sorted set of peers to query.
+func (bs *BlockSync) getPeers() []peer.ID {
+	return bs.syncPeers.prefSortedPeers()
+}
+
+func (bs *BlockSync) FetchMessagesByCids(ctx context.Context, cids []cid.Cid) ([]*types.Message, error) {
+	out := make([]*types.Message, len(cids))
+
+	err := bs.fetchCids(ctx, cids, func(i int, b blocks.Block) error {
+		msg, err := types.DecodeMessage(b.RawData())
+		if err != nil {
+			return err
+		}
+
+		if out[i] != nil {
+			return fmt.Errorf("received duplicate message")
+		}
+
+		out[i] = msg
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (bs *BlockSync) FetchSignedMessagesByCids(ctx context.Context, cids []cid.Cid) ([]*types.SignedMessage, error) {
+	out := make([]*types.SignedMessage, len(cids))
+
+	err := bs.fetchCids(ctx, cids, func(i int, b blocks.Block) error {
+		smsg, err := types.DecodeSignedMessage(b.RawData())
+		if err != nil {
+			return err
+		}
+
+		if out[i] != nil {
+			return fmt.Errorf("received duplicate message")
+		}
+
+		out[i] = smsg
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (bs *BlockSync) fetchCids(ctx context.Context, cids []cid.Cid, cb func(int, blocks.Block) error) error {
+	resp := bs.bserv.GetBlocks(context.TODO(), cids)
+
+	m := make(map[cid.Cid]int)
+	for i, c := range cids {
+		m[c] = i
+	}
+
+	for i := 0; i < len(cids); i++ {
+		select {
+		case v, ok := <-resp:
+			if !ok {
+				if i == len(cids)-1 {
+					break
+				}
+
+				return fmt.Errorf("failed to fetch all messages")
+			}
+
+			ix, ok := m[v.Cid()]
+			if !ok {
+				return fmt.Errorf("received message we didnt ask for")
+			}
+
+			if err := cb(ix, v); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+type peerStats struct {
+	successes   int
+	failures    int
+	firstSeen   time.Time
+	averageTime time.Duration
+}
+
+type bsPeerTracker struct {
+	lk sync.Mutex
+
+	peers         map[peer.ID]*peerStats
+	avgGlobalTime time.Duration
+
+	pmgr *peermgr.PeerMgr
+}
+
+func newPeerTracker(pmgr *peermgr.PeerMgr) *bsPeerTracker {
+	return &bsPeerTracker{
+		peers: make(map[peer.ID]*peerStats),
+		pmgr:  pmgr,
+	}
+}
+
+func (bpt *bsPeerTracker) addPeer(p peer.ID) {
+	bpt.lk.Lock()
+	defer bpt.lk.Unlock()
+	if _, ok := bpt.peers[p]; ok {
+		return
+	}
+	bpt.peers[p] = &peerStats{
+		firstSeen: build.Clock.Now(),
+	}
+
+}
+
+const (
+	// newPeerMul is how much better than average is the new peer assumed to be
+	// less than one to encourouge trying new peers
+	newPeerMul = 0.9
+)
+
+func (bpt *bsPeerTracker) prefSortedPeers() []peer.ID {
+	// TODO: this could probably be cached, but as long as its not too many peers, fine for now
+	bpt.lk.Lock()
+	defer bpt.lk.Unlock()
+	out := make([]peer.ID, 0, len(bpt.peers))
+	for p := range bpt.peers {
+		out = append(out, p)
+	}
+
+	// sort by 'expected cost' of requesting data from that peer
+	// additionally handle edge cases where not enough data is available
+	sort.Slice(out, func(i, j int) bool {
+		pi := bpt.peers[out[i]]
+		pj := bpt.peers[out[j]]
+
+		var costI, costJ float64
+
+		getPeerInitLat := func(p peer.ID) float64 {
+			var res float64
+			if bpt.pmgr != nil {
+				if lat, ok := bpt.pmgr.GetPeerLatency(p); ok {
+					res = float64(lat)
+				}
+			}
+			if res == 0 {
+				res = float64(bpt.avgGlobalTime)
+			}
+			return res * newPeerMul
+		}
+
+		if pi.successes+pi.failures > 0 {
+			failRateI := float64(pi.failures) / float64(pi.failures+pi.successes)
+			costI = float64(pi.averageTime) + failRateI*float64(bpt.avgGlobalTime)
+		} else {
+			costI = getPeerInitLat(out[i])
+		}
+
+		if pj.successes+pj.failures > 0 {
+			failRateJ := float64(pj.failures) / float64(pj.failures+pj.successes)
+			costJ = float64(pj.averageTime) + failRateJ*float64(bpt.avgGlobalTime)
+		} else {
+			costJ = getPeerInitLat(out[j])
+		}
+
+		return costI < costJ
+	})
+
+	return out
+}
+
+const (
+	// xInvAlpha = (N+1)/2
+
+	localInvAlpha  = 5  // 86% of the value is the last 9
+	globalInvAlpha = 20 // 86% of the value is the last 39
+)
+
+func (bpt *bsPeerTracker) logGlobalSuccess(dur time.Duration) {
+	bpt.lk.Lock()
+	defer bpt.lk.Unlock()
+
+	if bpt.avgGlobalTime == 0 {
+		bpt.avgGlobalTime = dur
+		return
+	}
+	delta := (dur - bpt.avgGlobalTime) / globalInvAlpha
+	bpt.avgGlobalTime += delta
+}
+
+func logTime(pi *peerStats, dur time.Duration) {
+	if pi.averageTime == 0 {
+		pi.averageTime = dur
+		return
+	}
+	delta := (dur - pi.averageTime) / localInvAlpha
+	pi.averageTime += delta
+
+}
+
+func (bpt *bsPeerTracker) logSuccess(p peer.ID, dur time.Duration) {
+	bpt.lk.Lock()
+	defer bpt.lk.Unlock()
+
+	var pi *peerStats
+	var ok bool
+	if pi, ok = bpt.peers[p]; !ok {
+		log.Warnw("log success called on peer not in tracker", "peerid", p.String())
+		return
+	}
+
+	pi.successes++
+	logTime(pi, dur)
+}
+
+func (bpt *bsPeerTracker) logFailure(p peer.ID, dur time.Duration) {
+	bpt.lk.Lock()
+	defer bpt.lk.Unlock()
+
+	var pi *peerStats
+	var ok bool
+	if pi, ok = bpt.peers[p]; !ok {
+		log.Warn("log failure called on peer not in tracker", "peerid", p.String())
+		return
+	}
+
+	pi.failures++
+	logTime(pi, dur)
+}
+
+func (bpt *bsPeerTracker) removePeer(p peer.ID) {
+	bpt.lk.Lock()
+	defer bpt.lk.Unlock()
+	delete(bpt.peers, p)
+}

chain/blocksync/cbor_gen.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"io"
 
-	types "github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/types"
 	cid "github.com/ipfs/go-cid"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	xerrors "golang.org/x/xerrors"
@@ -14,36 +14,36 @@ import (
 
 var _ = xerrors.Errorf
 
-var lengthBufRequest = []byte{131}
+var lengthBufBlockSyncRequest = []byte{131}
 
-func (t *Request) MarshalCBOR(w io.Writer) error {
+func (t *BlockSyncRequest) MarshalCBOR(w io.Writer) error {
 	if t == nil {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufRequest); err != nil {
+	if _, err := w.Write(lengthBufBlockSyncRequest); err != nil {
 		return err
 	}
 
 	scratch := make([]byte, 9)
 
-	// t.Head ([]cid.Cid) (slice)
-	if len(t.Head) > cbg.MaxLength {
-		return xerrors.Errorf("Slice value in field t.Head was too long")
+	// t.Start ([]cid.Cid) (slice)
+	if len(t.Start) > cbg.MaxLength {
+		return xerrors.Errorf("Slice value in field t.Start was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Head))); err != nil {
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Start))); err != nil {
 		return err
 	}
-	for _, v := range t.Head {
+	for _, v := range t.Start {
 		if err := cbg.WriteCidBuf(scratch, w, v); err != nil {
-			return xerrors.Errorf("failed writing cid field t.Head: %w", err)
+			return xerrors.Errorf("failed writing cid field t.Start: %w", err)
 		}
 	}
 
-	// t.Length (uint64) (uint64)
+	// t.RequestLength (uint64) (uint64)
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Length)); err != nil {
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.RequestLength)); err != nil {
 		return err
 	}
 
@@ -56,8 +56,8 @@ func (t *Request) MarshalCBOR(w io.Writer) error {
 	return nil
 }
 
-func (t *Request) UnmarshalCBOR(r io.Reader) error {
-	*t = Request{}
+func (t *BlockSyncRequest) UnmarshalCBOR(r io.Reader) error {
+	*t = BlockSyncRequest{}
 
 	br := cbg.GetPeeker(r)
 	scratch := make([]byte, 8)
@@ -74,7 +74,7 @@ func (t *Request) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("cbor input had wrong number of fields")
 	}
 
-	// t.Head ([]cid.Cid) (slice)
+	// t.Start ([]cid.Cid) (slice)
 
 	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
 	if err != nil {
@@ -82,7 +82,7 @@ func (t *Request) UnmarshalCBOR(r io.Reader) error {
 	}
 
 	if extra > cbg.MaxLength {
-		return fmt.Errorf("t.Head: array too large (%d)", extra)
+		return fmt.Errorf("t.Start: array too large (%d)", extra)
 	}
 
 	if maj != cbg.MajArray {
@@ -90,19 +90,19 @@ func (t *Request) UnmarshalCBOR(r io.Reader) error {
 	}
 
 	if extra > 0 {
-		t.Head = make([]cid.Cid, extra)
+		t.Start = make([]cid.Cid, extra)
 	}
 
 	for i := 0; i < int(extra); i++ {
 
 		c, err := cbg.ReadCid(br)
 		if err != nil {
-			return xerrors.Errorf("reading cid field t.Head failed: %w", err)
+			return xerrors.Errorf("reading cid field t.Start failed: %w", err)
 		}
-		t.Head[i] = c
+		t.Start[i] = c
 	}
 
-	// t.Length (uint64) (uint64)
+	// t.RequestLength (uint64) (uint64)
 
 	{
 
@@ -113,7 +113,7 @@ func (t *Request) UnmarshalCBOR(r io.Reader) error {
 		if maj != cbg.MajUnsignedInt {
 			return fmt.Errorf("wrong type for uint64 field")
 		}
-		t.Length = uint64(extra)
+		t.RequestLength = uint64(extra)
 
 	}
 	// t.Options (uint64) (uint64)
@@ -133,37 +133,19 @@ func (t *Request) UnmarshalCBOR(r io.Reader) error {
 	return nil
 }
 
-var lengthBufResponse = []byte{131}
+var lengthBufBlockSyncResponse = []byte{131}
 
-func (t *Response) MarshalCBOR(w io.Writer) error {
+func (t *BlockSyncResponse) MarshalCBOR(w io.Writer) error {
 	if t == nil {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufResponse); err != nil {
+	if _, err := w.Write(lengthBufBlockSyncResponse); err != nil {
 		return err
 	}
 
 	scratch := make([]byte, 9)
 
-	// t.Status (blocksync.status) (uint64)
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Status)); err != nil {
-		return err
-	}
-
-	// t.ErrorMessage (string) (string)
-	if len(t.ErrorMessage) > cbg.MaxLength {
-		return xerrors.Errorf("Value in field t.ErrorMessage was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.ErrorMessage))); err != nil {
-		return err
-	}
-	if _, err := io.WriteString(w, string(t.ErrorMessage)); err != nil {
-		return err
-	}
-
 	// t.Chain ([]*blocksync.BSTipSet) (slice)
 	if len(t.Chain) > cbg.MaxLength {
 		return xerrors.Errorf("Slice value in field t.Chain was too long")
@@ -177,11 +159,29 @@ func (t *Response) MarshalCBOR(w io.Writer) error {
 			return err
 		}
 	}
+
+	// t.Status (uint64) (uint64)
+
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Status)); err != nil {
+		return err
+	}
+
+	// t.Message (string) (string)
+	if len(t.Message) > cbg.MaxLength {
+		return xerrors.Errorf("Value in field t.Message was too long")
+	}
+
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Message))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, t.Message); err != nil {
+		return err
+	}
 	return nil
 }
 
-func (t *Response) UnmarshalCBOR(r io.Reader) error {
-	*t = Response{}
+func (t *BlockSyncResponse) UnmarshalCBOR(r io.Reader) error {
+	*t = BlockSyncResponse{}
 
 	br := cbg.GetPeeker(r)
 	scratch := make([]byte, 8)
@@ -198,30 +198,6 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("cbor input had wrong number of fields")
 	}
 
-	// t.Status (blocksync.status) (uint64)
-
-	{
-
-		maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
-		if err != nil {
-			return err
-		}
-		if maj != cbg.MajUnsignedInt {
-			return fmt.Errorf("wrong type for uint64 field")
-		}
-		t.Status = status(extra)
-
-	}
-	// t.ErrorMessage (string) (string)
-
-	{
-		sval, err := cbg.ReadStringBuf(br, scratch)
-		if err != nil {
-			return err
-		}
-
-		t.ErrorMessage = string(sval)
-	}
 	// t.Chain ([]*blocksync.BSTipSet) (slice)
 
 	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
@@ -251,296 +227,34 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error {
 		t.Chain[i] = &v
 	}
 
-	return nil
-}
+	// t.Status (uint64) (uint64)
 
-var lengthBufCompactedMessages = []byte{132}
+	{
 
-func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write(lengthBufCompactedMessages); err != nil {
-		return err
-	}
-
-	scratch := make([]byte, 9)
-
-	// t.Bls ([]*types.Message) (slice)
-	if len(t.Bls) > cbg.MaxLength {
-		return xerrors.Errorf("Slice value in field t.Bls was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Bls))); err != nil {
-		return err
-	}
-	for _, v := range t.Bls {
-		if err := v.MarshalCBOR(w); err != nil {
+		maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+		if err != nil {
 			return err
 		}
-	}
-
-	// t.BlsIncludes ([][]uint64) (slice)
-	if len(t.BlsIncludes) > cbg.MaxLength {
-		return xerrors.Errorf("Slice value in field t.BlsIncludes was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.BlsIncludes))); err != nil {
-		return err
-	}
-	for _, v := range t.BlsIncludes {
-		if len(v) > cbg.MaxLength {
-			return xerrors.Errorf("Slice value in field v was too long")
+		if maj != cbg.MajUnsignedInt {
+			return fmt.Errorf("wrong type for uint64 field")
 		}
+		t.Status = uint64(extra)
 
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(v))); err != nil {
+	}
+	// t.Message (string) (string)
+
+	{
+		sval, err := cbg.ReadStringBuf(br, scratch)
+		if err != nil {
 			return err
 		}
-		for _, v := range v {
-			if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, uint64(v)); err != nil {
-				return err
-			}
-		}
-	}
 
-	// t.Secpk ([]*types.SignedMessage) (slice)
-	if len(t.Secpk) > cbg.MaxLength {
-		return xerrors.Errorf("Slice value in field t.Secpk was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Secpk))); err != nil {
-		return err
-	}
-	for _, v := range t.Secpk {
-		if err := v.MarshalCBOR(w); err != nil {
-			return err
-		}
-	}
-
-	// t.SecpkIncludes ([][]uint64) (slice)
-	if len(t.SecpkIncludes) > cbg.MaxLength {
-		return xerrors.Errorf("Slice value in field t.SecpkIncludes was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.SecpkIncludes))); err != nil {
-		return err
-	}
-	for _, v := range t.SecpkIncludes {
-		if len(v) > cbg.MaxLength {
-			return xerrors.Errorf("Slice value in field v was too long")
-		}
-
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(v))); err != nil {
-			return err
-		}
-		for _, v := range v {
-			if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, uint64(v)); err != nil {
-				return err
-			}
-		}
+		t.Message = string(sval)
 	}
 	return nil
 }
 
-func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error {
-	*t = CompactedMessages{}
-
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
-
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 4 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.Bls ([]*types.Message) (slice)
-
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
-	if err != nil {
-		return err
-	}
-
-	if extra > cbg.MaxLength {
-		return fmt.Errorf("t.Bls: array too large (%d)", extra)
-	}
-
-	if maj != cbg.MajArray {
-		return fmt.Errorf("expected cbor array")
-	}
-
-	if extra > 0 {
-		t.Bls = make([]*types.Message, extra)
-	}
-
-	for i := 0; i < int(extra); i++ {
-
-		var v types.Message
-		if err := v.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-		t.Bls[i] = &v
-	}
-
-	// t.BlsIncludes ([][]uint64) (slice)
-
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
-	if err != nil {
-		return err
-	}
-
-	if extra > cbg.MaxLength {
-		return fmt.Errorf("t.BlsIncludes: array too large (%d)", extra)
-	}
-
-	if maj != cbg.MajArray {
-		return fmt.Errorf("expected cbor array")
-	}
-
-	if extra > 0 {
-		t.BlsIncludes = make([][]uint64, extra)
-	}
-
-	for i := 0; i < int(extra); i++ {
-		{
-			var maj byte
-			var extra uint64
-			var err error
-
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
-			if err != nil {
-				return err
-			}
-
-			if extra > cbg.MaxLength {
-				return fmt.Errorf("t.BlsIncludes[i]: array too large (%d)", extra)
-			}
-
-			if maj != cbg.MajArray {
-				return fmt.Errorf("expected cbor array")
-			}
-
-			if extra > 0 {
-				t.BlsIncludes[i] = make([]uint64, extra)
-			}
-
-			for j := 0; j < int(extra); j++ {
-
-				maj, val, err := cbg.CborReadHeaderBuf(br, scratch)
-				if err != nil {
-					return xerrors.Errorf("failed to read uint64 for t.BlsIncludes[i] slice: %w", err)
-				}
-
-				if maj != cbg.MajUnsignedInt {
-					return xerrors.Errorf("value read for array t.BlsIncludes[i] was not a uint, instead got %d", maj)
-				}
-
-				t.BlsIncludes[i][j] = uint64(val)
-			}
-
-		}
-	}
-
-	// t.Secpk ([]*types.SignedMessage) (slice)
-
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
-	if err != nil {
-		return err
-	}
-
-	if extra > cbg.MaxLength {
-		return fmt.Errorf("t.Secpk: array too large (%d)", extra)
-	}
-
-	if maj != cbg.MajArray {
-		return fmt.Errorf("expected cbor array")
-	}
-
-	if extra > 0 {
-		t.Secpk = make([]*types.SignedMessage, extra)
-	}
-
-	for i := 0; i < int(extra); i++ {
-
-		var v types.SignedMessage
-		if err := v.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-		t.Secpk[i] = &v
-	}
-
-	// t.SecpkIncludes ([][]uint64) (slice)
-
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
-	if err != nil {
-		return err
-	}
-
-	if extra > cbg.MaxLength {
-		return fmt.Errorf("t.SecpkIncludes: array too large (%d)", extra)
-	}
-
-	if maj != cbg.MajArray {
-		return fmt.Errorf("expected cbor array")
-	}
-
-	if extra > 0 {
-		t.SecpkIncludes = make([][]uint64, extra)
-	}
-
-	for i := 0; i < int(extra); i++ {
-		{
-			var maj byte
-			var extra uint64
-			var err error
-
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
-			if err != nil {
-				return err
-			}
-
-			if extra > cbg.MaxLength {
-				return fmt.Errorf("t.SecpkIncludes[i]: array too large (%d)", extra)
-			}
-
-			if maj != cbg.MajArray {
-				return fmt.Errorf("expected cbor array")
-			}
-
-			if extra > 0 {
-				t.SecpkIncludes[i] = make([]uint64, extra)
-			}
-
-			for j := 0; j < int(extra); j++ {
-
-				maj, val, err := cbg.CborReadHeaderBuf(br, scratch)
-				if err != nil {
-					return xerrors.Errorf("failed to read uint64 for t.SecpkIncludes[i] slice: %w", err)
-				}
-
-				if maj != cbg.MajUnsignedInt {
-					return xerrors.Errorf("value read for array t.SecpkIncludes[i] was not a uint, instead got %d", maj)
-				}
-
-				t.SecpkIncludes[i][j] = uint64(val)
-			}
-
-		}
-	}
-
-	return nil
-}
-
-var lengthBufBSTipSet = []byte{130}
+var lengthBufBSTipSet = []byte{133}
 
 func (t *BSTipSet) MarshalCBOR(w io.Writer) error {
 	if t == nil {
@@ -567,10 +281,79 @@ func (t *BSTipSet) MarshalCBOR(w io.Writer) error {
 		}
 	}
 
-	// t.Messages (blocksync.CompactedMessages) (struct)
-	if err := t.Messages.MarshalCBOR(w); err != nil {
+	// t.BlsMessages ([]*types.Message) (slice)
+	if len(t.BlsMessages) > cbg.MaxLength {
+		return xerrors.Errorf("Slice value in field t.BlsMessages was too long")
+	}
+
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.BlsMessages))); err != nil {
 		return err
 	}
+	for _, v := range t.BlsMessages {
+		if err := v.MarshalCBOR(w); err != nil {
+			return err
+		}
+	}
+
+	// t.BlsMsgIncludes ([][]uint64) (slice)
+	if len(t.BlsMsgIncludes) > cbg.MaxLength {
+		return xerrors.Errorf("Slice value in field t.BlsMsgIncludes was too long")
+	}
+
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.BlsMsgIncludes))); err != nil {
+		return err
+	}
+	for _, v := range t.BlsMsgIncludes {
+		if len(v) > cbg.MaxLength {
+			return xerrors.Errorf("Slice value in field v was too long")
+		}
+
+		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(v))); err != nil {
+			return err
+		}
+		for _, v := range v {
+			if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, uint64(v)); err != nil {
+				return err
+			}
+		}
+	}
+
+	// t.SecpkMessages ([]*types.SignedMessage) (slice)
+	if len(t.SecpkMessages) > cbg.MaxLength {
+		return xerrors.Errorf("Slice value in field t.SecpkMessages was too long")
+	}
+
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.SecpkMessages))); err != nil {
+		return err
+	}
+	for _, v := range t.SecpkMessages {
+		if err := v.MarshalCBOR(w); err != nil {
+			return err
+		}
+	}
+
+	// t.SecpkMsgIncludes ([][]uint64) (slice)
+	if len(t.SecpkMsgIncludes) > cbg.MaxLength {
+		return xerrors.Errorf("Slice value in field t.SecpkMsgIncludes was too long")
+	}
+
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.SecpkMsgIncludes))); err != nil {
+		return err
+	}
+	for _, v := range t.SecpkMsgIncludes {
+		if len(v) > cbg.MaxLength {
+			return xerrors.Errorf("Slice value in field v was too long")
+		}
+
+		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(v))); err != nil {
+			return err
+		}
+		for _, v := range v {
+			if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, uint64(v)); err != nil {
+				return err
+			}
+		}
+	}
 	return nil
 }
 
@@ -588,7 +371,7 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("cbor input should be of type array")
 	}
 
-	if extra != 2 {
+	if extra != 5 {
 		return fmt.Errorf("cbor input had wrong number of fields")
 	}
 
@@ -621,24 +404,181 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error {
 		t.Blocks[i] = &v
 	}
 
-	// t.Messages (blocksync.CompactedMessages) (struct)
+	// t.BlsMessages ([]*types.Message) (slice)
 
-	{
+	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	if err != nil {
+		return err
+	}
 
-		b, err := br.ReadByte()
-		if err != nil {
+	if extra > cbg.MaxLength {
+		return fmt.Errorf("t.BlsMessages: array too large (%d)", extra)
+	}
+
+	if maj != cbg.MajArray {
+		return fmt.Errorf("expected cbor array")
+	}
+
+	if extra > 0 {
+		t.BlsMessages = make([]*types.Message, extra)
+	}
+
+	for i := 0; i < int(extra); i++ {
+
+		var v types.Message
+		if err := v.UnmarshalCBOR(br); err != nil {
 			return err
 		}
-		if b != cbg.CborNull[0] {
-			if err := br.UnreadByte(); err != nil {
+
+		t.BlsMessages[i] = &v
+	}
+
+	// t.BlsMsgIncludes ([][]uint64) (slice)
+
+	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	if err != nil {
+		return err
+	}
+
+	if extra > cbg.MaxLength {
+		return fmt.Errorf("t.BlsMsgIncludes: array too large (%d)", extra)
+	}
+
+	if maj != cbg.MajArray {
+		return fmt.Errorf("expected cbor array")
+	}
+
+	if extra > 0 {
+		t.BlsMsgIncludes = make([][]uint64, extra)
+	}
+
+	for i := 0; i < int(extra); i++ {
+		{
+			var maj byte
+			var extra uint64
+			var err error
+
+			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			if err != nil {
 				return err
 			}
-			t.Messages = new(CompactedMessages)
-			if err := t.Messages.UnmarshalCBOR(br); err != nil {
-				return xerrors.Errorf("unmarshaling t.Messages pointer: %w", err)
+
+			if extra > cbg.MaxLength {
+				return fmt.Errorf("t.BlsMsgIncludes[i]: array too large (%d)", extra)
 			}
+
+			if maj != cbg.MajArray {
+				return fmt.Errorf("expected cbor array")
+			}
+
+			if extra > 0 {
+				t.BlsMsgIncludes[i] = make([]uint64, extra)
+			}
+
+			for j := 0; j < int(extra); j++ {
+
+				maj, val, err := cbg.CborReadHeaderBuf(br, scratch)
+				if err != nil {
+					return xerrors.Errorf("failed to read uint64 for t.BlsMsgIncludes[i] slice: %w", err)
+				}
+
+				if maj != cbg.MajUnsignedInt {
+					return xerrors.Errorf("value read for array t.BlsMsgIncludes[i] was not a uint, instead got %d", maj)
+				}
+
+				t.BlsMsgIncludes[i][j] = uint64(val)
+			}
+
+		}
+	}
+
+	// t.SecpkMessages ([]*types.SignedMessage) (slice)
+
+	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	if err != nil {
+		return err
+	}
+
+	if extra > cbg.MaxLength {
+		return fmt.Errorf("t.SecpkMessages: array too large (%d)", extra)
+	}
+
+	if maj != cbg.MajArray {
+		return fmt.Errorf("expected cbor array")
+	}
+
+	if extra > 0 {
+		t.SecpkMessages = make([]*types.SignedMessage, extra)
+	}
+
+	for i := 0; i < int(extra); i++ {
+
+		var v types.SignedMessage
+		if err := v.UnmarshalCBOR(br); err != nil {
+			return err
 		}
 
+		t.SecpkMessages[i] = &v
 	}
+
+	// t.SecpkMsgIncludes ([][]uint64) (slice)
+
+	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	if err != nil {
+		return err
+	}
+
+	if extra > cbg.MaxLength {
+		return fmt.Errorf("t.SecpkMsgIncludes: array too large (%d)", extra)
+	}
+
+	if maj != cbg.MajArray {
+		return fmt.Errorf("expected cbor array")
+	}
+
+	if extra > 0 {
+		t.SecpkMsgIncludes = make([][]uint64, extra)
+	}
+
+	for i := 0; i < int(extra); i++ {
+		{
+			var maj byte
+			var extra uint64
+			var err error
+
+			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			if err != nil {
+				return err
+			}
+
+			if extra > cbg.MaxLength {
+				return fmt.Errorf("t.SecpkMsgIncludes[i]: array too large (%d)", extra)
+			}
+
+			if maj != cbg.MajArray {
+				return fmt.Errorf("expected cbor array")
+			}
+
+			if extra > 0 {
+				t.SecpkMsgIncludes[i] = make([]uint64, extra)
+			}
+
+			for j := 0; j < int(extra); j++ {
+
+				maj, val, err := cbg.CborReadHeaderBuf(br, scratch)
+				if err != nil {
+					return xerrors.Errorf("failed to read uint64 for t.SecpkMsgIncludes[i] slice: %w", err)
+				}
+
+				if maj != cbg.MajUnsignedInt {
+					return xerrors.Errorf("value read for array t.SecpkMsgIncludes[i] was not a uint, instead got %d", maj)
+				}
+
+				t.SecpkMsgIncludes[i][j] = uint64(val)
+			}
+
+		}
+	}
+
 	return nil
 }

chain/blocksync/client.go

@@ -1,452 +0,0 @@
-package blocksync
-
-import (
-	"bufio"
-	"context"
-	"fmt"
-	"math/rand"
-	"time"
-
-	host "github.com/libp2p/go-libp2p-core/host"
-	inet "github.com/libp2p/go-libp2p-core/network"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"go.opencensus.io/trace"
-	"golang.org/x/xerrors"
-
-	cborutil "github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/store"
-	"github.com/filecoin-project/lotus/chain/types"
-	incrt "github.com/filecoin-project/lotus/lib/increadtimeout"
-	"github.com/filecoin-project/lotus/lib/peermgr"
-)
-
-// Protocol client.
-// FIXME: Rename to just `Client`. Not done at the moment to avoid
-//  disrupting too much of the consumer code, should be done along
-//  https://github.com/filecoin-project/lotus/issues/2612.
-type BlockSync struct {
-	// Connection manager used to contact the server.
-	// FIXME: We should have a reduced interface here, initialized
-	//  just with our protocol ID, we shouldn't be able to open *any*
-	//  connection.
-	host host.Host
-
-	peerTracker *bsPeerTracker
-}
-
-func NewClient(
-	host host.Host,
-	pmgr peermgr.MaybePeerMgr,
-) *BlockSync {
-	return &BlockSync{
-		host:        host,
-		peerTracker: newPeerTracker(pmgr.Mgr),
-	}
-}
-
-// Main logic of the client request service. The provided `Request`
-// is sent to the `singlePeer` if one is indicated or to all available
-// ones otherwise. The response is processed and validated according
-// to the `Request` options. Either a `validatedResponse` is returned
-// (which can be safely accessed), or an `error` that may represent
-// either a response error status, a failed validation or an internal
-// error.
-//
-// This is the internal single point of entry for all external-facing
-// APIs, currently we have 3 very heterogeneous services exposed:
-// * GetBlocks:         Headers
-// * GetFullTipSet:     Headers | Messages
-// * GetChainMessages:            Messages
-// This function handles all the different combinations of the available
-// request options without disrupting external calls. In the future the
-// consumers should be forced to use a more standardized service and
-// adhere to a single API derived from this function.
-func (client *BlockSync) doRequest(
-	ctx context.Context,
-	req *Request,
-	singlePeer *peer.ID,
-) (*validatedResponse, error) {
-	// Validate request.
-	if req.Length == 0 {
-		return nil, xerrors.Errorf("invalid request of length 0")
-	}
-	if req.Length > MaxRequestLength {
-		return nil, xerrors.Errorf("request length (%d) above maximum (%d)",
-			req.Length, MaxRequestLength)
-	}
-	if req.Options == 0 {
-		return nil, xerrors.Errorf("request with no options set")
-	}
-
-	// Generate the list of peers to be queried, either the
-	// `singlePeer` indicated or all peers available (sorted
-	// by an internal peer tracker with some randomness injected).
-	var peers []peer.ID
-	if singlePeer != nil {
-		peers = []peer.ID{*singlePeer}
-	} else {
-		peers = client.getShuffledPeers()
-		if len(peers) == 0 {
-			return nil, xerrors.Errorf("no peers available")
-		}
-	}
-
-	// Try the request for each peer in the list,
-	// return on the first successful response.
-	// FIXME: Doing this serially isn't great, but fetching in parallel
-	//  may not be a good idea either. Think about this more.
-	globalTime := build.Clock.Now()
-	// Global time used to track what is the expected time we will need to get
-	// a response if a client fails us.
-	for _, peer := range peers {
-		select {
-		case <-ctx.Done():
-			return nil, xerrors.Errorf("context cancelled: %w", ctx.Err())
-		default:
-		}
-
-		// Send request, read response.
-		res, err := client.sendRequestToPeer(ctx, peer, req)
-		if err != nil {
-			if !xerrors.Is(err, inet.ErrNoConn) {
-				log.Warnf("could not connect to peer %s: %s",
-					peer.String(), err)
-			}
-			continue
-		}
-
-		// Process and validate response.
-		validRes, err := client.processResponse(req, res)
-		if err != nil {
-			log.Warnf("processing peer %s response failed: %s",
-				peer.String(), err)
-			continue
-		}
-
-		client.peerTracker.logGlobalSuccess(build.Clock.Since(globalTime))
-		client.host.ConnManager().TagPeer(peer, "bsync", SUCCESS_PEER_TAG_VALUE)
-		return validRes, nil
-	}
-
-	errString := "doRequest failed for all peers"
-	if singlePeer != nil {
-		errString = fmt.Sprintf("doRequest failed for single peer %s", *singlePeer)
-	}
-	return nil, xerrors.Errorf(errString)
-}
-
-// Process and validate response. Check the status, the integrity of the
-// information returned, and that it matches the request. Extract the information
-// into a `validatedResponse` for the external-facing APIs to select what they
-// need.
-//
-// We are conflating in the single error returned both status and validation
-// errors. Peer penalization should happen here then, before returning, so
-// we can apply the correct penalties depending on the cause of the error.
-func (client *BlockSync) processResponse(
-	req *Request,
-	res *Response,
-	// FIXME: Add the `peer` as argument once we implement penalties.
-) (*validatedResponse, error) {
-	err := res.statusToError()
-	if err != nil {
-		return nil, xerrors.Errorf("status error: %s", err)
-	}
-
-	options := parseOptions(req.Options)
-	if options.noOptionsSet() {
-		// Safety check: this shouldn't have been sent, and even if it did
-		// it should have been caught by the peer in its error status.
-		return nil, xerrors.Errorf("nothing was requested")
-	}
-
-	// Verify that the chain segment returned is in the valid range.
-	// Note that the returned length might be less than requested.
-	resLength := len(res.Chain)
-	if resLength == 0 {
-		return nil, xerrors.Errorf("got no chain in successful response")
-	}
-	if resLength > int(req.Length) {
-		return nil, xerrors.Errorf("got longer response (%d) than requested (%d)",
-			resLength, req.Length)
-	}
-	if resLength < int(req.Length) && res.Status != Partial {
-		return nil, xerrors.Errorf("got less than requested without a proper status: %d", res.Status)
-	}
-
-	validRes := &validatedResponse{}
-	if options.IncludeHeaders {
-		// Check for valid block sets and extract them into `TipSet`s.
-		validRes.tipsets = make([]*types.TipSet, resLength)
-		for i := 0; i < resLength; i++ {
-			validRes.tipsets[i], err = types.NewTipSet(res.Chain[i].Blocks)
-			if err != nil {
-				return nil, xerrors.Errorf("invalid tipset blocks at height (head - %d): %w", i, err)
-			}
-		}
-
-		// Check that the returned head matches the one requested.
-		if !types.CidArrsEqual(validRes.tipsets[0].Cids(), req.Head) {
-			return nil, xerrors.Errorf("returned chain head does not match request")
-		}
-
-		// Check `TipSet`s are connected (valid chain).
-		for i := 0; i < len(validRes.tipsets)-1; i++ {
-			if validRes.tipsets[i].IsChildOf(validRes.tipsets[i+1]) == false {
-				return nil, fmt.Errorf("tipsets are not connected at height (head - %d)/(head - %d)",
-					i, i+1)
-				// FIXME: Maybe give more information here, like CIDs.
-			}
-		}
-	}
-
-	if options.IncludeMessages {
-		validRes.messages = make([]*CompactedMessages, resLength)
-		for i := 0; i < resLength; i++ {
-			if res.Chain[i].Messages == nil {
-				return nil, xerrors.Errorf("no messages included for tipset at height (head - %d)", i)
-			}
-			validRes.messages[i] = res.Chain[i].Messages
-		}
-
-		if options.IncludeHeaders {
-			// If the headers were also returned check that the compression
-			// indexes are valid before `toFullTipSets()` is called by the
-			// consumer.
-			for tipsetIdx := 0; tipsetIdx < resLength; tipsetIdx++ {
-				msgs := res.Chain[tipsetIdx].Messages
-				blocksNum := len(res.Chain[tipsetIdx].Blocks)
-				if len(msgs.BlsIncludes) != blocksNum {
-					return nil, xerrors.Errorf("BlsIncludes (%d) does not match number of blocks (%d)",
-						len(msgs.BlsIncludes), blocksNum)
-				}
-				if len(msgs.SecpkIncludes) != blocksNum {
-					return nil, xerrors.Errorf("SecpkIncludes (%d) does not match number of blocks (%d)",
-						len(msgs.SecpkIncludes), blocksNum)
-				}
-				for blockIdx := 0; blockIdx < blocksNum; blockIdx++ {
-					for _, mi := range msgs.BlsIncludes[blockIdx] {
-						if int(mi) >= len(msgs.Bls) {
-							return nil, xerrors.Errorf("index in BlsIncludes (%d) exceeds number of messages (%d)",
-								mi, len(msgs.Bls))
-						}
-					}
-					for _, mi := range msgs.SecpkIncludes[blockIdx] {
-						if int(mi) >= len(msgs.Secpk) {
-							return nil, xerrors.Errorf("index in SecpkIncludes (%d) exceeds number of messages (%d)",
-								mi, len(msgs.Secpk))
-						}
-					}
-				}
-			}
-		}
-	}
-
-	return validRes, nil
-}
-
-// GetBlocks fetches count blocks from the network, from the provided tipset
-// *backwards*, returning as many tipsets as count.
-//
-// {hint/usage}: This is used by the Syncer during normal chain syncing and when
-// resolving forks.
-func (client *BlockSync) GetBlocks(
-	ctx context.Context,
-	tsk types.TipSetKey,
-	count int,
-) ([]*types.TipSet, error) {
-	ctx, span := trace.StartSpan(ctx, "bsync.GetBlocks")
-	defer span.End()
-	if span.IsRecordingEvents() {
-		span.AddAttributes(
-			trace.StringAttribute("tipset", fmt.Sprint(tsk.Cids())),
-			trace.Int64Attribute("count", int64(count)),
-		)
-	}
-
-	req := &Request{
-		Head:    tsk.Cids(),
-		Length:  uint64(count),
-		Options: Headers,
-	}
-
-	validRes, err := client.doRequest(ctx, req, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	return validRes.tipsets, nil
-}
-
-func (client *BlockSync) GetFullTipSet(
-	ctx context.Context,
-	peer peer.ID,
-	tsk types.TipSetKey,
-) (*store.FullTipSet, error) {
-	// TODO: round robin through these peers on error
-
-	req := &Request{
-		Head:    tsk.Cids(),
-		Length:  1,
-		Options: Headers | Messages,
-	}
-
-	validRes, err := client.doRequest(ctx, req, &peer)
-	if err != nil {
-		return nil, err
-	}
-
-	return validRes.toFullTipSets()[0], nil
-	// If `doRequest` didn't fail we are guaranteed to have at least
-	//  *one* tipset here, so it's safe to index directly.
-}
-
-func (client *BlockSync) GetChainMessages(
-	ctx context.Context,
-	head *types.TipSet,
-	length uint64,
-) ([]*CompactedMessages, error) {
-	ctx, span := trace.StartSpan(ctx, "GetChainMessages")
-	if span.IsRecordingEvents() {
-		span.AddAttributes(
-			trace.StringAttribute("tipset", fmt.Sprint(head.Cids())),
-			trace.Int64Attribute("count", int64(length)),
-		)
-	}
-	defer span.End()
-
-	req := &Request{
-		Head:    head.Cids(),
-		Length:  length,
-		Options: Messages,
-	}
-
-	validRes, err := client.doRequest(ctx, req, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	return validRes.messages, nil
-}
-
-// Send a request to a peer. Write request in the stream and read the
-// response back. We do not do any processing of the request/response
-// here.
-func (client *BlockSync) sendRequestToPeer(
-	ctx context.Context,
-	peer peer.ID,
-	req *Request,
-) (_ *Response, err error) {
-	// Trace code.
-	ctx, span := trace.StartSpan(ctx, "sendRequestToPeer")
-	defer span.End()
-	if span.IsRecordingEvents() {
-		span.AddAttributes(
-			trace.StringAttribute("peer", peer.Pretty()),
-		)
-	}
-	defer func() {
-		if err != nil {
-			if span.IsRecordingEvents() {
-				span.SetStatus(trace.Status{
-					Code:    5,
-					Message: err.Error(),
-				})
-			}
-		}
-	}()
-	// -- TRACE --
-
-	supported, err := client.host.Peerstore().SupportsProtocols(peer, BlockSyncProtocolID)
-	if err != nil {
-		return nil, xerrors.Errorf("failed to get protocols for peer: %w", err)
-	}
-	if len(supported) == 0 || supported[0] != BlockSyncProtocolID {
-		return nil, xerrors.Errorf("peer %s does not support protocol %s",
-			peer, BlockSyncProtocolID)
-		// FIXME: `ProtoBook` should support a *single* protocol check that returns
-		//  a bool instead of a list.
-	}
-
-	connectionStart := build.Clock.Now()
-
-	// Open stream to peer.
-	stream, err := client.host.NewStream(
-		inet.WithNoDial(ctx, "should already have connection"),
-		peer,
-		BlockSyncProtocolID)
-	if err != nil {
-		client.RemovePeer(peer)
-		return nil, xerrors.Errorf("failed to open stream to peer: %w", err)
-	}
-
-	// Write request.
-	_ = stream.SetWriteDeadline(time.Now().Add(WRITE_REQ_DEADLINE))
-	if err := cborutil.WriteCborRPC(stream, req); err != nil {
-		_ = stream.SetWriteDeadline(time.Time{})
-		client.peerTracker.logFailure(peer, build.Clock.Since(connectionStart))
-		// FIXME: Should we also remove peer here?
-		return nil, err
-	}
-	_ = stream.SetWriteDeadline(time.Time{}) // clear deadline // FIXME: Needs
-	//  its own API (https://github.com/libp2p/go-libp2p-core/issues/162).
-
-	// Read response.
-	var res Response
-	err = cborutil.ReadCborRPC(
-		bufio.NewReader(incrt.New(stream, READ_RES_MIN_SPEED, READ_RES_DEADLINE)),
-		&res)
-	if err != nil {
-		client.peerTracker.logFailure(peer, build.Clock.Since(connectionStart))
-		return nil, xerrors.Errorf("failed to read blocksync response: %w", err)
-	}
-
-	// FIXME: Move all this together at the top using a defer as done elsewhere.
-	//  Maybe we need to declare `res` in the signature.
-	if span.IsRecordingEvents() {
-		span.AddAttributes(
-			trace.Int64Attribute("resp_status", int64(res.Status)),
-			trace.StringAttribute("msg", res.ErrorMessage),
-			trace.Int64Attribute("chain_len", int64(len(res.Chain))),
-		)
-	}
-
-	client.peerTracker.logSuccess(peer, build.Clock.Since(connectionStart))
-	// FIXME: We should really log a success only after we validate the response.
-	//  It might be a bit hard to do.
-	return &res, nil
-}
-
-func (client *BlockSync) AddPeer(p peer.ID) {
-	client.peerTracker.addPeer(p)
-}
-
-func (client *BlockSync) RemovePeer(p peer.ID) {
-	client.peerTracker.removePeer(p)
-}
-
-// getShuffledPeers returns a preference-sorted set of peers (by latency
-// and failure counting), shuffling the first few peers so we don't always
-// pick the same peer.
-// FIXME: Consider merging with `shufflePrefix()s`.
-func (client *BlockSync) getShuffledPeers() []peer.ID {
-	peers := client.peerTracker.prefSortedPeers()
-	shufflePrefix(peers)
-	return peers
-}
-
-func shufflePrefix(peers []peer.ID) {
-	prefix := SHUFFLE_PEERS_PREFIX
-	if len(peers) < prefix {
-		prefix = len(peers)
-	}
-
-	buf := make([]peer.ID, prefix)
-	perm := rand.Perm(prefix)
-	for i, v := range perm {
-		buf[i] = peers[v]
-	}
-
-	copy(peers, buf)
-}

chain/blocksync/graphsync_client.go

@@ -0,0 +1,151 @@
+package blocksync
+
+import (
+	"context"
+
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-datastore"
+	"github.com/ipfs/go-graphsync"
+	"github.com/ipld/go-ipld-prime"
+	"github.com/libp2p/go-libp2p-core/peer"
+	"golang.org/x/xerrors"
+
+	store "github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/types"
+
+	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+	basicnode "github.com/ipld/go-ipld-prime/node/basic"
+	ipldselector "github.com/ipld/go-ipld-prime/traversal/selector"
+	selectorbuilder "github.com/ipld/go-ipld-prime/traversal/selector/builder"
+)
+
+const (
+
+	// AMT selector recursion. An AMT has arity of 8 so this gives allows
+	// us to retrieve trees with 8^10 (1,073,741,824) elements.
+	amtRecursionDepth = uint32(10)
+
+	// some constants for looking up tuple encoded struct fields
+	// field index of Parents field in a block header
+	blockIndexParentsField = 5
+
+	// field index of Messages field in a block header
+	blockIndexMessagesField = 10
+
+	// field index of AMT node in AMT head
+	amtHeadNodeFieldIndex = 2
+
+	// field index of links array AMT node
+	amtNodeLinksFieldIndex = 1
+
+	// field index of values array AMT node
+	amtNodeValuesFieldIndex = 2
+
+	// maximum depth per traversal
+	maxRequestLength = 50
+)
+
+var amtSelector selectorbuilder.SelectorSpec
+
+func init() {
+	// builer for selectors
+	ssb := selectorbuilder.NewSelectorSpecBuilder(basicnode.Style.Any)
+	// amt selector -- needed to selector through a messages AMT
+	amtSelector = ssb.ExploreIndex(amtHeadNodeFieldIndex,
+		ssb.ExploreRecursive(ipldselector.RecursionLimitDepth(int(amtRecursionDepth)),
+			ssb.ExploreUnion(
+				ssb.ExploreIndex(amtNodeLinksFieldIndex,
+					ssb.ExploreAll(ssb.ExploreRecursiveEdge())),
+				ssb.ExploreIndex(amtNodeValuesFieldIndex,
+					ssb.ExploreAll(ssb.Matcher())))))
+}
+
+func selectorForRequest(req *BlockSyncRequest) ipld.Node {
+	// builer for selectors
+	ssb := selectorbuilder.NewSelectorSpecBuilder(basicnode.Style.Any)
+
+	bso := ParseBSOptions(req.Options)
+	if bso.IncludeMessages {
+		return ssb.ExploreRecursive(ipldselector.RecursionLimitDepth(int(req.RequestLength)),
+			ssb.ExploreIndex(blockIndexParentsField,
+				ssb.ExploreUnion(
+					ssb.ExploreAll(
+						ssb.ExploreIndex(blockIndexMessagesField,
+							ssb.ExploreRange(0, 2, amtSelector),
+						)),
+					ssb.ExploreIndex(0, ssb.ExploreRecursiveEdge()),
+				))).Node()
+	}
+	return ssb.ExploreRecursive(ipldselector.RecursionLimitDepth(int(req.RequestLength)), ssb.ExploreIndex(blockIndexParentsField,
+		ssb.ExploreUnion(
+			ssb.ExploreAll(
+				ssb.Matcher(),
+			),
+			ssb.ExploreIndex(0, ssb.ExploreRecursiveEdge()),
+		))).Node()
+}
+
+func firstTipsetSelector(req *BlockSyncRequest) ipld.Node {
+	// builer for selectors
+	ssb := selectorbuilder.NewSelectorSpecBuilder(basicnode.Style.Any)
+
+	bso := ParseBSOptions(req.Options)
+	if bso.IncludeMessages {
+		return ssb.ExploreIndex(blockIndexMessagesField,
+			ssb.ExploreRange(0, 2, amtSelector),
+		).Node()
+	}
+	return ssb.Matcher().Node()
+
+}
+
+func (bs *BlockSync) executeGsyncSelector(ctx context.Context, p peer.ID, root cid.Cid, sel ipld.Node) error {
+	extension := graphsync.ExtensionData{
+		Name: "chainsync",
+		Data: nil,
+	}
+	_, errs := bs.gsync.Request(ctx, p, cidlink.Link{Cid: root}, sel, extension)
+
+	for err := range errs {
+		return xerrors.Errorf("failed to complete graphsync request: %w", err)
+	}
+	return nil
+}
+
+// Fallback for interacting with other non-lotus nodes
+func (bs *BlockSync) fetchBlocksGraphSync(ctx context.Context, p peer.ID, req *BlockSyncRequest) (*BlockSyncResponse, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	immediateTsSelector := firstTipsetSelector(req)
+
+	// Do this because we can only request one root at a time
+	for _, r := range req.Start {
+		if err := bs.executeGsyncSelector(ctx, p, r, immediateTsSelector); err != nil {
+			return nil, err
+		}
+	}
+
+	if req.RequestLength > maxRequestLength {
+		req.RequestLength = maxRequestLength
+	}
+
+	sel := selectorForRequest(req)
+
+	// execute the selector forreal
+	if err := bs.executeGsyncSelector(ctx, p, req.Start[0], sel); err != nil {
+		return nil, err
+	}
+
+	// Now pull the data we fetched out of the chainstore (where it should now be persisted)
+	tempcs := store.NewChainStore(bs.bserv.Blockstore(), datastore.NewMapDatastore(), nil)
+
+	opts := ParseBSOptions(req.Options)
+	tsk := types.NewTipSetKey(req.Start...)
+	chain, err := collectChainSegment(tempcs, tsk, req.RequestLength, opts)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to load chain data from chainstore after successful graphsync response (start = %v): %w", req.Start, err)
+	}
+
+	return &BlockSyncResponse{Chain: chain}, nil
+}

chain/blocksync/peer_tracker.go

@@ -1,170 +0,0 @@
-package blocksync
-
-// FIXME: This needs to be reviewed.
-
-import (
-	"sort"
-	"sync"
-	"time"
-
-	"github.com/libp2p/go-libp2p-core/peer"
-
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/lib/peermgr"
-)
-
-type peerStats struct {
-	successes   int
-	failures    int
-	firstSeen   time.Time
-	averageTime time.Duration
-}
-
-type bsPeerTracker struct {
-	lk sync.Mutex
-
-	peers         map[peer.ID]*peerStats
-	avgGlobalTime time.Duration
-
-	pmgr *peermgr.PeerMgr
-}
-
-func newPeerTracker(pmgr *peermgr.PeerMgr) *bsPeerTracker {
-	return &bsPeerTracker{
-		peers: make(map[peer.ID]*peerStats),
-		pmgr:  pmgr,
-	}
-}
-
-func (bpt *bsPeerTracker) addPeer(p peer.ID) {
-	bpt.lk.Lock()
-	defer bpt.lk.Unlock()
-	if _, ok := bpt.peers[p]; ok {
-		return
-	}
-	bpt.peers[p] = &peerStats{
-		firstSeen: build.Clock.Now(),
-	}
-
-}
-
-const (
-	// newPeerMul is how much better than average is the new peer assumed to be
-	// less than one to encourouge trying new peers
-	newPeerMul = 0.9
-)
-
-func (bpt *bsPeerTracker) prefSortedPeers() []peer.ID {
-	// TODO: this could probably be cached, but as long as its not too many peers, fine for now
-	bpt.lk.Lock()
-	defer bpt.lk.Unlock()
-	out := make([]peer.ID, 0, len(bpt.peers))
-	for p := range bpt.peers {
-		out = append(out, p)
-	}
-
-	// sort by 'expected cost' of requesting data from that peer
-	// additionally handle edge cases where not enough data is available
-	sort.Slice(out, func(i, j int) bool {
-		pi := bpt.peers[out[i]]
-		pj := bpt.peers[out[j]]
-
-		var costI, costJ float64
-
-		getPeerInitLat := func(p peer.ID) float64 {
-			var res float64
-			if bpt.pmgr != nil {
-				if lat, ok := bpt.pmgr.GetPeerLatency(p); ok {
-					res = float64(lat)
-				}
-			}
-			if res == 0 {
-				res = float64(bpt.avgGlobalTime)
-			}
-			return res * newPeerMul
-		}
-
-		if pi.successes+pi.failures > 0 {
-			failRateI := float64(pi.failures) / float64(pi.failures+pi.successes)
-			costI = float64(pi.averageTime) + failRateI*float64(bpt.avgGlobalTime)
-		} else {
-			costI = getPeerInitLat(out[i])
-		}
-
-		if pj.successes+pj.failures > 0 {
-			failRateJ := float64(pj.failures) / float64(pj.failures+pj.successes)
-			costJ = float64(pj.averageTime) + failRateJ*float64(bpt.avgGlobalTime)
-		} else {
-			costJ = getPeerInitLat(out[j])
-		}
-
-		return costI < costJ
-	})
-
-	return out
-}
-
-const (
-	// xInvAlpha = (N+1)/2
-
-	localInvAlpha  = 5  // 86% of the value is the last 9
-	globalInvAlpha = 20 // 86% of the value is the last 39
-)
-
-func (bpt *bsPeerTracker) logGlobalSuccess(dur time.Duration) {
-	bpt.lk.Lock()
-	defer bpt.lk.Unlock()
-
-	if bpt.avgGlobalTime == 0 {
-		bpt.avgGlobalTime = dur
-		return
-	}
-	delta := (dur - bpt.avgGlobalTime) / globalInvAlpha
-	bpt.avgGlobalTime += delta
-}
-
-func logTime(pi *peerStats, dur time.Duration) {
-	if pi.averageTime == 0 {
-		pi.averageTime = dur
-		return
-	}
-	delta := (dur - pi.averageTime) / localInvAlpha
-	pi.averageTime += delta
-
-}
-
-func (bpt *bsPeerTracker) logSuccess(p peer.ID, dur time.Duration) {
-	bpt.lk.Lock()
-	defer bpt.lk.Unlock()
-
-	var pi *peerStats
-	var ok bool
-	if pi, ok = bpt.peers[p]; !ok {
-		log.Warnw("log success called on peer not in tracker", "peerid", p.String())
-		return
-	}
-
-	pi.successes++
-	logTime(pi, dur)
-}
-
-func (bpt *bsPeerTracker) logFailure(p peer.ID, dur time.Duration) {
-	bpt.lk.Lock()
-	defer bpt.lk.Unlock()
-
-	var pi *peerStats
-	var ok bool
-	if pi, ok = bpt.peers[p]; !ok {
-		log.Warn("log failure called on peer not in tracker", "peerid", p.String())
-		return
-	}
-
-	pi.failures++
-	logTime(pi, dur)
-}
-
-func (bpt *bsPeerTracker) removePeer(p peer.ID) {
-	bpt.lk.Lock()
-	defer bpt.lk.Unlock()
-	delete(bpt.peers, p)
-}

chain/blocksync/protocol.go

@@ -1,195 +0,0 @@
-package blocksync
-
-import (
-	"time"
-
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/store"
-
-	"github.com/ipfs/go-cid"
-	logging "github.com/ipfs/go-log"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-var log = logging.Logger("blocksync")
-
-const BlockSyncProtocolID = "/fil/sync/blk/0.0.1"
-
-// FIXME: Bumped from original 800 to this to accommodate `syncFork()`
-//  use of `GetBlocks()`. It seems the expectation of that API is to
-//  fetch any amount of blocks leaving it to the internal logic here
-//  to partition and reassemble the requests if they go above the maximum.
-//  (Also as a consequence of this temporarily removing the `const`
-//   qualifier to avoid "const initializer [...] is not a constant" error.)
-var MaxRequestLength = uint64(build.ForkLengthThreshold)
-
-// Extracted constants from the code.
-// FIXME: Should be reviewed and confirmed.
-const SUCCESS_PEER_TAG_VALUE = 25
-const WRITE_REQ_DEADLINE = 5 * time.Second
-const READ_RES_DEADLINE = WRITE_REQ_DEADLINE
-const READ_RES_MIN_SPEED = 50 << 10
-const SHUFFLE_PEERS_PREFIX = 5
-const WRITE_RES_DEADLINE = 60 * time.Second
-
-// FIXME: Rename. Make private.
-type Request struct {
-	// List of ordered CIDs comprising a `TipSetKey` from where to start
-	// fetching backwards.
-	// FIXME: Consider using `TipSetKey` now (introduced after the creation
-	//  of this protocol) instead of converting back and forth.
-	Head []cid.Cid
-	// Number of block sets to fetch from `Head` (inclusive, should always
-	// be in the range `[1, MaxRequestLength]`).
-	Length uint64
-	// Request options, see `Options` type for more details. Compressed
-	// in a single `uint64` to save space.
-	Options uint64
-}
-
-// `Request` processed and validated to query the tipsets needed.
-type validatedRequest struct {
-	head    types.TipSetKey
-	length  uint64
-	options *parsedOptions
-}
-
-// Request options. When fetching the chain segment we can fetch
-// either block headers, messages, or both.
-const (
-	Headers = 1 << iota
-	Messages
-)
-
-// Decompressed options into separate struct members for easy access
-// during internal processing..
-type parsedOptions struct {
-	IncludeHeaders  bool
-	IncludeMessages bool
-}
-
-func (options *parsedOptions) noOptionsSet() bool {
-	return options.IncludeHeaders == false &&
-		options.IncludeMessages == false
-}
-
-func parseOptions(optfield uint64) *parsedOptions {
-	return &parsedOptions{
-		IncludeHeaders:  optfield&(uint64(Headers)) != 0,
-		IncludeMessages: optfield&(uint64(Messages)) != 0,
-	}
-}
-
-// FIXME: Rename. Make private.
-type Response struct {
-	Status status
-	// String that complements the error status when converting to an
-	// internal error (see `statusToError()`).
-	ErrorMessage string
-
-	Chain []*BSTipSet
-}
-
-type status uint64
-
-const (
-	Ok status = 0
-	// We could not fetch all blocks requested (but at least we returned
-	// the `Head` requested). Not considered an error.
-	Partial = 101
-
-	// Errors
-	NotFound      = 201
-	GoAway        = 202
-	InternalError = 203
-	BadRequest    = 204
-)
-
-// Convert status to internal error.
-func (res *Response) statusToError() error {
-	switch res.Status {
-	case Ok, Partial:
-		return nil
-		// FIXME: Consider if we want to not process `Partial` responses
-		//  and return an error instead.
-	case NotFound:
-		return xerrors.Errorf("not found")
-	case GoAway:
-		return xerrors.Errorf("not handling 'go away' blocksync responses yet")
-	case InternalError:
-		return xerrors.Errorf("block sync peer errored: %s", res.ErrorMessage)
-	case BadRequest:
-		return xerrors.Errorf("block sync request invalid: %s", res.ErrorMessage)
-	default:
-		return xerrors.Errorf("unrecognized response code: %d", res.Status)
-	}
-}
-
-// FIXME: Rename.
-type BSTipSet struct {
-	Blocks   []*types.BlockHeader
-	Messages *CompactedMessages
-}
-
-// All messages of a single tipset compacted together instead
-// of grouped by block to save space, since there are normally
-// many repeated messages per tipset in different blocks.
-//
-// `BlsIncludes`/`SecpkIncludes` matches `Bls`/`Secpk` messages
-// to blocks in the tipsets with the format:
-// `BlsIncludes[BI][MI]`
-//  * BI: block index in the tipset.
-//  * MI: message index in `Bls` list
-//
-// FIXME: The logic to decompress this structure should belong
-//  to itself, not to the consumer.
-type CompactedMessages struct {
-	Bls         []*types.Message
-	BlsIncludes [][]uint64
-
-	Secpk         []*types.SignedMessage
-	SecpkIncludes [][]uint64
-}
-
-// Response that has been validated according to the protocol
-// and can be safely accessed.
-type validatedResponse struct {
-	tipsets []*types.TipSet
-	// List of all messages per tipset (grouped by tipset,
-	// not by block, hence a single index like `tipsets`).
-	messages []*CompactedMessages
-}
-
-// Decompress messages and form full tipsets with them. The headers
-// need to have been requested as well.
-func (res *validatedResponse) toFullTipSets() []*store.FullTipSet {
-	if len(res.tipsets) == 0 || len(res.tipsets) != len(res.messages) {
-		// This decompression can only be done if both headers and
-		// messages are returned in the response. (The second check
-		// is already implied by the guarantees of `validatedResponse`,
-		// added here just for completeness.)
-		return nil
-	}
-	ftsList := make([]*store.FullTipSet, len(res.tipsets))
-	for tipsetIdx := range res.tipsets {
-		fts := &store.FullTipSet{} // FIXME: We should use the `NewFullTipSet` API.
-		msgs := res.messages[tipsetIdx]
-		for blockIdx, b := range res.tipsets[tipsetIdx].Blocks() {
-			fb := &types.FullBlock{
-				Header: b,
-			}
-			for _, mi := range msgs.BlsIncludes[blockIdx] {
-				fb.BlsMessages = append(fb.BlsMessages, msgs.Bls[mi])
-			}
-			for _, mi := range msgs.SecpkIncludes[blockIdx] {
-				fb.SecpkMessages = append(fb.SecpkMessages, msgs.Secpk[mi])
-			}
-
-			fts.Blocks = append(fts.Blocks, fb)
-		}
-		ftsList[tipsetIdx] = fts
-	}
-	return ftsList
-}

chain/blocksync/server.go

@@ -1,272 +0,0 @@
-package blocksync
-
-import (
-	"bufio"
-	"context"
-	"fmt"
-	"time"
-
-	"go.opencensus.io/trace"
-	"golang.org/x/xerrors"
-
-	cborutil "github.com/filecoin-project/go-cbor-util"
-
-	"github.com/filecoin-project/lotus/chain/store"
-	"github.com/filecoin-project/lotus/chain/types"
-
-	"github.com/ipfs/go-cid"
-	inet "github.com/libp2p/go-libp2p-core/network"
-)
-
-// BlockSyncService is the component that services BlockSync requests from
-// peers.
-//
-// BlockSync is the basic chain synchronization protocol of Filecoin. BlockSync
-// is an RPC-oriented protocol, with a single operation to request blocks.
-//
-// A request contains a start anchor block (referred to with a CID), and a
-// amount of blocks requested beyond the anchor (including the anchor itself).
-//
-// A client can also pass options, encoded as a 64-bit bitfield. Lotus supports
-// two options at the moment:
-//
-//  - include block contents
-//  - include block messages
-//
-// The response will include a status code, an optional message, and the
-// response payload in case of success. The payload is a slice of serialized
-// tipsets.
-// FIXME: Rename to just `Server` (will be done later, see note on `BlockSync`).
-type BlockSyncService struct {
-	cs *store.ChainStore
-}
-
-func NewBlockSyncService(cs *store.ChainStore) *BlockSyncService {
-	return &BlockSyncService{
-		cs: cs,
-	}
-}
-
-// Entry point of the service, handles `Request`s.
-func (server *BlockSyncService) HandleStream(stream inet.Stream) {
-	ctx, span := trace.StartSpan(context.Background(), "blocksync.HandleStream")
-	defer span.End()
-
-	defer stream.Close() //nolint:errcheck
-
-	var req Request
-	if err := cborutil.ReadCborRPC(bufio.NewReader(stream), &req); err != nil {
-		log.Warnf("failed to read block sync request: %s", err)
-		return
-	}
-	log.Infow("block sync request",
-		"start", req.Head, "len", req.Length)
-
-	resp, err := server.processRequest(ctx, &req)
-	if err != nil {
-		log.Warn("failed to process request: ", err)
-		return
-	}
-
-	_ = stream.SetDeadline(time.Now().Add(WRITE_RES_DEADLINE))
-	if err := cborutil.WriteCborRPC(stream, resp); err != nil {
-		_ = stream.SetDeadline(time.Time{})
-		log.Warnw("failed to write back response for handle stream",
-			"err", err, "peer", stream.Conn().RemotePeer())
-		return
-	}
-	_ = stream.SetDeadline(time.Time{})
-}
-
-// Validate and service the request. We return either a protocol
-// response or an internal error.
-func (server *BlockSyncService) processRequest(
-	ctx context.Context,
-	req *Request,
-) (*Response, error) {
-	validReq, errResponse := validateRequest(ctx, req)
-	if errResponse != nil {
-		// The request did not pass validation, return the response
-		//  indicating it.
-		return errResponse, nil
-	}
-
-	return server.serviceRequest(ctx, validReq)
-}
-
-// Validate request. We either return a `validatedRequest`, or an error
-// `Response` indicating why we can't process it. We do not return any
-// internal errors here, we just signal protocol ones.
-func validateRequest(
-	ctx context.Context,
-	req *Request,
-) (*validatedRequest, *Response) {
-	_, span := trace.StartSpan(ctx, "blocksync.ValidateRequest")
-	defer span.End()
-
-	validReq := validatedRequest{}
-
-	validReq.options = parseOptions(req.Options)
-	if validReq.options.noOptionsSet() {
-		return nil, &Response{
-			Status:       BadRequest,
-			ErrorMessage: "no options set",
-		}
-	}
-
-	validReq.length = req.Length
-	if validReq.length > MaxRequestLength {
-		return nil, &Response{
-			Status: BadRequest,
-			ErrorMessage: fmt.Sprintf("request length over maximum allowed (%d)",
-				MaxRequestLength),
-		}
-	}
-	if validReq.length == 0 {
-		return nil, &Response{
-			Status:       BadRequest,
-			ErrorMessage: "invalid request length of zero",
-		}
-	}
-
-	if len(req.Head) == 0 {
-		return nil, &Response{
-			Status:       BadRequest,
-			ErrorMessage: "no cids in request",
-		}
-	}
-	validReq.head = types.NewTipSetKey(req.Head...)
-
-	// FIXME: Add as a defer at the start.
-	span.AddAttributes(
-		trace.BoolAttribute("blocks", validReq.options.IncludeHeaders),
-		trace.BoolAttribute("messages", validReq.options.IncludeMessages),
-		trace.Int64Attribute("reqlen", int64(validReq.length)),
-	)
-
-	return &validReq, nil
-}
-
-func (server *BlockSyncService) serviceRequest(
-	ctx context.Context,
-	req *validatedRequest,
-) (*Response, error) {
-	_, span := trace.StartSpan(ctx, "blocksync.ServiceRequest")
-	defer span.End()
-
-	chain, err := collectChainSegment(server.cs, req)
-	if err != nil {
-		log.Warn("block sync request: collectChainSegment failed: ", err)
-		return &Response{
-			Status:       InternalError,
-			ErrorMessage: err.Error(),
-		}, nil
-	}
-
-	status := Ok
-	if len(chain) < int(req.length) {
-		status = Partial
-	}
-
-	return &Response{
-		Chain:  chain,
-		Status: status,
-	}, nil
-}
-
-func collectChainSegment(
-	cs *store.ChainStore,
-	req *validatedRequest,
-) ([]*BSTipSet, error) {
-	var bstips []*BSTipSet
-
-	cur := req.head
-	for {
-		var bst BSTipSet
-		ts, err := cs.LoadTipSet(cur)
-		if err != nil {
-			return nil, xerrors.Errorf("failed loading tipset %s: %w", cur, err)
-		}
-
-		if req.options.IncludeHeaders {
-			bst.Blocks = ts.Blocks()
-		}
-
-		if req.options.IncludeMessages {
-			bmsgs, bmincl, smsgs, smincl, err := gatherMessages(cs, ts)
-			if err != nil {
-				return nil, xerrors.Errorf("gather messages failed: %w", err)
-			}
-
-			// FIXME: Pass the response to `gatherMessages()` and set all this there.
-			bst.Messages = &CompactedMessages{}
-			bst.Messages.Bls = bmsgs
-			bst.Messages.BlsIncludes = bmincl
-			bst.Messages.Secpk = smsgs
-			bst.Messages.SecpkIncludes = smincl
-		}
-
-		bstips = append(bstips, &bst)
-
-		// If we collected the length requested or if we reached the
-		// start (genesis), then stop.
-		if uint64(len(bstips)) >= req.length || ts.Height() == 0 {
-			return bstips, nil
-		}
-
-		cur = ts.Parents()
-	}
-}
-
-func gatherMessages(cs *store.ChainStore, ts *types.TipSet) ([]*types.Message, [][]uint64, []*types.SignedMessage, [][]uint64, error) {
-	blsmsgmap := make(map[cid.Cid]uint64)
-	secpkmsgmap := make(map[cid.Cid]uint64)
-	var secpkincl, blsincl [][]uint64
-
-	var blscids, secpkcids []cid.Cid
-	for _, block := range ts.Blocks() {
-		bc, sc, err := cs.ReadMsgMetaCids(block.Messages)
-		if err != nil {
-			return nil, nil, nil, nil, err
-		}
-
-		// FIXME: DRY. Use `chain.Message` interface.
-		bmi := make([]uint64, 0, len(bc))
-		for _, m := range bc {
-			i, ok := blsmsgmap[m]
-			if !ok {
-				i = uint64(len(blscids))
-				blscids = append(blscids, m)
-				blsmsgmap[m] = i
-			}
-
-			bmi = append(bmi, i)
-		}
-		blsincl = append(blsincl, bmi)
-
-		smi := make([]uint64, 0, len(sc))
-		for _, m := range sc {
-			i, ok := secpkmsgmap[m]
-			if !ok {
-				i = uint64(len(secpkcids))
-				secpkcids = append(secpkcids, m)
-				secpkmsgmap[m] = i
-			}
-
-			smi = append(smi, i)
-		}
-		secpkincl = append(secpkincl, smi)
-	}
-
-	blsmsgs, err := cs.LoadMessagesFromCids(blscids)
-	if err != nil {
-		return nil, nil, nil, nil, err
-	}
-
-	secpkmsgs, err := cs.LoadSignedMessagesFromCids(secpkcids)
-	if err != nil {
-		return nil, nil, nil, nil, err
-	}
-
-	return blsmsgs, blsincl, secpkmsgs, secpkincl, nil
-}

chain/events/events.go

@@ -5,7 +5,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log/v2"
 	"golang.org/x/xerrors"
@@ -35,7 +35,6 @@ type eventAPI interface {
 	ChainNotify(context.Context) (<-chan []*api.HeadChange, error)
 	ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error)
 	ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)
-	ChainHead(context.Context) (*types.TipSet, error)
 	StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)
 	ChainGetTipSet(context.Context, types.TipSetKey) (*types.TipSet, error)
 
@@ -58,7 +57,7 @@ type Events struct {
 func NewEvents(ctx context.Context, api eventAPI) *Events {
 	gcConfidence := 2 * build.ForkLengthThreshold
 
-	tsc := newTSCache(gcConfidence, api)
+	tsc := newTSCache(gcConfidence, api.ChainGetTipSetByHeight)
 
 	e := &Events{
 		api: api,

chain/events/events_called.go

@@ -5,7 +5,7 @@ import (
 	"math"
 	"sync"
 
-	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
 
@@ -307,10 +307,7 @@ func (e *hcEvents) onHeadChanged(check CheckFunc, hnd EventHandler, rev RevertHa
 	defer e.lk.Unlock()
 
 	// Check if the event has already occurred
-	ts, err := e.tsc.best()
-	if err != nil {
-		return 0, xerrors.Errorf("error getting best tipset: %w", err)
-	}
+	ts := e.tsc.best()
 	done, more, err := check(ts)
 	if err != nil {
 		return 0, xerrors.Errorf("called check error (h: %d): %w", ts.Height(), err)

chain/events/events_height.go

@@ -4,9 +4,8 @@ import (
 	"context"
 	"sync"
 
-	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	"go.opencensus.io/trace"
-	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/chain/types"
 )
@@ -153,12 +152,8 @@ func (e *heightEvents) ChainAt(hnd HeightHandler, rev RevertHandler, confidence
 
 	e.lk.Lock() // Tricky locking, check your locks if you modify this function!
 
-	best, err := e.tsc.best()
-	if err != nil {
-		return xerrors.Errorf("error getting best tipset: %w", err)
-	}
+	bestH := e.tsc.best().Height()
 
-	bestH := best.Height()
 	if bestH >= h+abi.ChainEpoch(confidence) {
 		ts, err := e.tsc.getNonNull(h)
 		if err != nil {
@@ -177,11 +172,7 @@ func (e *heightEvents) ChainAt(hnd HeightHandler, rev RevertHandler, confidence
 		}
 
 		e.lk.Lock()
-		best, err = e.tsc.best()
-		if err != nil {
-			return xerrors.Errorf("error getting best tipset: %w", err)
-		}
-		bestH = best.Height()
+		bestH = e.tsc.best().Height()
 	}
 
 	defer e.lk.Unlock()

chain/events/events_test.go

@@ -11,8 +11,8 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
@@ -46,10 +46,6 @@ type fakeCS struct {
 	sub func(rev, app []*types.TipSet)
 }
 
-func (fcs *fakeCS) ChainHead(ctx context.Context) (*types.TipSet, error) {
-	panic("implement me")
-}
-
 func (fcs *fakeCS) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) {
 	return fcs.tipsets[key], nil
 }
@@ -114,11 +110,7 @@ func (fcs *fakeCS) makeTs(t *testing.T, parents []cid.Cid, h abi.ChainEpoch, msg
 
 func (fcs *fakeCS) ChainNotify(context.Context) (<-chan []*api.HeadChange, error) {
 	out := make(chan []*api.HeadChange, 1)
-	best, err := fcs.tsc.best()
-	if err != nil {
-		return nil, err
-	}
-	out <- []*api.HeadChange{{Type: store.HCCurrent, Val: best}}
+	out <- []*api.HeadChange{{Type: store.HCCurrent, Val: fcs.tsc.best()}}
 
 	fcs.sub = func(rev, app []*types.TipSet) {
 		notif := make([]*api.HeadChange, len(rev)+len(app))
@@ -182,8 +174,7 @@ func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { /
 
 	var revs []*types.TipSet
 	for i := 0; i < rev; i++ {
-		ts, err := fcs.tsc.best()
-		require.NoError(fcs.t, err)
+		ts := fcs.tsc.best()
 
 		if _, ok := nullm[int(ts.Height())]; !ok {
 			revs = append(revs, ts)
@@ -205,9 +196,7 @@ func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { /
 			continue
 		}
 
-		best, err := fcs.tsc.best()
-		require.NoError(fcs.t, err)
-		ts := fcs.makeTs(fcs.t, best.Key().Cids(), fcs.h, mc)
+		ts := fcs.makeTs(fcs.t, fcs.tsc.best().Key().Cids(), fcs.h, mc)
 		require.NoError(fcs.t, fcs.tsc.add(ts))
 
 		if hasMsgs {

chain/events/state/diff_adt.go

@@ -1,8 +1,6 @@
 package state
 
 import (
-	"bytes"
-
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
 	typegen "github.com/whyrusleeping/cbor-gen"
 )
@@ -41,11 +39,8 @@ func DiffAdtArray(preArr, curArr *adt.Array, out AdtArrayDiff) error {
 			return nil
 		}
 
-		// no modification
-		if !bytes.Equal(prevVal.Raw, curVal.Raw) {
-			if err := out.Modify(uint64(i), prevVal, curVal); err != nil {
-				return err
-			}
+		if err := out.Modify(uint64(i), prevVal, curVal); err != nil {
+			return err
 		}
 
 		return curArr.Delete(uint64(i))
@@ -95,11 +90,8 @@ func DiffAdtMap(preMap, curMap *adt.Map, out AdtMapDiff) error {
 			return nil
 		}
 
-		// no modification
-		if !bytes.Equal(prevVal.Raw, curVal.Raw) {
-			if err := out.Modify(key, prevVal, curVal); err != nil {
-				return err
-			}
+		if err := out.Modify(key, prevVal, curVal); err != nil {
+			return err
 		}
 
 		return curMap.Delete(k)

chain/events/state/diff_adt_test.go

@@ -8,13 +8,14 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	ds "github.com/ipfs/go-datastore"
+	ds_sync "github.com/ipfs/go-datastore/sync"
+	bstore "github.com/ipfs/go-ipfs-blockstore"
 	cbornode "github.com/ipfs/go-ipld-cbor"
 	typegen "github.com/whyrusleeping/cbor-gen"
 
 	"github.com/filecoin-project/specs-actors/actors/runtime"
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
-
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 func TestDiffAdtArray(t *testing.T) {
@@ -40,7 +41,7 @@ func TestDiffAdtArray(t *testing.T) {
 	require.NoError(t, arrB.Set(5, runtime.CBORBytes{8})) // add
 	require.NoError(t, arrB.Set(6, runtime.CBORBytes{9})) // add
 
-	changes := new(TestDiffArray)
+	changes := new(TestAdtDiff)
 
 	assert.NoError(t, DiffAdtArray(arrA, arrB, changes))
 	assert.NotNil(t, changes)
@@ -71,186 +72,38 @@ func TestDiffAdtArray(t *testing.T) {
 	assert.EqualValues(t, []byte{1}, changes.Removed[1].val)
 }
 
-func TestDiffAdtMap(t *testing.T) {
-	ctxstoreA := newContextStore()
-	ctxstoreB := newContextStore()
-
-	mapA := adt.MakeEmptyMap(ctxstoreA)
-	mapB := adt.MakeEmptyMap(ctxstoreB)
-
-	require.NoError(t, mapA.Put(adt.UIntKey(0), runtime.CBORBytes([]byte{0}))) // delete
-
-	require.NoError(t, mapA.Put(adt.UIntKey(1), runtime.CBORBytes([]byte{0}))) // modify
-	require.NoError(t, mapB.Put(adt.UIntKey(1), runtime.CBORBytes([]byte{1})))
-
-	require.NoError(t, mapA.Put(adt.UIntKey(2), runtime.CBORBytes([]byte{1}))) // delete
-
-	require.NoError(t, mapA.Put(adt.UIntKey(3), runtime.CBORBytes([]byte{0}))) // noop
-	require.NoError(t, mapB.Put(adt.UIntKey(3), runtime.CBORBytes([]byte{0})))
-
-	require.NoError(t, mapA.Put(adt.UIntKey(4), runtime.CBORBytes([]byte{0}))) // modify
-	require.NoError(t, mapB.Put(adt.UIntKey(4), runtime.CBORBytes([]byte{6})))
-
-	require.NoError(t, mapB.Put(adt.UIntKey(5), runtime.CBORBytes{8})) // add
-	require.NoError(t, mapB.Put(adt.UIntKey(6), runtime.CBORBytes{9})) // add
-
-	changes := new(TestDiffMap)
-
-	assert.NoError(t, DiffAdtMap(mapA, mapB, changes))
-	assert.NotNil(t, changes)
-
-	assert.Equal(t, 2, len(changes.Added))
-	// keys 5 and 6 were added
-	assert.EqualValues(t, uint64(6), changes.Added[0].key)
-	assert.EqualValues(t, []byte{9}, changes.Added[0].val)
-	assert.EqualValues(t, uint64(5), changes.Added[1].key)
-	assert.EqualValues(t, []byte{8}, changes.Added[1].val)
-
-	assert.Equal(t, 2, len(changes.Modified))
-	// keys 1 and 4 were modified
-	assert.EqualValues(t, uint64(1), changes.Modified[0].From.key)
-	assert.EqualValues(t, []byte{0}, changes.Modified[0].From.val)
-	assert.EqualValues(t, uint64(1), changes.Modified[0].To.key)
-	assert.EqualValues(t, []byte{1}, changes.Modified[0].To.val)
-	assert.EqualValues(t, uint64(4), changes.Modified[1].From.key)
-	assert.EqualValues(t, []byte{0}, changes.Modified[1].From.val)
-	assert.EqualValues(t, uint64(4), changes.Modified[1].To.key)
-	assert.EqualValues(t, []byte{6}, changes.Modified[1].To.val)
-
-	assert.Equal(t, 2, len(changes.Removed))
-	// keys 0 and 2 were deleted
-	assert.EqualValues(t, uint64(0), changes.Removed[0].key)
-	assert.EqualValues(t, []byte{0}, changes.Removed[0].val)
-	assert.EqualValues(t, uint64(2), changes.Removed[1].key)
-	assert.EqualValues(t, []byte{1}, changes.Removed[1].val)
-
-}
-
-type TestDiffMap struct {
-	Added    []adtMapDiffResult
-	Modified []TestAdtMapDiffModified
-	Removed  []adtMapDiffResult
-}
-
-var _ AdtMapDiff = &TestDiffMap{}
-
-func (t *TestDiffMap) AsKey(key string) (adt.Keyer, error) {
-	k, err := adt.ParseUIntKey(key)
-	if err != nil {
-		return nil, err
-	}
-	return adt.UIntKey(k), nil
-}
-
-func (t *TestDiffMap) Add(key string, val *typegen.Deferred) error {
-	v := new(runtime.CBORBytes)
-	err := v.UnmarshalCBOR(bytes.NewReader(val.Raw))
-	if err != nil {
-		return err
-	}
-	k, err := adt.ParseUIntKey(key)
-	if err != nil {
-		return err
-	}
-	t.Added = append(t.Added, adtMapDiffResult{
-		key: k,
-		val: *v,
-	})
-	return nil
-}
-
-func (t *TestDiffMap) Modify(key string, from, to *typegen.Deferred) error {
-	vFrom := new(runtime.CBORBytes)
-	err := vFrom.UnmarshalCBOR(bytes.NewReader(from.Raw))
-	if err != nil {
-		return err
-	}
-
-	vTo := new(runtime.CBORBytes)
-	err = vTo.UnmarshalCBOR(bytes.NewReader(to.Raw))
-	if err != nil {
-		return err
-	}
-
-	k, err := adt.ParseUIntKey(key)
-	if err != nil {
-		return err
-	}
-
-	if !bytes.Equal(*vFrom, *vTo) {
-		t.Modified = append(t.Modified, TestAdtMapDiffModified{
-			From: adtMapDiffResult{
-				key: k,
-				val: *vFrom,
-			},
-			To: adtMapDiffResult{
-				key: k,
-				val: *vTo,
-			},
-		})
-	}
-	return nil
-}
-
-func (t *TestDiffMap) Remove(key string, val *typegen.Deferred) error {
-	v := new(runtime.CBORBytes)
-	err := v.UnmarshalCBOR(bytes.NewReader(val.Raw))
-	if err != nil {
-		return err
-	}
-	k, err := adt.ParseUIntKey(key)
-	if err != nil {
-		return err
-	}
-	t.Removed = append(t.Removed, adtMapDiffResult{
-		key: k,
-		val: *v,
-	})
-	return nil
-}
-
-type adtMapDiffResult struct {
+type adtDiffResult struct {
 	key uint64
 	val runtime.CBORBytes
 }
 
-type TestAdtMapDiffModified struct {
-	From adtMapDiffResult
-	To   adtMapDiffResult
+type TestAdtDiff struct {
+	Added    []adtDiffResult
+	Modified []TestAdtDiffModified
+	Removed  []adtDiffResult
 }
 
-type adtArrayDiffResult struct {
-	key uint64
-	val runtime.CBORBytes
+var _ AdtArrayDiff = &TestAdtDiff{}
+
+type TestAdtDiffModified struct {
+	From adtDiffResult
+	To   adtDiffResult
 }
 
-type TestDiffArray struct {
-	Added    []adtArrayDiffResult
-	Modified []TestAdtArrayDiffModified
-	Removed  []adtArrayDiffResult
-}
-
-var _ AdtArrayDiff = &TestDiffArray{}
-
-type TestAdtArrayDiffModified struct {
-	From adtArrayDiffResult
-	To   adtArrayDiffResult
-}
-
-func (t *TestDiffArray) Add(key uint64, val *typegen.Deferred) error {
+func (t *TestAdtDiff) Add(key uint64, val *typegen.Deferred) error {
 	v := new(runtime.CBORBytes)
 	err := v.UnmarshalCBOR(bytes.NewReader(val.Raw))
 	if err != nil {
 		return err
 	}
-	t.Added = append(t.Added, adtArrayDiffResult{
+	t.Added = append(t.Added, adtDiffResult{
 		key: key,
 		val: *v,
 	})
 	return nil
 }
 
-func (t *TestDiffArray) Modify(key uint64, from, to *typegen.Deferred) error {
+func (t *TestAdtDiff) Modify(key uint64, from, to *typegen.Deferred) error {
 	vFrom := new(runtime.CBORBytes)
 	err := vFrom.UnmarshalCBOR(bytes.NewReader(from.Raw))
 	if err != nil {
@@ -264,12 +117,12 @@ func (t *TestDiffArray) Modify(key uint64, from, to *typegen.Deferred) error {
 	}
 
 	if !bytes.Equal(*vFrom, *vTo) {
-		t.Modified = append(t.Modified, TestAdtArrayDiffModified{
-			From: adtArrayDiffResult{
+		t.Modified = append(t.Modified, TestAdtDiffModified{
+			From: adtDiffResult{
 				key: key,
 				val: *vFrom,
 			},
-			To: adtArrayDiffResult{
+			To: adtDiffResult{
 				key: key,
 				val: *vTo,
 			},
@@ -278,13 +131,13 @@ func (t *TestDiffArray) Modify(key uint64, from, to *typegen.Deferred) error {
 	return nil
 }
 
-func (t *TestDiffArray) Remove(key uint64, val *typegen.Deferred) error {
+func (t *TestAdtDiff) Remove(key uint64, val *typegen.Deferred) error {
 	v := new(runtime.CBORBytes)
 	err := v.UnmarshalCBOR(bytes.NewReader(val.Raw))
 	if err != nil {
 		return err
 	}
-	t.Removed = append(t.Removed, adtArrayDiffResult{
+	t.Removed = append(t.Removed, adtDiffResult{
 		key: key,
 		val: *v,
 	})
@@ -293,7 +146,7 @@ func (t *TestDiffArray) Remove(key uint64, val *typegen.Deferred) error {
 
 func newContextStore() *contextStore {
 	ctx := context.Background()
-	bs := bstore.NewTemporarySync()
+	bs := bstore.NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
 	store := cbornode.NewCborStore(bs)
 	return &contextStore{
 		ctx: ctx,

chain/events/state/predicates.go

@@ -3,20 +3,17 @@ package state
 import (
 	"bytes"
 	"context"
-
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/specs-actors/actors/builtin"
-	init_ "github.com/filecoin-project/specs-actors/actors/builtin/init"
-	"github.com/filecoin-project/specs-actors/actors/builtin/market"
-	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
-	"github.com/filecoin-project/specs-actors/actors/builtin/paych"
-	"github.com/filecoin-project/specs-actors/actors/util/adt"
 	"github.com/ipfs/go-cid"
 	cbor "github.com/ipfs/go-ipld-cbor"
 	typegen "github.com/whyrusleeping/cbor-gen"
 
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/builtin"
+	"github.com/filecoin-project/specs-actors/actors/builtin/market"
+	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
+	"github.com/filecoin-project/specs-actors/actors/util/adt"
+
 	"github.com/filecoin-project/lotus/api/apibstore"
 	"github.com/filecoin-project/lotus/chain/types"
 )
@@ -87,50 +84,6 @@ func (sp *StatePredicates) OnStorageMarketActorChanged(diffStorageMarketState Di
 	})
 }
 
-type BalanceTables struct {
-	EscrowTable *adt.BalanceTable
-	LockedTable *adt.BalanceTable
-}
-
-// DiffBalanceTablesFunc compares two balance tables
-type DiffBalanceTablesFunc func(ctx context.Context, oldBalanceTable, newBalanceTable BalanceTables) (changed bool, user UserData, err error)
-
-// OnBalanceChanged runs when the escrow table for available balances changes
-func (sp *StatePredicates) OnBalanceChanged(diffBalances DiffBalanceTablesFunc) DiffStorageMarketStateFunc {
-	return func(ctx context.Context, oldState *market.State, newState *market.State) (changed bool, user UserData, err error) {
-		if oldState.EscrowTable.Equals(newState.EscrowTable) && oldState.LockedTable.Equals(newState.LockedTable) {
-			return false, nil, nil
-		}
-
-		ctxStore := &contextStore{
-			ctx: ctx,
-			cst: sp.cst,
-		}
-
-		oldEscrowRoot, err := adt.AsBalanceTable(ctxStore, oldState.EscrowTable)
-		if err != nil {
-			return false, nil, err
-		}
-
-		oldLockedRoot, err := adt.AsBalanceTable(ctxStore, oldState.LockedTable)
-		if err != nil {
-			return false, nil, err
-		}
-
-		newEscrowRoot, err := adt.AsBalanceTable(ctxStore, newState.EscrowTable)
-		if err != nil {
-			return false, nil, err
-		}
-
-		newLockedRoot, err := adt.AsBalanceTable(ctxStore, newState.LockedTable)
-		if err != nil {
-			return false, nil, err
-		}
-
-		return diffBalances(ctx, BalanceTables{oldEscrowRoot, oldLockedRoot}, BalanceTables{newEscrowRoot, newLockedRoot})
-	}
-}
-
 type DiffAdtArraysFunc func(ctx context.Context, oldDealStateRoot, newDealStateRoot *adt.Array) (changed bool, user UserData, err error)
 
 // OnDealStateChanged calls diffDealStates when the market deal state changes
@@ -354,74 +307,8 @@ func (sp *StatePredicates) DealStateChangedForIDs(dealIds []abi.DealID) DiffAdtA
 	}
 }
 
-// ChangedBalances is a set of changes to deal state
-type ChangedBalances map[address.Address]BalanceChange
-
-// BalanceChange is a change in balance from -> to
-type BalanceChange struct {
-	From abi.TokenAmount
-	To   abi.TokenAmount
-}
-
-// AvailableBalanceChangedForAddresses detects changes in the escrow table for the given addresses
-func (sp *StatePredicates) AvailableBalanceChangedForAddresses(getAddrs func() []address.Address) DiffBalanceTablesFunc {
-	return func(ctx context.Context, oldBalances, newBalances BalanceTables) (changed bool, user UserData, err error) {
-		changedBalances := make(ChangedBalances)
-		addrs := getAddrs()
-		for _, addr := range addrs {
-			// If the deal has been removed, we just set it to nil
-			oldEscrowBalance, err := oldBalances.EscrowTable.Get(addr)
-			if err != nil {
-				return false, nil, err
-			}
-
-			oldLockedBalance, err := oldBalances.LockedTable.Get(addr)
-			if err != nil {
-				return false, nil, err
-			}
-
-			oldBalance := big.Sub(oldEscrowBalance, oldLockedBalance)
-
-			newEscrowBalance, err := newBalances.EscrowTable.Get(addr)
-			if err != nil {
-				return false, nil, err
-			}
-
-			newLockedBalance, err := newBalances.LockedTable.Get(addr)
-			if err != nil {
-				return false, nil, err
-			}
-
-			newBalance := big.Sub(newEscrowBalance, newLockedBalance)
-
-			if !oldBalance.Equals(newBalance) {
-				changedBalances[addr] = BalanceChange{oldBalance, newBalance}
-			}
-		}
-		if len(changedBalances) > 0 {
-			return true, changedBalances, nil
-		}
-		return false, nil, nil
-	}
-}
-
 type DiffMinerActorStateFunc func(ctx context.Context, oldState *miner.State, newState *miner.State) (changed bool, user UserData, err error)
 
-func (sp *StatePredicates) OnInitActorChange(diffInitActorState DiffInitActorStateFunc) DiffTipSetKeyFunc {
-	return sp.OnActorStateChanged(builtin.InitActorAddr, func(ctx context.Context, oldActorStateHead, newActorStateHead cid.Cid) (changed bool, user UserData, err error) {
-		var oldState init_.State
-		if err := sp.cst.Get(ctx, oldActorStateHead, &oldState); err != nil {
-			return false, nil, err
-		}
-		var newState init_.State
-		if err := sp.cst.Get(ctx, newActorStateHead, &newState); err != nil {
-			return false, nil, err
-		}
-		return diffInitActorState(ctx, &oldState, &newState)
-	})
-
-}
-
 func (sp *StatePredicates) OnMinerActorChange(minerAddr address.Address, diffMinerActorState DiffMinerActorStateFunc) DiffTipSetKeyFunc {
 	return sp.OnActorStateChanged(minerAddr, func(ctx context.Context, oldActorStateHead, newActorStateHead cid.Cid) (changed bool, user UserData, err error) {
 		var oldState miner.State
@@ -606,182 +493,3 @@ func (sp *StatePredicates) OnMinerPreCommitChange() DiffMinerActorStateFunc {
 		return true, precommitChanges, nil
 	}
 }
-
-// DiffPaymentChannelStateFunc is function that compares two states for the payment channel
-type DiffPaymentChannelStateFunc func(ctx context.Context, oldState *paych.State, newState *paych.State) (changed bool, user UserData, err error)
-
-// OnPaymentChannelActorChanged calls diffPaymentChannelState when the state changes for the the payment channel actor
-func (sp *StatePredicates) OnPaymentChannelActorChanged(paychAddr address.Address, diffPaymentChannelState DiffPaymentChannelStateFunc) DiffTipSetKeyFunc {
-	return sp.OnActorStateChanged(paychAddr, func(ctx context.Context, oldActorStateHead, newActorStateHead cid.Cid) (changed bool, user UserData, err error) {
-		var oldState paych.State
-		if err := sp.cst.Get(ctx, oldActorStateHead, &oldState); err != nil {
-			return false, nil, err
-		}
-		var newState paych.State
-		if err := sp.cst.Get(ctx, newActorStateHead, &newState); err != nil {
-			return false, nil, err
-		}
-		return diffPaymentChannelState(ctx, &oldState, &newState)
-	})
-}
-
-// PayChToSendChange is a difference in the amount to send on a payment channel when the money is collected
-type PayChToSendChange struct {
-	OldToSend abi.TokenAmount
-	NewToSend abi.TokenAmount
-}
-
-// OnToSendAmountChanges monitors changes on the total amount to send from one party to the other on a payment channel
-func (sp *StatePredicates) OnToSendAmountChanges() DiffPaymentChannelStateFunc {
-	return func(ctx context.Context, oldState *paych.State, newState *paych.State) (changed bool, user UserData, err error) {
-		if oldState.ToSend.Equals(newState.ToSend) {
-			return false, nil, nil
-		}
-		return true, &PayChToSendChange{
-			OldToSend: oldState.ToSend,
-			NewToSend: newState.ToSend,
-		}, nil
-	}
-}
-
-type AddressPair struct {
-	ID address.Address
-	PK address.Address
-}
-
-type InitActorAddressChanges struct {
-	Added    []AddressPair
-	Modified []AddressChange
-	Removed  []AddressPair
-}
-
-type AddressChange struct {
-	From AddressPair
-	To   AddressPair
-}
-
-type DiffInitActorStateFunc func(ctx context.Context, oldState *init_.State, newState *init_.State) (changed bool, user UserData, err error)
-
-func (i *InitActorAddressChanges) AsKey(key string) (adt.Keyer, error) {
-	addr, err := address.NewFromBytes([]byte(key))
-	if err != nil {
-		return nil, err
-	}
-	return adt.AddrKey(addr), nil
-}
-
-func (i *InitActorAddressChanges) Add(key string, val *typegen.Deferred) error {
-	pkAddr, err := address.NewFromBytes([]byte(key))
-	if err != nil {
-		return err
-	}
-	id := new(typegen.CborInt)
-	if err := id.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil {
-		return err
-	}
-	idAddr, err := address.NewIDAddress(uint64(*id))
-	if err != nil {
-		return err
-	}
-	i.Added = append(i.Added, AddressPair{
-		ID: idAddr,
-		PK: pkAddr,
-	})
-	return nil
-}
-
-func (i *InitActorAddressChanges) Modify(key string, from, to *typegen.Deferred) error {
-	pkAddr, err := address.NewFromBytes([]byte(key))
-	if err != nil {
-		return err
-	}
-
-	fromID := new(typegen.CborInt)
-	if err := fromID.UnmarshalCBOR(bytes.NewReader(from.Raw)); err != nil {
-		return err
-	}
-	fromIDAddr, err := address.NewIDAddress(uint64(*fromID))
-	if err != nil {
-		return err
-	}
-
-	toID := new(typegen.CborInt)
-	if err := toID.UnmarshalCBOR(bytes.NewReader(to.Raw)); err != nil {
-		return err
-	}
-	toIDAddr, err := address.NewIDAddress(uint64(*toID))
-	if err != nil {
-		return err
-	}
-
-	i.Modified = append(i.Modified, AddressChange{
-		From: AddressPair{
-			ID: fromIDAddr,
-			PK: pkAddr,
-		},
-		To: AddressPair{
-			ID: toIDAddr,
-			PK: pkAddr,
-		},
-	})
-	return nil
-}
-
-func (i *InitActorAddressChanges) Remove(key string, val *typegen.Deferred) error {
-	pkAddr, err := address.NewFromBytes([]byte(key))
-	if err != nil {
-		return err
-	}
-	id := new(typegen.CborInt)
-	if err := id.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil {
-		return err
-	}
-	idAddr, err := address.NewIDAddress(uint64(*id))
-	if err != nil {
-		return err
-	}
-	i.Removed = append(i.Removed, AddressPair{
-		ID: idAddr,
-		PK: pkAddr,
-	})
-	return nil
-}
-
-func (sp *StatePredicates) OnAddressMapChange() DiffInitActorStateFunc {
-	return func(ctx context.Context, oldState, newState *init_.State) (changed bool, user UserData, err error) {
-		ctxStore := &contextStore{
-			ctx: ctx,
-			cst: sp.cst,
-		}
-
-		addressChanges := &InitActorAddressChanges{
-			Added:    []AddressPair{},
-			Modified: []AddressChange{},
-			Removed:  []AddressPair{},
-		}
-
-		if oldState.AddressMap.Equals(newState.AddressMap) {
-			return false, nil, nil
-		}
-
-		oldAddrs, err := adt.AsMap(ctxStore, oldState.AddressMap)
-		if err != nil {
-			return false, nil, err
-		}
-
-		newAddrs, err := adt.AsMap(ctxStore, newState.AddressMap)
-		if err != nil {
-			return false, nil, err
-		}
-
-		if err := DiffAdtMap(oldAddrs, newAddrs, addressChanges); err != nil {
-			return false, nil, err
-		}
-
-		if len(addressChanges.Added)+len(addressChanges.Removed)+len(addressChanges.Modified) == 0 {
-			return false, nil, nil
-		}
-
-		return true, addressChanges, nil
-	}
-}

chain/events/state/predicates_test.go

@@ -4,25 +4,27 @@ import (
 	"context"
 	"testing"
 
-	"github.com/filecoin-project/go-bitfield"
-
 	"github.com/stretchr/testify/require"
 	"golang.org/x/xerrors"
 
 	"github.com/ipfs/go-cid"
+	ds "github.com/ipfs/go-datastore"
+	ds_sync "github.com/ipfs/go-datastore/sync"
+	"github.com/ipfs/go-hamt-ipld"
+	bstore "github.com/ipfs/go-ipfs-blockstore"
 	cbornode "github.com/ipfs/go-ipld-cbor"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/go-amt-ipld/v2"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin/market"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
 	tutils "github.com/filecoin-project/specs-actors/support/testing"
 
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 var dummyCid cid.Cid
@@ -66,8 +68,8 @@ func (m mockAPI) setActor(tsk types.TipSetKey, act *types.Actor) {
 
 func TestMarketPredicates(t *testing.T) {
 	ctx := context.Background()
-	bs := bstore.NewTemporarySync()
-	store := adt.WrapStore(ctx, cbornode.NewCborStore(bs))
+	bs := bstore.NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
+	store := cbornode.NewCborStore(bs)
 
 	oldDeal1 := &market.DealState{
 		SectorStartEpoch: 1,
@@ -113,14 +115,7 @@ func TestMarketPredicates(t *testing.T) {
 		abi.DealID(2): oldProp2,
 	}
 
-	oldBalances := map[address.Address]balance{
-		tutils.NewIDAddr(t, 1): {abi.NewTokenAmount(1000), abi.NewTokenAmount(1000)},
-		tutils.NewIDAddr(t, 2): {abi.NewTokenAmount(2000), abi.NewTokenAmount(500)},
-		tutils.NewIDAddr(t, 3): {abi.NewTokenAmount(3000), abi.NewTokenAmount(2000)},
-		tutils.NewIDAddr(t, 5): {abi.NewTokenAmount(3000), abi.NewTokenAmount(1000)},
-	}
-
-	oldStateC := createMarketState(ctx, t, store, oldDeals, oldProps, oldBalances)
+	oldStateC := createMarketState(ctx, t, store, oldDeals, oldProps)
 
 	newDeal1 := &market.DealState{
 		SectorStartEpoch: 1,
@@ -161,14 +156,7 @@ func TestMarketPredicates(t *testing.T) {
 		abi.DealID(3): newProp3, // new
 		// NB: DealProposals cannot be modified, so don't test that case.
 	}
-	newBalances := map[address.Address]balance{
-		tutils.NewIDAddr(t, 1): {abi.NewTokenAmount(3000), abi.NewTokenAmount(0)},
-		tutils.NewIDAddr(t, 2): {abi.NewTokenAmount(2000), abi.NewTokenAmount(500)},
-		tutils.NewIDAddr(t, 4): {abi.NewTokenAmount(5000), abi.NewTokenAmount(0)},
-		tutils.NewIDAddr(t, 5): {abi.NewTokenAmount(1000), abi.NewTokenAmount(3000)},
-	}
-
-	newStateC := createMarketState(ctx, t, store, newDeals, newProps, newBalances)
+	newStateC := createMarketState(ctx, t, store, newDeals, newProps)
 
 	minerAddr, err := address.NewFromString("t00")
 	require.NoError(t, err)
@@ -291,69 +279,12 @@ func TestMarketPredicates(t *testing.T) {
 		require.Equal(t, abi.DealID(2), changedProps.Removed[0].ID)
 		require.Equal(t, *oldProp2, changedProps.Removed[0].Proposal)
 	})
-
-	t.Run("balances predicate", func(t *testing.T) {
-		preds := NewStatePredicates(api)
-
-		getAddresses := func() []address.Address {
-			return []address.Address{tutils.NewIDAddr(t, 1), tutils.NewIDAddr(t, 2), tutils.NewIDAddr(t, 3), tutils.NewIDAddr(t, 4)}
-		}
-		diffBalancesFn := preds.OnStorageMarketActorChanged(preds.OnBalanceChanged(preds.AvailableBalanceChangedForAddresses(getAddresses)))
-
-		// Diff a state against itself: expect no change
-		changed, _, err := diffBalancesFn(ctx, oldState.Key(), oldState.Key())
-		require.NoError(t, err)
-		require.False(t, changed)
-
-		// Diff old state against new state
-		changed, valIDs, err := diffBalancesFn(ctx, oldState.Key(), newState.Key())
-		require.NoError(t, err)
-		require.True(t, changed)
-
-		changedBalances, ok := valIDs.(ChangedBalances)
-		require.True(t, ok)
-		require.Len(t, changedBalances, 3)
-		require.Contains(t, changedBalances, tutils.NewIDAddr(t, 1))
-		require.Contains(t, changedBalances, tutils.NewIDAddr(t, 3))
-		require.Contains(t, changedBalances, tutils.NewIDAddr(t, 4))
-
-		balance1 := changedBalances[tutils.NewIDAddr(t, 1)]
-		if !balance1.From.Equals(abi.NewTokenAmount(1000)) || !balance1.To.Equals(abi.NewTokenAmount(3000)) {
-			t.Fatal("Unexpected change to balance")
-		}
-		balance3 := changedBalances[tutils.NewIDAddr(t, 3)]
-		if !balance3.From.Equals(abi.NewTokenAmount(3000)) || !balance3.To.Equals(abi.NewTokenAmount(0)) {
-			t.Fatal("Unexpected change to balance")
-		}
-		balance4 := changedBalances[tutils.NewIDAddr(t, 4)]
-		if !balance4.From.Equals(abi.NewTokenAmount(0)) || !balance4.To.Equals(abi.NewTokenAmount(5000)) {
-			t.Fatal("Unexpected change to balance")
-		}
-
-		// Diff with non-existent address.
-		getNoAddress := func() []address.Address { return []address.Address{tutils.NewIDAddr(t, 6)} }
-		diffNoAddressFn := preds.OnStorageMarketActorChanged(preds.OnBalanceChanged(preds.AvailableBalanceChangedForAddresses(getNoAddress)))
-		changed, _, err = diffNoAddressFn(ctx, oldState.Key(), newState.Key())
-		require.NoError(t, err)
-		require.False(t, changed)
-
-		// Test that OnBalanceChanged does not call the callback if the state has not changed
-		diffDealBalancesFn := preds.OnBalanceChanged(func(context.Context, BalanceTables, BalanceTables) (bool, UserData, error) {
-			t.Fatal("No state change so this should not be called")
-			return false, nil, nil
-		})
-		marketState := createEmptyMarketState(t, store)
-		changed, _, err = diffDealBalancesFn(ctx, marketState, marketState)
-		require.NoError(t, err)
-		require.False(t, changed)
-	})
-
 }
 
 func TestMinerSectorChange(t *testing.T) {
 	ctx := context.Background()
-	bs := bstore.NewTemporarySync()
-	store := adt.WrapStore(ctx, cbornode.NewCborStore(bs))
+	bs := bstore.NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
+	store := cbornode.NewCborStore(bs)
 
 	nextID := uint64(0)
 	nextIDAddrF := func() address.Address {
@@ -445,83 +376,50 @@ func mockTipset(minerAddr address.Address, timestamp uint64) (*types.TipSet, err
 	}})
 }
 
-type balance struct {
-	available abi.TokenAmount
-	locked    abi.TokenAmount
-}
-
-func createMarketState(ctx context.Context, t *testing.T, store adt.Store, deals map[abi.DealID]*market.DealState, props map[abi.DealID]*market.DealProposal, balances map[address.Address]balance) cid.Cid {
+func createMarketState(ctx context.Context, t *testing.T, store *cbornode.BasicIpldStore, deals map[abi.DealID]*market.DealState, props map[abi.DealID]*market.DealProposal) cid.Cid {
 	dealRootCid := createDealAMT(ctx, t, store, deals)
 	propRootCid := createProposalAMT(ctx, t, store, props)
-	balancesCids := createBalanceTable(ctx, t, store, balances)
+
 	state := createEmptyMarketState(t, store)
 	state.States = dealRootCid
 	state.Proposals = propRootCid
-	state.EscrowTable = balancesCids[0]
-	state.LockedTable = balancesCids[1]
 
 	stateC, err := store.Put(ctx, state)
 	require.NoError(t, err)
 	return stateC
 }
 
-func createEmptyMarketState(t *testing.T, store adt.Store) *market.State {
-	emptyArrayCid, err := adt.MakeEmptyArray(store).Root()
+func createEmptyMarketState(t *testing.T, store *cbornode.BasicIpldStore) *market.State {
+	emptyArrayCid, err := amt.NewAMT(store).Flush(context.TODO())
 	require.NoError(t, err)
-	emptyMap, err := adt.MakeEmptyMap(store).Root()
+	emptyMap, err := store.Put(context.TODO(), hamt.NewNode(store, hamt.UseTreeBitWidth(5)))
 	require.NoError(t, err)
 	return market.ConstructState(emptyArrayCid, emptyMap, emptyMap)
 }
 
-func createDealAMT(ctx context.Context, t *testing.T, store adt.Store, deals map[abi.DealID]*market.DealState) cid.Cid {
-	root := adt.MakeEmptyArray(store)
+func createDealAMT(ctx context.Context, t *testing.T, store *cbornode.BasicIpldStore, deals map[abi.DealID]*market.DealState) cid.Cid {
+	root := amt.NewAMT(store)
 	for dealID, dealState := range deals {
-		err := root.Set(uint64(dealID), dealState)
+		err := root.Set(ctx, uint64(dealID), dealState)
 		require.NoError(t, err)
 	}
-	rootCid, err := root.Root()
+	rootCid, err := root.Flush(ctx)
 	require.NoError(t, err)
 	return rootCid
 }
 
-func createProposalAMT(ctx context.Context, t *testing.T, store adt.Store, props map[abi.DealID]*market.DealProposal) cid.Cid {
-	root := adt.MakeEmptyArray(store)
+func createProposalAMT(ctx context.Context, t *testing.T, store *cbornode.BasicIpldStore, props map[abi.DealID]*market.DealProposal) cid.Cid {
+	root := amt.NewAMT(store)
 	for dealID, prop := range props {
-		err := root.Set(uint64(dealID), prop)
+		err := root.Set(ctx, uint64(dealID), prop)
 		require.NoError(t, err)
 	}
-	rootCid, err := root.Root()
+	rootCid, err := root.Flush(ctx)
 	require.NoError(t, err)
 	return rootCid
 }
 
-func createBalanceTable(ctx context.Context, t *testing.T, store adt.Store, balances map[address.Address]balance) [2]cid.Cid {
-	escrowMapRoot := adt.MakeEmptyMap(store)
-	escrowMapRootCid, err := escrowMapRoot.Root()
-	require.NoError(t, err)
-	escrowRoot, err := adt.AsBalanceTable(store, escrowMapRootCid)
-	require.NoError(t, err)
-	lockedMapRoot := adt.MakeEmptyMap(store)
-	lockedMapRootCid, err := lockedMapRoot.Root()
-	require.NoError(t, err)
-	lockedRoot, err := adt.AsBalanceTable(store, lockedMapRootCid)
-	require.NoError(t, err)
-
-	for addr, balance := range balances {
-		err := escrowRoot.Add(addr, big.Add(balance.available, balance.locked))
-		require.NoError(t, err)
-		err = lockedRoot.Add(addr, balance.locked)
-		require.NoError(t, err)
-
-	}
-	escrowRootCid, err := escrowRoot.Root()
-	require.NoError(t, err)
-	lockedRootCid, err := lockedRoot.Root()
-	require.NoError(t, err)
-	return [2]cid.Cid{escrowRootCid, lockedRootCid}
-}
-
-func createMinerState(ctx context.Context, t *testing.T, store adt.Store, owner, worker address.Address, sectors []miner.SectorOnChainInfo) cid.Cid {
+func createMinerState(ctx context.Context, t *testing.T, store *cbornode.BasicIpldStore, owner, worker address.Address, sectors []miner.SectorOnChainInfo) cid.Cid {
 	rootCid := createSectorsAMT(ctx, t, store, sectors)
 
 	state := createEmptyMinerState(ctx, t, store, owner, worker)
@@ -532,43 +430,41 @@ func createMinerState(ctx context.Context, t *testing.T, store adt.Store, owner,
 	return stateC
 }
 
-func createEmptyMinerState(ctx context.Context, t *testing.T, store adt.Store, owner, worker address.Address) *miner.State {
-	emptyArrayCid, err := adt.MakeEmptyArray(store).Root()
+func createEmptyMinerState(ctx context.Context, t *testing.T, store *cbornode.BasicIpldStore, owner, worker address.Address) *miner.State {
+	emptyArrayCid, err := amt.NewAMT(store).Flush(context.TODO())
 	require.NoError(t, err)
-	emptyMap, err := adt.MakeEmptyMap(store).Root()
+	emptyMap, err := store.Put(context.TODO(), hamt.NewNode(store, hamt.UseTreeBitWidth(5)))
 	require.NoError(t, err)
 
-	emptyDeadline, err := store.Put(store.Context(), miner.ConstructDeadline(emptyArrayCid))
-	require.NoError(t, err)
-
-	emptyVestingFunds := miner.ConstructVestingFunds()
-	emptyVestingFundsCid, err := store.Put(store.Context(), emptyVestingFunds)
+	emptyDeadline, err := store.Put(context.TODO(), &miner.Deadline{
+		Partitions:        emptyArrayCid,
+		ExpirationsEpochs: emptyArrayCid,
+		PostSubmissions:   abi.NewBitField(),
+		EarlyTerminations: abi.NewBitField(),
+		LiveSectors:       0,
+	})
 	require.NoError(t, err)
 
 	emptyDeadlines := miner.ConstructDeadlines(emptyDeadline)
-	emptyDeadlinesCid, err := store.Put(store.Context(), emptyDeadlines)
+	emptyDeadlinesCid, err := store.Put(context.Background(), emptyDeadlines)
 	require.NoError(t, err)
 
 	minerInfo := emptyMap
 
-	emptyBitfield := bitfield.NewFromSet(nil)
-	emptyBitfieldCid, err := store.Put(store.Context(), emptyBitfield)
-	require.NoError(t, err)
-
-	state, err := miner.ConstructState(minerInfo, 123, emptyBitfieldCid, emptyArrayCid, emptyMap, emptyDeadlinesCid, emptyVestingFundsCid)
+	state, err := miner.ConstructState(minerInfo, 123, emptyArrayCid, emptyMap, emptyDeadlinesCid)
 	require.NoError(t, err)
 	return state
 
 }
 
-func createSectorsAMT(ctx context.Context, t *testing.T, store adt.Store, sectors []miner.SectorOnChainInfo) cid.Cid {
-	root := adt.MakeEmptyArray(store)
+func createSectorsAMT(ctx context.Context, t *testing.T, store *cbornode.BasicIpldStore, sectors []miner.SectorOnChainInfo) cid.Cid {
+	root := amt.NewAMT(store)
 	for _, sector := range sectors {
 		sector := sector
-		err := root.Set(uint64(sector.SectorNumber), &sector)
+		err := root.Set(ctx, uint64(sector.SectorNumber), &sector)
 		require.NoError(t, err)
 	}
-	rootCid, err := root.Root()
+	rootCid, err := root.Flush(ctx)
 	require.NoError(t, err)
 	return rootCid
 }
@@ -583,12 +479,10 @@ func newSectorOnChainInfo(sectorNo abi.SectorNumber, sealed cid.Cid, weight big.
 		DealIDs:      info.DealIDs,
 		Expiration:   info.Expiration,
 
-		Activation:            activation,
-		DealWeight:            weight,
-		VerifiedDealWeight:    weight,
-		InitialPledge:         big.Zero(),
-		ExpectedDayReward:     big.Zero(),
-		ExpectedStoragePledge: big.Zero(),
+		Activation:         activation,
+		DealWeight:         weight,
+		VerifiedDealWeight: weight,
+		InitialPledge:      big.Zero(),
 	}
 }
 

chain/events/tscache.go

@@ -3,16 +3,13 @@ package events
 import (
 	"context"
 
-	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
-type tsCacheAPI interface {
-	ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)
-	ChainHead(context.Context) (*types.TipSet, error)
-}
+type tsByHFunc func(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)
 
 // tipSetCache implements a simple ring-buffer cache to keep track of recent
 // tipsets
@@ -21,10 +18,10 @@ type tipSetCache struct {
 	start int
 	len   int
 
-	storage tsCacheAPI
+	storage tsByHFunc
 }
 
-func newTSCache(cap abi.ChainEpoch, storage tsCacheAPI) *tipSetCache {
+func newTSCache(cap abi.ChainEpoch, storage tsByHFunc) *tipSetCache {
 	return &tipSetCache{
 		cache: make([]*types.TipSet, cap),
 		start: 0,
@@ -97,7 +94,7 @@ func (tsc *tipSetCache) getNonNull(height abi.ChainEpoch) (*types.TipSet, error)
 func (tsc *tipSetCache) get(height abi.ChainEpoch) (*types.TipSet, error) {
 	if tsc.len == 0 {
 		log.Warnf("tipSetCache.get: cache is empty, requesting from storage (h=%d)", height)
-		return tsc.storage.ChainGetTipSetByHeight(context.TODO(), height, types.EmptyTSK)
+		return tsc.storage(context.TODO(), height, types.EmptyTSK)
 	}
 
 	headH := tsc.cache[tsc.start].Height()
@@ -117,18 +114,14 @@ func (tsc *tipSetCache) get(height abi.ChainEpoch) (*types.TipSet, error) {
 
 	if height < tail.Height() {
 		log.Warnf("tipSetCache.get: requested tipset not in cache, requesting from storage (h=%d; tail=%d)", height, tail.Height())
-		return tsc.storage.ChainGetTipSetByHeight(context.TODO(), height, tail.Key())
+		return tsc.storage(context.TODO(), height, tail.Key())
 	}
 
 	return tsc.cache[normalModulo(tsc.start-int(headH-height), clen)], nil
 }
 
-func (tsc *tipSetCache) best() (*types.TipSet, error) {
-	best := tsc.cache[tsc.start]
-	if best == nil {
-		return tsc.storage.ChainHead(context.TODO())
-	}
-	return best, nil
+func (tsc *tipSetCache) best() *types.TipSet {
+	return tsc.cache[tsc.start]
 }
 
 func normalModulo(n, m int) int {

chain/events/tscache_test.go

@@ -4,8 +4,8 @@ import (
 	"context"
 	"testing"
 
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/go-address"
@@ -13,7 +13,10 @@ import (
 )
 
 func TestTsCache(t *testing.T) {
-	tsc := newTSCache(50, &tsCacheAPIFailOnStorageCall{t: t})
+	tsc := newTSCache(50, func(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) {
+		t.Fatal("storage call")
+		return &types.TipSet{}, nil
+	})
 
 	h := abi.ChainEpoch(75)
 
@@ -40,12 +43,7 @@ func TestTsCache(t *testing.T) {
 
 	for i := 0; i < 9000; i++ {
 		if i%90 > 60 {
-			best, err := tsc.best()
-			if err != nil {
-				t.Fatal(err, "; i:", i)
-				return
-			}
-			if err := tsc.revert(best); err != nil {
+			if err := tsc.revert(tsc.best()); err != nil {
 				t.Fatal(err, "; i:", i)
 				return
 			}
@@ -57,21 +55,11 @@ func TestTsCache(t *testing.T) {
 
 }
 
-type tsCacheAPIFailOnStorageCall struct {
-	t *testing.T
-}
-
-func (tc *tsCacheAPIFailOnStorageCall) ChainGetTipSetByHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) {
-	tc.t.Fatal("storage call")
-	return &types.TipSet{}, nil
-}
-func (tc *tsCacheAPIFailOnStorageCall) ChainHead(ctx context.Context) (*types.TipSet, error) {
-	tc.t.Fatal("storage call")
-	return &types.TipSet{}, nil
-}
-
 func TestTsCacheNulls(t *testing.T) {
-	tsc := newTSCache(50, &tsCacheAPIFailOnStorageCall{t: t})
+	tsc := newTSCache(50, func(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) {
+		t.Fatal("storage call")
+		return &types.TipSet{}, nil
+	})
 
 	h := abi.ChainEpoch(75)
 
@@ -103,9 +91,7 @@ func TestTsCacheNulls(t *testing.T) {
 	add()
 	add()
 
-	best, err := tsc.best()
-	require.NoError(t, err)
-	require.Equal(t, h-1, best.Height())
+	require.Equal(t, h-1, tsc.best().Height())
 
 	ts, err := tsc.get(h - 1)
 	require.NoError(t, err)
@@ -123,17 +109,9 @@ func TestTsCacheNulls(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, h-8, ts.Height())
 
-	best, err = tsc.best()
-	require.NoError(t, err)
-	require.NoError(t, tsc.revert(best))
-
-	best, err = tsc.best()
-	require.NoError(t, err)
-	require.NoError(t, tsc.revert(best))
-
-	best, err = tsc.best()
-	require.NoError(t, err)
-	require.Equal(t, h-8, best.Height())
+	require.NoError(t, tsc.revert(tsc.best()))
+	require.NoError(t, tsc.revert(tsc.best()))
+	require.Equal(t, h-8, tsc.best().Height())
 
 	h += 50
 	add()
@@ -142,27 +120,3 @@ func TestTsCacheNulls(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, h-1, ts.Height())
 }
-
-type tsCacheAPIStorageCallCounter struct {
-	t                      *testing.T
-	chainGetTipSetByHeight int
-	chainHead              int
-}
-
-func (tc *tsCacheAPIStorageCallCounter) ChainGetTipSetByHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) {
-	tc.chainGetTipSetByHeight++
-	return &types.TipSet{}, nil
-}
-func (tc *tsCacheAPIStorageCallCounter) ChainHead(ctx context.Context) (*types.TipSet, error) {
-	tc.chainHead++
-	return &types.TipSet{}, nil
-}
-
-func TestTsCacheEmpty(t *testing.T) {
-	// Calling best on an empty cache should just call out to the chain API
-	callCounter := &tsCacheAPIStorageCallCounter{t: t}
-	tsc := newTSCache(50, callCounter)
-	_, err := tsc.best()
-	require.NoError(t, err)
-	require.Equal(t, 1, callCounter.chainHead)
-}

chain/gen/gen.go

@@ -8,16 +8,14 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/filecoin-project/specs-actors/actors/runtime/proof"
-
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	saminer "github.com/filecoin-project/specs-actors/actors/builtin/miner"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
 	block "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-cid"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	offline "github.com/ipfs/go-ipfs-exchange-offline"
 	format "github.com/ipfs/go-ipld-format"
 	logging "github.com/ipfs/go-log/v2"
@@ -36,19 +34,17 @@ import (
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/chain/wallet"
 	"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
-	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/lotus/genesis"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/lib/sigs"
 	"github.com/filecoin-project/lotus/node/repo"
+	"github.com/filecoin-project/sector-storage/ffiwrapper"
 )
 
+var log = logging.Logger("gen")
+
 const msgsPerBlock = 20
 
-//nolint:deadcode,varcheck
-var log = logging.Logger("gen")
-
-var ValidWpostForTesting = []proof.PoStProof{{
+var ValidWpostForTesting = []abi.PoStProof{{
 	ProofBytes: []byte("valid proof"),
 }}
 
@@ -95,31 +91,6 @@ func (m mybs) Get(c cid.Cid) (block.Block, error) {
 	return b, nil
 }
 
-var rootkeyMultisig = genesis.MultisigMeta{
-	Signers:         []address.Address{remAccTestKey},
-	Threshold:       1,
-	VestingDuration: 0,
-	VestingStart:    0,
-}
-
-var DefaultVerifregRootkeyActor = genesis.Actor{
-	Type:    genesis.TMultisig,
-	Balance: big.NewInt(0),
-	Meta:    rootkeyMultisig.ActorMeta(),
-}
-
-var remAccTestKey, _ = address.NewFromString("t1ceb34gnsc6qk5dt6n7xg6ycwzasjhbxm3iylkiy")
-var remAccMeta = genesis.MultisigMeta{
-	Signers:   []address.Address{remAccTestKey},
-	Threshold: 1,
-}
-
-var DefaultRemainderAccountActor = genesis.Actor{
-	Type:    genesis.TMultisig,
-	Balance: big.NewInt(0),
-	Meta:    remAccMeta.ActorMeta(),
-}
-
 func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
 	saminer.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{
 		abi.RegisteredSealProof_StackedDrg2KiBV1: {},
@@ -141,7 +112,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
 		return nil, xerrors.Errorf("failed to get blocks datastore: %w", err)
 	}
 
-	bs := mybs{blockstore.NewBlockstore(bds)}
+	bs := mybs{blockstore.NewIdStore(blockstore.NewBlockstore(bds))}
 
 	ks, err := lr.KeyStore()
 	if err != nil {
@@ -223,10 +194,8 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
 			*genm1,
 			*genm2,
 		},
-		VerifregRootKey:  DefaultVerifregRootkeyActor,
-		RemainderAccount: DefaultRemainderAccountActor,
-		NetworkName:      "",
-		Timestamp:        uint64(build.Clock.Now().Add(-500 * time.Duration(build.BlockDelaySecs) * time.Second).Unix()),
+		NetworkName: "",
+		Timestamp:   uint64(build.Clock.Now().Add(-500 * time.Duration(build.BlockDelaySecs) * time.Second).Unix()),
 	}
 
 	genb, err := genesis2.MakeGenesisBlock(context.TODO(), bs, sys, tpl)
@@ -286,10 +255,6 @@ func NewGenerator() (*ChainGen, error) {
 	return NewGeneratorWithSectors(1)
 }
 
-func (cg *ChainGen) StateManager() *stmgr.StateManager {
-	return cg.sm
-}
-
 func (cg *ChainGen) SetStateManager(sm *stmgr.StateManager) {
 	cg.sm = sm
 }
@@ -318,8 +283,7 @@ func (cg *ChainGen) GenesisCar() ([]byte, error) {
 
 func CarWalkFunc(nd format.Node) (out []*format.Link, err error) {
 	for _, link := range nd.Links() {
-		pref := link.Cid.Prefix()
-		if pref.Codec == cid.FilCommitmentSealed || pref.Codec == cid.FilCommitmentUnsealed {
+		if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
 			continue
 		}
 		out = append(out, link)
@@ -397,37 +361,16 @@ func (cg *ChainGen) NextTipSet() (*MinedTipSet, error) {
 	return mts, nil
 }
 
-func (cg *ChainGen) SetWinningPoStProver(m address.Address, wpp WinningPoStProver) {
-	cg.eppProvs[m] = wpp
-}
-
 func (cg *ChainGen) NextTipSetFromMiners(base *types.TipSet, miners []address.Address) (*MinedTipSet, error) {
-	ms, err := cg.GetMessages(cg)
+	var blks []*types.FullBlock
+
+	msgs, err := cg.GetMessages(cg)
 	if err != nil {
 		return nil, xerrors.Errorf("get random messages: %w", err)
 	}
 
-	msgs := make([][]*types.SignedMessage, len(miners))
-	for i := range msgs {
-		msgs[i] = ms
-	}
-
-	fts, err := cg.NextTipSetFromMinersWithMessages(base, miners, msgs)
-	if err != nil {
-		return nil, err
-	}
-
-	return &MinedTipSet{
-		TipSet:   fts,
-		Messages: ms,
-	}, nil
-}
-
-func (cg *ChainGen) NextTipSetFromMinersWithMessages(base *types.TipSet, miners []address.Address, msgs [][]*types.SignedMessage) (*store.FullTipSet, error) {
-	var blks []*types.FullBlock
-
 	for round := base.Height() + 1; len(blks) == 0; round++ {
-		for mi, m := range miners {
+		for _, m := range miners {
 			bvals, et, ticket, err := cg.nextBlockProof(context.TODO(), base, m, round)
 			if err != nil {
 				return nil, xerrors.Errorf("next block proof: %w", err)
@@ -440,7 +383,7 @@ func (cg *ChainGen) NextTipSetFromMinersWithMessages(base *types.TipSet, miners
 					return nil, err
 				}
 
-				fblk, err := cg.makeBlock(base, m, ticket, et, bvals, round, wpost, msgs[mi])
+				fblk, err := cg.makeBlock(base, m, ticket, et, bvals, round, wpost, msgs)
 				if err != nil {
 					return nil, xerrors.Errorf("making a block for next tipset failed: %w", err)
 				}
@@ -454,12 +397,17 @@ func (cg *ChainGen) NextTipSetFromMinersWithMessages(base *types.TipSet, miners
 		}
 	}
 
-	return store.NewFullTipSet(blks), nil
+	fts := store.NewFullTipSet(blks)
+
+	return &MinedTipSet{
+		TipSet:   fts,
+		Messages: msgs,
+	}, nil
 }
 
 func (cg *ChainGen) makeBlock(parents *types.TipSet, m address.Address, vrfticket *types.Ticket,
 	eticket *types.ElectionProof, bvals []types.BeaconEntry, height abi.ChainEpoch,
-	wpost []proof.PoStProof, msgs []*types.SignedMessage) (*types.FullBlock, error) {
+	wpost []abi.PoStProof, msgs []*types.SignedMessage) (*types.FullBlock, error) {
 
 	var ts uint64
 	if cg.Timestamper != nil {
@@ -520,9 +468,8 @@ func getRandomMessages(cg *ChainGen) ([]*types.SignedMessage, error) {
 
 			Method: 0,
 
-			GasLimit:   100_000_000,
-			GasFeeCap:  types.NewInt(0),
-			GasPremium: types.NewInt(0),
+			GasLimit: 100_000_000,
+			GasPrice: types.NewInt(0),
 		}
 
 		sig, err := cg.w.Sign(context.TODO(), cg.banker, msg.Cid().Bytes())
@@ -547,8 +494,7 @@ func (cg *ChainGen) YieldRepo() (repo.Repo, error) {
 }
 
 type MiningCheckAPI interface {
-	ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error)
-	ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error)
+	ChainGetRandomness(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error)
 
 	MinerGetBaseInfo(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*api.MiningBaseInfo, error)
 
@@ -562,22 +508,13 @@ type mca struct {
 	bcn beacon.RandomBeacon
 }
 
-func (mca mca) ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
+func (mca mca) ChainGetRandomness(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
 	pts, err := mca.sm.ChainStore().LoadTipSet(tsk)
 	if err != nil {
 		return nil, xerrors.Errorf("loading tipset key: %w", err)
 	}
 
-	return mca.sm.ChainStore().GetChainRandomness(ctx, pts.Cids(), personalization, randEpoch, entropy)
-}
-
-func (mca mca) ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
-	pts, err := mca.sm.ChainStore().LoadTipSet(tsk)
-	if err != nil {
-		return nil, xerrors.Errorf("loading tipset key: %w", err)
-	}
-
-	return mca.sm.ChainStore().GetBeaconRandomness(ctx, pts.Cids(), personalization, randEpoch, entropy)
+	return mca.sm.ChainStore().GetRandomness(ctx, pts.Cids(), personalization, randEpoch, entropy)
 }
 
 func (mca mca) MinerGetBaseInfo(ctx context.Context, maddr address.Address, epoch abi.ChainEpoch, tsk types.TipSetKey) (*api.MiningBaseInfo, error) {
@@ -590,7 +527,7 @@ func (mca mca) WalletSign(ctx context.Context, a address.Address, v []byte) (*cr
 
 type WinningPoStProver interface {
 	GenerateCandidates(context.Context, abi.PoStRandomness, uint64) ([]uint64, error)
-	ComputeProof(context.Context, []proof.SectorInfo, abi.PoStRandomness) ([]proof.PoStProof, error)
+	ComputeProof(context.Context, []abi.SectorInfo, abi.PoStRandomness) ([]abi.PoStProof, error)
 }
 
 type wppProvider struct{}
@@ -599,7 +536,7 @@ func (wpp *wppProvider) GenerateCandidates(ctx context.Context, _ abi.PoStRandom
 	return []uint64{0}, nil
 }
 
-func (wpp *wppProvider) ComputeProof(context.Context, []proof.SectorInfo, abi.PoStRandomness) ([]proof.PoStProof, error) {
+func (wpp *wppProvider) ComputeProof(context.Context, []abi.SectorInfo, abi.PoStRandomness) ([]abi.PoStProof, error) {
 	return ValidWpostForTesting, nil
 }
 
@@ -608,7 +545,7 @@ func IsRoundWinner(ctx context.Context, ts *types.TipSet, round abi.ChainEpoch,
 
 	buf := new(bytes.Buffer)
 	if err := miner.MarshalCBOR(buf); err != nil {
-		return nil, xerrors.Errorf("failed to cbor marshal address: %w", err)
+		return nil, xerrors.Errorf("failed to cbor marshal address: %w")
 	}
 
 	electionRand, err := store.DrawRandomness(brand.Data, crypto.DomainSeparationTag_ElectionProofProduction, round, buf.Bytes())
@@ -666,15 +603,15 @@ type genFakeVerifier struct{}
 
 var _ ffiwrapper.Verifier = (*genFakeVerifier)(nil)
 
-func (m genFakeVerifier) VerifySeal(svi proof.SealVerifyInfo) (bool, error) {
+func (m genFakeVerifier) VerifySeal(svi abi.SealVerifyInfo) (bool, error) {
 	return true, nil
 }
 
-func (m genFakeVerifier) VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error) {
+func (m genFakeVerifier) VerifyWinningPoSt(ctx context.Context, info abi.WinningPoStVerifyInfo) (bool, error) {
 	panic("not supported")
 }
 
-func (m genFakeVerifier) VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error) {
+func (m genFakeVerifier) VerifyWindowPoSt(ctx context.Context, info abi.WindowPoStVerifyInfo) (bool, error) {
 	panic("not supported")
 }
 

chain/gen/gen_test.go

@@ -3,8 +3,8 @@ package gen
 import (
 	"testing"
 
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
@@ -39,8 +39,8 @@ func testGeneration(t testing.TB, n int, msgs int, sectors int) {
 }
 
 func TestChainGeneration(t *testing.T) {
-	t.Run("10-20-1", func(t *testing.T) { testGeneration(t, 10, 20, 1) })
-	t.Run("10-20-25", func(t *testing.T) { testGeneration(t, 10, 20, 25) })
+	testGeneration(t, 10, 20, 1)
+	testGeneration(t, 10, 20, 25)
 }
 
 func BenchmarkChainGeneration(b *testing.B) {

chain/gen/genesis/genblock.go

@@ -1,41 +0,0 @@
-package genesis
-
-import (
-	"encoding/hex"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	"github.com/multiformats/go-multihash"
-)
-
-const genesisMultihashString = "1220107d821c25dc0735200249df94a8bebc9c8e489744f86a4ca8919e81f19dcd72"
-const genesisBlockHex = "a5684461746574696d6573323031372d30352d30352030313a32373a3531674e6574776f726b6846696c65636f696e65546f6b656e6846696c65636f696e6c546f6b656e416d6f756e7473a36b546f74616c537570706c796d322c3030302c3030302c303030664d696e6572736d312c3430302c3030302c3030306c50726f746f636f6c4c616273a36b446576656c6f706d656e746b3330302c3030302c3030306b46756e6472616973696e676b3230302c3030302c3030306a466f756e646174696f6e6b3130302c3030302c303030674d657373616765784854686973206973207468652047656e6573697320426c6f636b206f66207468652046696c65636f696e20446563656e7472616c697a65642053746f72616765204e6574776f726b2e"
-
-var cidBuilder = cid.V1Builder{Codec: cid.DagCBOR, MhType: multihash.SHA2_256}
-
-func expectedCid() cid.Cid {
-	mh, err := multihash.FromHexString(genesisMultihashString)
-	if err != nil {
-		panic(err)
-	}
-	return cid.NewCidV1(cidBuilder.Codec, mh)
-}
-
-func getGenesisBlock() (blocks.Block, error) {
-	genesisBlockData, err := hex.DecodeString(genesisBlockHex)
-	if err != nil {
-		return nil, err
-	}
-
-	genesisCid, err := cidBuilder.Sum(genesisBlockData)
-	if err != nil {
-		return nil, err
-	}
-
-	block, err := blocks.NewBlockWithCid(genesisBlockData, genesisCid)
-	if err != nil {
-		return nil, err
-	}
-
-	return block, nil
-}

chain/gen/genesis/genesis.go

@@ -2,37 +2,30 @@ package genesis
 
 import (
 	"context"
-	"crypto/rand"
 	"encoding/json"
-	"fmt"
 
+	"github.com/filecoin-project/go-amt-ipld/v2"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/builtin"
+	"github.com/filecoin-project/specs-actors/actors/builtin/account"
+	"github.com/filecoin-project/specs-actors/actors/builtin/multisig"
+	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
 	"github.com/filecoin-project/specs-actors/actors/runtime"
-
+	"github.com/filecoin-project/specs-actors/actors/util/adt"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
+	bstore "github.com/ipfs/go-ipfs-blockstore"
 	cbor "github.com/ipfs/go-ipld-cbor"
 	logging "github.com/ipfs/go-log/v2"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
 
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/go-state-types/crypto"
-	"github.com/filecoin-project/specs-actors/actors/builtin"
-	"github.com/filecoin-project/specs-actors/actors/builtin/account"
-	"github.com/filecoin-project/specs-actors/actors/builtin/multisig"
-	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
-	"github.com/filecoin-project/specs-actors/actors/util/adt"
-
-	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/state"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/genesis"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
-	"github.com/filecoin-project/lotus/lib/sigs"
 )
 
 const AccountStart = 100
@@ -69,7 +62,6 @@ The process:
       - market.AddFunds with correct value
       - market.PublishDeals for related sectors
     - Set network power in the power actor to what we'll have after genesis creation
-	- Recreate reward actor state with the right power
     - For each precommitted sector
       - Get deal weight
       - Calculate QA Power
@@ -107,354 +99,217 @@ Genesis: {
 
 */
 
-func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template genesis.Template) (*state.StateTree, map[address.Address]address.Address, error) {
+func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template genesis.Template) (*state.StateTree, error) {
 	// Create empty state tree
 
 	cst := cbor.NewCborStore(bs)
 	_, err := cst.Put(context.TODO(), []struct{}{})
 	if err != nil {
-		return nil, nil, xerrors.Errorf("putting empty object: %w", err)
+		return nil, xerrors.Errorf("putting empty object: %w", err)
 	}
 
 	state, err := state.NewStateTree(cst)
 	if err != nil {
-		return nil, nil, xerrors.Errorf("making new state tree: %w", err)
+		return nil, xerrors.Errorf("making new state tree: %w", err)
+	}
+
+	emptyobject, err := cst.Put(context.TODO(), []struct{}{})
+	if err != nil {
+		return nil, xerrors.Errorf("failed putting empty object: %w", err)
 	}
 
 	// Create system actor
 
 	sysact, err := SetupSystemActor(bs)
 	if err != nil {
-		return nil, nil, xerrors.Errorf("setup init actor: %w", err)
+		return nil, xerrors.Errorf("setup init actor: %w", err)
 	}
 	if err := state.SetActor(builtin.SystemActorAddr, sysact); err != nil {
-		return nil, nil, xerrors.Errorf("set init actor: %w", err)
+		return nil, xerrors.Errorf("set init actor: %w", err)
 	}
 
 	// Create init actor
 
-	idStart, initact, keyIDs, err := SetupInitActor(bs, template.NetworkName, template.Accounts, template.VerifregRootKey)
+	initact, err := SetupInitActor(bs, template.NetworkName, template.Accounts)
 	if err != nil {
-		return nil, nil, xerrors.Errorf("setup init actor: %w", err)
+		return nil, xerrors.Errorf("setup init actor: %w", err)
 	}
 	if err := state.SetActor(builtin.InitActorAddr, initact); err != nil {
-		return nil, nil, xerrors.Errorf("set init actor: %w", err)
+		return nil, xerrors.Errorf("set init actor: %w", err)
 	}
 
 	// Setup reward
-	// RewardActor's state is overrwritten by SetupStorageMiners
-	rewact, err := SetupRewardActor(bs, big.Zero())
+	rewact, err := SetupRewardActor(bs)
 	if err != nil {
-		return nil, nil, xerrors.Errorf("setup init actor: %w", err)
+		return nil, xerrors.Errorf("setup init actor: %w", err)
 	}
 
 	err = state.SetActor(builtin.RewardActorAddr, rewact)
 	if err != nil {
-		return nil, nil, xerrors.Errorf("set network account actor: %w", err)
+		return nil, xerrors.Errorf("set network account actor: %w", err)
 	}
 
 	// Setup cron
 	cronact, err := SetupCronActor(bs)
 	if err != nil {
-		return nil, nil, xerrors.Errorf("setup cron actor: %w", err)
+		return nil, xerrors.Errorf("setup cron actor: %w", err)
 	}
 	if err := state.SetActor(builtin.CronActorAddr, cronact); err != nil {
-		return nil, nil, xerrors.Errorf("set cron actor: %w", err)
+		return nil, xerrors.Errorf("set cron actor: %w", err)
 	}
 
 	// Create empty power actor
 	spact, err := SetupStoragePowerActor(bs)
 	if err != nil {
-		return nil, nil, xerrors.Errorf("setup storage market actor: %w", err)
+		return nil, xerrors.Errorf("setup storage market actor: %w", err)
 	}
 	if err := state.SetActor(builtin.StoragePowerActorAddr, spact); err != nil {
-		return nil, nil, xerrors.Errorf("set storage market actor: %w", err)
+		return nil, xerrors.Errorf("set storage market actor: %w", err)
 	}
 
 	// Create empty market actor
 	marketact, err := SetupStorageMarketActor(bs)
 	if err != nil {
-		return nil, nil, xerrors.Errorf("setup storage market actor: %w", err)
+		return nil, xerrors.Errorf("setup storage market actor: %w", err)
 	}
 	if err := state.SetActor(builtin.StorageMarketActorAddr, marketact); err != nil {
-		return nil, nil, xerrors.Errorf("set market actor: %w", err)
+		return nil, xerrors.Errorf("set market actor: %w", err)
 	}
 
 	// Create verified registry
 	verifact, err := SetupVerifiedRegistryActor(bs)
 	if err != nil {
-		return nil, nil, xerrors.Errorf("setup storage market actor: %w", err)
+		return nil, xerrors.Errorf("setup storage market actor: %w", err)
 	}
 	if err := state.SetActor(builtin.VerifiedRegistryActorAddr, verifact); err != nil {
-		return nil, nil, xerrors.Errorf("set market actor: %w", err)
-	}
-
-	burntRoot, err := cst.Put(ctx, &account.State{
-		Address: builtin.BurntFundsActorAddr,
-	})
-	if err != nil {
-		return nil, nil, xerrors.Errorf("failed to setup burnt funds actor state: %w", err)
+		return nil, xerrors.Errorf("set market actor: %w", err)
 	}
 
 	// Setup burnt-funds
 	err = state.SetActor(builtin.BurntFundsActorAddr, &types.Actor{
 		Code:    builtin.AccountActorCodeID,
 		Balance: types.NewInt(0),
-		Head:    burntRoot,
+		Head:    emptyobject,
 	})
 	if err != nil {
-		return nil, nil, xerrors.Errorf("set burnt funds account actor: %w", err)
+		return nil, xerrors.Errorf("set burnt funds account actor: %w", err)
 	}
 
 	// Create accounts
-	for _, info := range template.Accounts {
+	for id, info := range template.Accounts {
+		if info.Type != genesis.TAccount && info.Type != genesis.TMultisig {
+			return nil, xerrors.New("unsupported account type")
+		}
 
-		switch info.Type {
-		case genesis.TAccount:
-			if err := createAccountActor(ctx, cst, state, info, keyIDs); err != nil {
-				return nil, nil, xerrors.Errorf("failed to create account actor: %w", err)
+		ida, err := address.NewIDAddress(uint64(AccountStart + id))
+		if err != nil {
+			return nil, err
+		}
+
+		// var newAddress address.Address
+
+		if info.Type == genesis.TAccount {
+			var ainfo genesis.AccountMeta
+			if err := json.Unmarshal(info.Meta, &ainfo); err != nil {
+				return nil, xerrors.Errorf("unmarshaling account meta: %w", err)
 			}
-
-		case genesis.TMultisig:
-
-			ida, err := address.NewIDAddress(uint64(idStart))
+			st, err := cst.Put(ctx, &account.State{Address: ainfo.Owner})
 			if err != nil {
-				return nil, nil, err
+				return nil, err
+			}
+			err = state.SetActor(ida, &types.Actor{
+				Code:    builtin.AccountActorCodeID,
+				Balance: info.Balance,
+				Head:    st,
+			})
+			if err != nil {
+				return nil, xerrors.Errorf("setting account from actmap: %w", err)
+			}
+		} else if info.Type == genesis.TMultisig {
+			var ainfo genesis.MultisigMeta
+			if err := json.Unmarshal(info.Meta, &ainfo); err != nil {
+				return nil, xerrors.Errorf("unmarshaling account meta: %w", err)
 			}
-			idStart++
 
-			if err := createMultisigAccount(ctx, bs, cst, state, ida, info, keyIDs); err != nil {
-				return nil, nil, err
+			pending, err := adt.MakeEmptyMap(adt.WrapStore(ctx, cst)).Root()
+			if err != nil {
+				return nil, xerrors.Errorf("failed to create empty map: %v", err)
+			}
+
+			st, err := cst.Put(ctx, &multisig.State{
+				Signers:               ainfo.Signers,
+				NumApprovalsThreshold: uint64(ainfo.Threshold),
+				StartEpoch:            abi.ChainEpoch(ainfo.VestingStart),
+				UnlockDuration:        abi.ChainEpoch(ainfo.VestingDuration),
+				PendingTxns:           pending,
+				InitialBalance:        info.Balance,
+			})
+			if err != nil {
+				return nil, err
+			}
+			err = state.SetActor(ida, &types.Actor{
+				Code:    builtin.MultisigActorCodeID,
+				Balance: info.Balance,
+				Head:    st,
+			})
+			if err != nil {
+				return nil, xerrors.Errorf("setting account from actmap: %w", err)
 			}
-		default:
-			return nil, nil, xerrors.New("unsupported account type")
 		}
 
 	}
 
 	vregroot, err := address.NewIDAddress(80)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	if err = createMultisigAccount(ctx, bs, cst, state, vregroot, template.VerifregRootKey, keyIDs); err != nil {
-		return nil, nil, xerrors.Errorf("failed to set up verified registry signer: %w", err)
-	}
-
-	// Setup the first verifier as ID-address 81
-	// TODO: remove this
-	skBytes, err := sigs.Generate(crypto.SigTypeBLS)
+	vrst, err := cst.Put(ctx, &account.State{Address: RootVerifierAddr})
 	if err != nil {
-		return nil, nil, xerrors.Errorf("creating random verifier secret key: %w", err)
+		return nil, err
 	}
 
-	verifierPk, err := sigs.ToPublic(crypto.SigTypeBLS, skBytes)
-	if err != nil {
-		return nil, nil, xerrors.Errorf("creating random verifier public key: %w", err)
-	}
-
-	verifierAd, err := address.NewBLSAddress(verifierPk)
-	if err != nil {
-		return nil, nil, xerrors.Errorf("creating random verifier address: %w", err)
-	}
-
-	verifierId, err := address.NewIDAddress(81)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	verifierState, err := cst.Put(ctx, &account.State{Address: verifierAd})
-	if err != nil {
-		return nil, nil, err
-	}
-
-	err = state.SetActor(verifierId, &types.Actor{
+	err = state.SetActor(vregroot, &types.Actor{
 		Code:    builtin.AccountActorCodeID,
 		Balance: types.NewInt(0),
-		Head:    verifierState,
+		Head:    vrst,
 	})
+
 	if err != nil {
-		return nil, nil, xerrors.Errorf("setting account from actmap: %w", err)
+		return nil, xerrors.Errorf("setting account from actmap: %w", err)
 	}
 
-	totalFilAllocated := big.Zero()
-
-	// flush as ForEach works on the HAMT
-	if _, err := state.Flush(ctx); err != nil {
-		return nil, nil, err
-	}
-	err = state.ForEach(func(addr address.Address, act *types.Actor) error {
-		totalFilAllocated = big.Add(totalFilAllocated, act.Balance)
-		return nil
-	})
-	if err != nil {
-		return nil, nil, xerrors.Errorf("summing account balances in state tree: %w", err)
-	}
-
-	totalFil := big.Mul(big.NewInt(int64(build.FilBase)), big.NewInt(int64(build.FilecoinPrecision)))
-	remainingFil := big.Sub(totalFil, totalFilAllocated)
-	if remainingFil.Sign() < 0 {
-		return nil, nil, xerrors.Errorf("somehow overallocated filecoin (allocated = %s)", types.FIL(totalFilAllocated))
-	}
-
-	remAccKey, err := address.NewIDAddress(90)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	template.RemainderAccount.Balance = remainingFil
-
-	if err := createMultisigAccount(ctx, bs, cst, state, remAccKey, template.RemainderAccount, keyIDs); err != nil {
-		return nil, nil, xerrors.Errorf("failed to set up remainder account: %w", err)
-	}
-
-	return state, keyIDs, nil
+	return state, nil
 }
 
-func createAccountActor(ctx context.Context, cst cbor.IpldStore, state *state.StateTree, info genesis.Actor, keyIDs map[address.Address]address.Address) error {
-	var ainfo genesis.AccountMeta
-	if err := json.Unmarshal(info.Meta, &ainfo); err != nil {
-		return xerrors.Errorf("unmarshaling account meta: %w", err)
-	}
-	st, err := cst.Put(ctx, &account.State{Address: ainfo.Owner})
-	if err != nil {
-		return err
-	}
-
-	ida, ok := keyIDs[ainfo.Owner]
-	if !ok {
-		return fmt.Errorf("no registered ID for account actor: %s", ainfo.Owner)
-	}
-
-	err = state.SetActor(ida, &types.Actor{
-		Code:    builtin.AccountActorCodeID,
-		Balance: info.Balance,
-		Head:    st,
-	})
-	if err != nil {
-		return xerrors.Errorf("setting account from actmap: %w", err)
-	}
-	return nil
-}
-
-func createMultisigAccount(ctx context.Context, bs bstore.Blockstore, cst cbor.IpldStore, state *state.StateTree, ida address.Address, info genesis.Actor, keyIDs map[address.Address]address.Address) error {
-	if info.Type != genesis.TMultisig {
-		return fmt.Errorf("can only call createMultisigAccount with multisig Actor info")
-	}
-	var ainfo genesis.MultisigMeta
-	if err := json.Unmarshal(info.Meta, &ainfo); err != nil {
-		return xerrors.Errorf("unmarshaling account meta: %w", err)
-	}
-	pending, err := adt.MakeEmptyMap(adt.WrapStore(ctx, cst)).Root()
-	if err != nil {
-		return xerrors.Errorf("failed to create empty map: %v", err)
-	}
-
-	var signers []address.Address
-
-	for _, e := range ainfo.Signers {
-		idAddress, ok := keyIDs[e]
-		if !ok {
-			return fmt.Errorf("no registered key ID for signer: %s", e)
-		}
-
-		// Check if actor already exists
-		_, err := state.GetActor(e)
-		if err == nil {
-			signers = append(signers, idAddress)
-			continue
-		}
-
-		st, err := cst.Put(ctx, &account.State{Address: e})
-		if err != nil {
-			return err
-		}
-		err = state.SetActor(idAddress, &types.Actor{
-			Code:    builtin.AccountActorCodeID,
-			Balance: types.NewInt(0),
-			Head:    st,
-		})
-		if err != nil {
-			return xerrors.Errorf("setting account from actmap: %w", err)
-		}
-		signers = append(signers, idAddress)
-	}
-
-	st, err := cst.Put(ctx, &multisig.State{
-		Signers:               signers,
-		NumApprovalsThreshold: uint64(ainfo.Threshold),
-		StartEpoch:            abi.ChainEpoch(ainfo.VestingStart),
-		UnlockDuration:        abi.ChainEpoch(ainfo.VestingDuration),
-		PendingTxns:           pending,
-		InitialBalance:        info.Balance,
-	})
-	if err != nil {
-		return err
-	}
-	err = state.SetActor(ida, &types.Actor{
-		Code:    builtin.MultisigActorCodeID,
-		Balance: info.Balance,
-		Head:    st,
-	})
-	if err != nil {
-		return xerrors.Errorf("setting account from actmap: %w", err)
-	}
-	return nil
-}
-
-func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot cid.Cid, template genesis.Template, keyIDs map[address.Address]address.Address) (cid.Cid, error) {
+func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot cid.Cid, template genesis.Template) (cid.Cid, error) {
 	verifNeeds := make(map[address.Address]abi.PaddedPieceSize)
 	var sum abi.PaddedPieceSize
-
-	nwv := func(context.Context, abi.ChainEpoch) runtime.NetworkVersion {
-		return runtime.NetworkVersion1
-	}
-
-	vmopt := vm.VMOpts{
-		StateBase:      stateroot,
-		Epoch:          0,
-		Rand:           &fakeRand{},
-		Bstore:         cs.Blockstore(),
-		Syscalls:       mkFakedSigSyscalls(cs.VMSys()),
-		CircSupplyCalc: nil,
-		NtwkVersion:    nwv,
-		BaseFee:        types.NewInt(0),
-	}
-	vm, err := vm.NewVM(&vmopt)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("failed to create NewVM: %w", err)
-	}
-
-	for mi, m := range template.Miners {
-		for si, s := range m.Sectors {
-			if s.Deal.Provider != m.ID {
-				return cid.Undef, xerrors.Errorf("Sector %d in miner %d in template had mismatch in provider and miner ID: %s != %s", si, mi, s.Deal.Provider, m.ID)
-			}
-
+	for _, m := range template.Miners {
+		for _, s := range m.Sectors {
 			amt := s.Deal.PieceSize
-			verifNeeds[keyIDs[s.Deal.Client]] += amt
+			verifNeeds[s.Deal.Client] += amt
 			sum += amt
 		}
 	}
 
-	verifregRoot, err := address.NewIDAddress(80)
+	verifier, err := address.NewIDAddress(80)
 	if err != nil {
 		return cid.Undef, err
 	}
 
-	verifier, err := address.NewIDAddress(81)
+	vm, err := vm.NewVM(stateroot, 0, &fakeRand{}, cs.Blockstore(), &fakedSigSyscalls{cs.VMSys()})
 	if err != nil {
-		return cid.Undef, err
+		return cid.Undef, xerrors.Errorf("failed to create NewVM: %w", err)
 	}
 
-	_, err = doExecValue(ctx, vm, builtin.VerifiedRegistryActorAddr, verifregRoot, types.NewInt(0), builtin.MethodsVerifiedRegistry.AddVerifier, mustEnc(&verifreg.AddVerifierParams{
-
+	_, err = doExecValue(ctx, vm, builtin.VerifiedRegistryActorAddr, RootVerifierAddr, types.NewInt(0), builtin.MethodsVerifiedRegistry.AddVerifier, mustEnc(&verifreg.AddVerifierParams{
 		Address:   verifier,
 		Allowance: abi.NewStoragePower(int64(sum)), // eh, close enough
 
 	}))
 	if err != nil {
-		return cid.Undef, xerrors.Errorf("failed to create verifier: %w", err)
+		return cid.Undef, xerrors.Errorf("failed to failed to create verifier: %w", err)
 	}
 
 	for c, amt := range verifNeeds {
@@ -475,8 +330,8 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci
 	return st, nil
 }
 
-func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallBuilder, template genesis.Template) (*GenesisBootstrap, error) {
-	st, keyIDs, err := MakeInitialStateTree(ctx, bs, template)
+func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys runtime.Syscalls, template genesis.Template) (*GenesisBootstrap, error) {
+	st, err := MakeInitialStateTree(ctx, bs, template)
 	if err != nil {
 		return nil, xerrors.Errorf("make initial state tree failed: %w", err)
 	}
@@ -490,7 +345,7 @@ func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallB
 	cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys)
 
 	// Verify PreSealed Data
-	stateroot, err = VerifyPreSealedData(ctx, cs, stateroot, template, keyIDs)
+	stateroot, err = VerifyPreSealedData(ctx, cs, stateroot, template)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to verify presealed data: %w", err)
 	}
@@ -500,8 +355,9 @@ func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallB
 		return nil, xerrors.Errorf("setup miners failed: %w", err)
 	}
 
-	store := adt.WrapStore(ctx, cbor.NewCborStore(bs))
-	emptyroot, err := adt.MakeEmptyArray(store).Root()
+	cst := cbor.NewCborStore(bs)
+
+	emptyroot, err := amt.FromArray(ctx, cst, nil)
 	if err != nil {
 		return nil, xerrors.Errorf("amt build failed: %w", err)
 	}
@@ -520,38 +376,14 @@ func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallB
 
 	log.Infof("Empty Genesis root: %s", emptyroot)
 
-	tickBuf := make([]byte, 32)
-	_, _ = rand.Read(tickBuf)
 	genesisticket := &types.Ticket{
-		VRFProof: tickBuf,
-	}
-
-	filecoinGenesisCid, err := cid.Decode("bafyreiaqpwbbyjo4a42saasj36kkrpv4tsherf2e7bvezkert2a7dhonoi")
-	if err != nil {
-		return nil, xerrors.Errorf("failed to decode filecoin genesis block CID: %w", err)
-	}
-
-	if !expectedCid().Equals(filecoinGenesisCid) {
-		return nil, xerrors.Errorf("expectedCid != filecoinGenesisCid")
-	}
-
-	gblk, err := getGenesisBlock()
-	if err != nil {
-		return nil, xerrors.Errorf("failed to construct filecoin genesis block: %w", err)
-	}
-
-	if !filecoinGenesisCid.Equals(gblk.Cid()) {
-		return nil, xerrors.Errorf("filecoinGenesisCid != gblk.Cid")
-	}
-
-	if err := bs.Put(gblk); err != nil {
-		return nil, xerrors.Errorf("failed writing filecoin genesis block to blockstore: %w", err)
+		VRFProof: []byte("vrf proof0000000vrf proof0000000"),
 	}
 
 	b := &types.BlockHeader{
 		Miner:                 builtin.SystemActorAddr,
 		Ticket:                genesisticket,
-		Parents:               []cid.Cid{filecoinGenesisCid},
+		Parents:               []cid.Cid{},
 		Height:                0,
 		ParentWeight:          types.NewInt(0),
 		ParentStateRoot:       stateroot,
@@ -567,7 +399,6 @@ func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallB
 				Data:  make([]byte, 32),
 			},
 		},
-		ParentBaseFee: abi.NewTokenAmount(build.InitialBaseFee),
 	}
 
 	sb, err := b.ToStorageBlock()

chain/gen/genesis/miners.go

@@ -6,23 +6,21 @@ import (
 	"fmt"
 	"math/rand"
 
-	"github.com/filecoin-project/lotus/chain/state"
-
 	"github.com/ipfs/go-cid"
 	cbor "github.com/ipfs/go-ipld-cbor"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/go-state-types/crypto"
-	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
+	"github.com/filecoin-project/sector-storage/ffiwrapper"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/market"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 	"github.com/filecoin-project/specs-actors/actors/builtin/reward"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
 	"github.com/filecoin-project/specs-actors/actors/runtime"
 
 	"github.com/filecoin-project/lotus/chain/store"
@@ -48,35 +46,8 @@ func (fss *fakedSigSyscalls) VerifySignature(signature crypto.Signature, signer
 	return nil
 }
 
-func mkFakedSigSyscalls(base vm.SyscallBuilder) vm.SyscallBuilder {
-	return func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime.Syscalls {
-		return &fakedSigSyscalls{
-			base(ctx, cstate, cst),
-		}
-	}
-}
-
 func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid, miners []genesis.Miner) (cid.Cid, error) {
-	csc := func(context.Context, abi.ChainEpoch, *state.StateTree) (abi.TokenAmount, error) {
-		return big.Zero(), nil
-	}
-
-	nwv := func(context.Context, abi.ChainEpoch) runtime.NetworkVersion {
-		return runtime.NetworkVersion1
-	}
-
-	vmopt := &vm.VMOpts{
-		StateBase:      sroot,
-		Epoch:          0,
-		Rand:           &fakeRand{},
-		Bstore:         cs.Blockstore(),
-		Syscalls:       mkFakedSigSyscalls(cs.VMSys()),
-		CircSupplyCalc: csc,
-		NtwkVersion:    nwv,
-		BaseFee:        types.NewInt(0),
-	}
-
-	vm, err := vm.NewVM(vmopt)
+	vm, err := vm.NewVM(sroot, 0, &fakeRand{}, cs.Blockstore(), &fakedSigSyscalls{cs.VMSys()})
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("failed to create NewVM: %w", err)
 	}
@@ -134,18 +105,22 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 
 				return nil
 			})
-			if err != nil {
-				return cid.Undef, xerrors.Errorf("mutating state: %w", err)
-			}
 		}
 
 		// Add market funds
 
-		if m.MarketBalance.GreaterThan(big.Zero()) {
+		{
 			params := mustEnc(&minerInfos[i].maddr)
 			_, err := doExecValue(ctx, vm, builtin.StorageMarketActorAddr, m.Worker, m.MarketBalance, builtin.MethodsMarket.AddBalance, params)
 			if err != nil {
-				return cid.Undef, xerrors.Errorf("failed to create genesis miner (add balance): %w", err)
+				return cid.Undef, xerrors.Errorf("failed to create genesis miner: %w", err)
+			}
+		}
+		{
+			params := mustEnc(&m.Worker)
+			_, err := doExecValue(ctx, vm, builtin.StorageMarketActorAddr, m.Worker, big.Zero(), builtin.MethodsMarket.AddBalance, params)
+			if err != nil {
+				return cid.Undef, xerrors.Errorf("failed to create genesis miner: %w", err)
 			}
 		}
 
@@ -157,7 +132,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 
 				ret, err := doExecValue(ctx, vm, builtin.StorageMarketActorAddr, m.Worker, big.Zero(), builtin.MethodsMarket.PublishStorageDeals, mustEnc(params))
 				if err != nil {
-					return xerrors.Errorf("failed to create genesis miner (publish deals): %w", err)
+					return xerrors.Errorf("failed to create genesis miner: %w", err)
 				}
 				var ids market.PublishStorageDealsReturn
 				if err := ids.UnmarshalCBOR(bytes.NewReader(ret)); err != nil {
@@ -223,14 +198,6 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 		if err != nil {
 			return cid.Undef, xerrors.Errorf("mutating state: %w", err)
 		}
-
-		err = vm.MutateState(ctx, builtin.RewardActorAddr, func(sct cbor.IpldStore, st *reward.State) error {
-			*st = *reward.ConstructState(qaPow)
-			return nil
-		})
-		if err != nil {
-			return cid.Undef, xerrors.Errorf("mutating state: %w", err)
-		}
 	}
 
 	for i, m := range miners {
@@ -255,7 +222,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 
 				// we've added fake power for this sector above, remove it now
 				err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *power.State) error {
-					st.TotalQualityAdjPower = types.BigSub(st.TotalQualityAdjPower, sectorWeight) //nolint:scopelint
+					st.TotalQualityAdjPower = types.BigSub(st.TotalQualityAdjPower, sectorWeight)
 					st.TotalRawBytePower = types.BigSub(st.TotalRawBytePower, types.NewInt(uint64(m.SectorSize)))
 					return nil
 				})
@@ -273,19 +240,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 					return cid.Undef, xerrors.Errorf("getting current total power: %w", err)
 				}
 
-				pcd := miner.PreCommitDepositForPower(epochReward.ThisEpochRewardSmoothed, tpow.QualityAdjPowerSmoothed, sectorWeight)
-
-				pledge := miner.InitialPledgeForPower(
-					sectorWeight,
-					epochReward.ThisEpochBaselinePower,
-					tpow.PledgeCollateral,
-					epochReward.ThisEpochRewardSmoothed,
-					tpow.QualityAdjPowerSmoothed,
-					circSupply(ctx, vm, minerInfos[i].maddr),
-				)
-
-				pledge = big.Add(pcd, pledge)
-
+				pledge := miner.InitialPledgeForPower(sectorWeight, tpow.QualityAdjPower, epochReward.ThisEpochBaselinePower, tpow.PledgeCollateral, epochReward.ThisEpochReward, circSupply(ctx, vm, minerInfos[i].maddr))
 				fmt.Println(types.FIL(pledge))
 				_, err = doExecValue(ctx, vm, minerInfos[i].maddr, m.Worker, pledge, builtin.MethodsMiner.PreCommitSector, mustEnc(params))
 				if err != nil {
@@ -333,15 +288,9 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 // TODO: copied from actors test harness, deduplicate or remove from here
 type fakeRand struct{}
 
-func (fr *fakeRand) GetChainRandomness(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
+func (fr *fakeRand) GetRandomness(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
 	out := make([]byte, 32)
-	_, _ = rand.New(rand.NewSource(int64(randEpoch * 1000))).Read(out) //nolint
-	return out, nil
-}
-
-func (fr *fakeRand) GetBeaconRandomness(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
-	out := make([]byte, 32)
-	_, _ = rand.New(rand.NewSource(int64(randEpoch))).Read(out) //nolint
+	_, _ = rand.New(rand.NewSource(int64(randEpoch))).Read(out)
 	return out, nil
 }
 

chain/gen/genesis/t00_system.go

@@ -2,14 +2,12 @@ package genesis
 
 import (
 	"context"
-
 	"github.com/filecoin-project/specs-actors/actors/builtin/system"
 
-	"github.com/filecoin-project/specs-actors/actors/builtin"
-	cbor "github.com/ipfs/go-ipld-cbor"
-
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
+	"github.com/filecoin-project/specs-actors/actors/builtin"
+	bstore "github.com/ipfs/go-ipfs-blockstore"
+	cbor "github.com/ipfs/go-ipld-cbor"
 )
 
 func SetupSystemActor(bs bstore.Blockstore) (*types.Actor, error) {

chain/gen/genesis/t01_init.go

@@ -5,133 +5,68 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"github.com/filecoin-project/go-address"
-
 	"github.com/filecoin-project/specs-actors/actors/builtin"
-	"github.com/filecoin-project/specs-actors/actors/util/adt"
 
 	init_ "github.com/filecoin-project/specs-actors/actors/builtin/init"
+	"github.com/ipfs/go-hamt-ipld"
+	bstore "github.com/ipfs/go-ipfs-blockstore"
 	cbor "github.com/ipfs/go-ipld-cbor"
-	cbg "github.com/whyrusleeping/cbor-gen"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/genesis"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
-func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesis.Actor, rootVerifier genesis.Actor) (int64, *types.Actor, map[address.Address]address.Address, error) {
+func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesis.Actor) (*types.Actor, error) {
 	if len(initialActors) > MaxAccounts {
-		return 0, nil, nil, xerrors.New("too many initial actors")
+		return nil, xerrors.New("too many initial actors")
 	}
 
 	var ias init_.State
 	ias.NextID = MinerStart
 	ias.NetworkName = netname
 
-	store := adt.WrapStore(context.TODO(), cbor.NewCborStore(bs))
-	amap := adt.MakeEmptyMap(store)
+	cst := cbor.NewCborStore(bs)
+	amap := hamt.NewNode(cst, hamt.UseTreeBitWidth(5)) // TODO: use spec adt map
 
-	keyToId := map[address.Address]address.Address{}
-	counter := int64(AccountStart)
-
-	for _, a := range initialActors {
+	for i, a := range initialActors {
 		if a.Type == genesis.TMultisig {
-			var ainfo genesis.MultisigMeta
-			if err := json.Unmarshal(a.Meta, &ainfo); err != nil {
-				return 0, nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
-			}
-			for _, e := range ainfo.Signers {
-
-				if _, ok := keyToId[e]; ok {
-					continue
-				}
-
-				fmt.Printf("init set %s t0%d\n", e, counter)
-
-				value := cbg.CborInt(counter)
-				if err := amap.Put(adt.AddrKey(e), &value); err != nil {
-					return 0, nil, nil, err
-				}
-				counter = counter + 1
-				var err error
-				keyToId[e], err = address.NewIDAddress(uint64(value))
-				if err != nil {
-					return 0, nil, nil, err
-				}
-
-			}
-			// Need to add actors for all multisigs too
 			continue
 		}
 
 		if a.Type != genesis.TAccount {
-			return 0, nil, nil, xerrors.Errorf("unsupported account type: %s", a.Type)
+			return nil, xerrors.Errorf("unsupported account type: %s", a.Type) // TODO: Support msig (skip here)
 		}
 
 		var ainfo genesis.AccountMeta
 		if err := json.Unmarshal(a.Meta, &ainfo); err != nil {
-			return 0, nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
+			return nil, xerrors.Errorf("unmarshaling account meta: %w", err)
 		}
 
-		fmt.Printf("init set %s t0%d\n", ainfo.Owner, counter)
+		fmt.Printf("init set %s t0%d\n", ainfo.Owner, AccountStart+uint64(i))
 
-		value := cbg.CborInt(counter)
-		if err := amap.Put(adt.AddrKey(ainfo.Owner), &value); err != nil {
-			return 0, nil, nil, err
-		}
-		counter = counter + 1
-
-		var err error
-		keyToId[ainfo.Owner], err = address.NewIDAddress(uint64(value))
-		if err != nil {
-			return 0, nil, nil, err
+		if err := amap.Set(context.TODO(), string(ainfo.Owner.Bytes()), AccountStart+uint64(i)); err != nil {
+			return nil, err
 		}
 	}
 
-	if rootVerifier.Type == genesis.TAccount {
-		var ainfo genesis.AccountMeta
-		if err := json.Unmarshal(rootVerifier.Meta, &ainfo); err != nil {
-			return 0, nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
-		}
-		value := cbg.CborInt(80)
-		if err := amap.Put(adt.AddrKey(ainfo.Owner), &value); err != nil {
-			return 0, nil, nil, err
-		}
-	} else if rootVerifier.Type == genesis.TMultisig {
-		var ainfo genesis.MultisigMeta
-		if err := json.Unmarshal(rootVerifier.Meta, &ainfo); err != nil {
-			return 0, nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
-		}
-		for _, e := range ainfo.Signers {
-			if _, ok := keyToId[e]; ok {
-				continue
-			}
-			fmt.Printf("init set %s t0%d\n", e, counter)
-
-			value := cbg.CborInt(counter)
-			if err := amap.Put(adt.AddrKey(e), &value); err != nil {
-				return 0, nil, nil, err
-			}
-			counter = counter + 1
-			var err error
-			keyToId[e], err = address.NewIDAddress(uint64(value))
-			if err != nil {
-				return 0, nil, nil, err
-			}
-
-		}
+	if err := amap.Set(context.TODO(), string(RootVerifierAddr.Bytes()), 80); err != nil {
+		return nil, err
 	}
 
-	amapaddr, err := amap.Root()
+	if err := amap.Flush(context.TODO()); err != nil {
+		return nil, err
+	}
+	amapcid, err := cst.Put(context.TODO(), amap)
 	if err != nil {
-		return 0, nil, nil, err
+		return nil, err
 	}
-	ias.AddressMap = amapaddr
 
-	statecid, err := store.Put(store.Context(), &ias)
+	ias.AddressMap = amapcid
+
+	statecid, err := cst.Put(context.TODO(), &ias)
 	if err != nil {
-		return 0, nil, nil, err
+		return nil, err
 	}
 
 	act := &types.Actor{
@@ -139,5 +74,5 @@ func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesi
 		Head: statecid,
 	}
 
-	return counter, act, keyToId, nil
+	return act, nil
 }

chain/gen/genesis/t02_reward.go

@@ -2,22 +2,22 @@ package genesis
 
 import (
 	"context"
-
-	"github.com/filecoin-project/go-state-types/big"
-
-	"github.com/filecoin-project/specs-actors/actors/builtin"
-	"github.com/filecoin-project/specs-actors/actors/builtin/reward"
-	cbor "github.com/ipfs/go-ipld-cbor"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
 
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
+	"github.com/filecoin-project/specs-actors/actors/builtin"
+	"github.com/filecoin-project/specs-actors/actors/builtin/reward"
+	bstore "github.com/ipfs/go-ipfs-blockstore"
+	cbor "github.com/ipfs/go-ipld-cbor"
 )
 
-func SetupRewardActor(bs bstore.Blockstore, qaPower big.Int) (*types.Actor, error) {
+func SetupRewardActor(bs bstore.Blockstore) (*types.Actor, error) {
 	cst := cbor.NewCborStore(bs)
 
-	st := reward.ConstructState(qaPower)
+	z := big.Zero()
+	st := reward.ConstructState(z)
+	st.ThisEpochReward = types.FromFil(100)
 
 	hcid, err := cst.Put(context.TODO(), st)
 	if err != nil {

chain/gen/genesis/t03_cron.go

@@ -5,10 +5,10 @@ import (
 
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/cron"
+	bstore "github.com/ipfs/go-ipfs-blockstore"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 func SetupCronActor(bs bstore.Blockstore) (*types.Actor, error) {

chain/gen/genesis/t04_power.go

@@ -2,37 +2,41 @@ package genesis
 
 import (
 	"context"
-
 	"github.com/filecoin-project/specs-actors/actors/builtin"
-	"github.com/filecoin-project/specs-actors/actors/util/adt"
 
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
+	"github.com/ipfs/go-hamt-ipld"
+	bstore "github.com/ipfs/go-ipfs-blockstore"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 func SetupStoragePowerActor(bs bstore.Blockstore) (*types.Actor, error) {
-	store := adt.WrapStore(context.TODO(), cbor.NewCborStore(bs))
-	emptyMap, err := adt.MakeEmptyMap(store).Root()
+	ctx := context.TODO()
+	cst := cbor.NewCborStore(bs)
+	nd := hamt.NewNode(cst, hamt.UseTreeBitWidth(5))
+	emptyhamt, err := cst.Put(ctx, nd)
 	if err != nil {
 		return nil, err
 	}
 
-	multiMap, err := adt.AsMultimap(store, emptyMap)
-	if err != nil {
-		return nil, err
+	sms := &power.State{
+		TotalRawBytePower:       big.NewInt(0),
+		TotalBytesCommitted:     big.NewInt(0),
+		TotalQualityAdjPower:    big.NewInt(0),
+		TotalQABytesCommitted:   big.NewInt(0),
+		TotalPledgeCollateral:   big.NewInt(0),
+		MinerCount:              0,
+		MinerAboveMinPowerCount: 0,
+		CronEventQueue:          emptyhamt,
+		FirstCronEpoch:          0,
+		Claims:                  emptyhamt,
+		ProofValidationBatch:    nil,
 	}
 
-	emptyMultiMap, err := multiMap.Root()
-	if err != nil {
-		return nil, err
-	}
-
-	sms := power.ConstructState(emptyMap, emptyMultiMap)
-
-	stcid, err := store.Put(store.Context(), sms)
+	stcid, err := cst.Put(ctx, sms)
 	if err != nil {
 		return nil, err
 	}

chain/gen/genesis/t05_market.go

@@ -2,31 +2,32 @@ package genesis
 
 import (
 	"context"
+	"github.com/ipfs/go-hamt-ipld"
 
+	"github.com/filecoin-project/go-amt-ipld/v2"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/market"
-	"github.com/filecoin-project/specs-actors/actors/util/adt"
+	bstore "github.com/ipfs/go-ipfs-blockstore"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 func SetupStorageMarketActor(bs bstore.Blockstore) (*types.Actor, error) {
-	store := adt.WrapStore(context.TODO(), cbor.NewCborStore(bs))
+	cst := cbor.NewCborStore(bs)
 
-	a, err := adt.MakeEmptyArray(store).Root()
+	a, err := amt.NewAMT(cst).Flush(context.TODO())
 	if err != nil {
 		return nil, err
 	}
-	h, err := adt.MakeEmptyMap(store).Root()
+	h, err := cst.Put(context.TODO(), hamt.NewNode(cst, hamt.UseTreeBitWidth(5)))
 	if err != nil {
 		return nil, err
 	}
 
 	sms := market.ConstructState(a, h, h)
 
-	stcid, err := store.Put(store.Context(), sms)
+	stcid, err := cst.Put(context.TODO(), sms)
 	if err != nil {
 		return nil, err
 	}

chain/gen/genesis/t06_vreg.go

@@ -4,19 +4,27 @@ import (
 	"context"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/ipfs/go-hamt-ipld"
+	bstore "github.com/ipfs/go-ipfs-blockstore"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
-	"github.com/filecoin-project/specs-actors/actors/util/adt"
 
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
+var RootVerifierAddr address.Address
+
 var RootVerifierID address.Address
 
 func init() {
+	k, err := address.NewFromString("t3qfoulel6fy6gn3hjmbhpdpf6fs5aqjb5fkurhtwvgssizq4jey5nw4ptq5up6h7jk7frdvvobv52qzmgjinq")
+	if err != nil {
+		panic(err)
+	}
+
+	RootVerifierAddr = k
 
 	idk, err := address.NewFromString("t080")
 	if err != nil {
@@ -27,16 +35,16 @@ func init() {
 }
 
 func SetupVerifiedRegistryActor(bs bstore.Blockstore) (*types.Actor, error) {
-	store := adt.WrapStore(context.TODO(), cbor.NewCborStore(bs))
+	cst := cbor.NewCborStore(bs)
 
-	h, err := adt.MakeEmptyMap(store).Root()
+	h, err := cst.Put(context.TODO(), hamt.NewNode(cst, hamt.UseTreeBitWidth(5)))
 	if err != nil {
 		return nil, err
 	}
 
 	sms := verifreg.ConstructState(h, RootVerifierID)
 
-	stcid, err := store.Put(store.Context(), sms)
+	stcid, err := cst.Put(context.TODO(), sms)
 	if err != nil {
 		return nil, err
 	}

chain/gen/genesis/util.go

@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"golang.org/x/xerrors"
 
@@ -21,10 +21,14 @@ func mustEnc(i cbg.CBORMarshaler) []byte {
 	return enc
 }
 
+func doExec(ctx context.Context, vm *vm.VM, to, from address.Address, method abi.MethodNum, params []byte) ([]byte, error) {
+	return doExecValue(ctx, vm, to, from, types.NewInt(0), method, params)
+}
+
 func doExecValue(ctx context.Context, vm *vm.VM, to, from address.Address, value types.BigInt, method abi.MethodNum, params []byte) ([]byte, error) {
 	act, err := vm.StateTree().GetActor(from)
 	if err != nil {
-		return nil, xerrors.Errorf("doExec failed to get from actor (%s): %w", from, err)
+		return nil, xerrors.Errorf("doExec failed to get from actor: %w", err)
 	}
 
 	ret, err := vm.ApplyImplicitMessage(ctx, &types.Message{
@@ -33,6 +37,7 @@ func doExecValue(ctx context.Context, vm *vm.VM, to, from address.Address, value
 		Method:   method,
 		Params:   params,
 		GasLimit: 1_000_000_000_000_000,
+		GasPrice: types.NewInt(0),
 		Value:    value,
 		Nonce:    act.Nonce,
 	})

chain/gen/mining.go

@@ -3,8 +3,9 @@ package gen
 import (
 	"context"
 
-	"github.com/filecoin-project/go-state-types/crypto"
-	"github.com/filecoin-project/specs-actors/actors/util/adt"
+	bls "github.com/filecoin-project/filecoin-ffi"
+	amt "github.com/filecoin-project/go-amt-ipld/v2"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
 	cid "github.com/ipfs/go-cid"
 	cbor "github.com/ipfs/go-ipld-cbor"
 	cbg "github.com/whyrusleeping/cbor-gen"
@@ -16,7 +17,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/chain/wallet"
-	"github.com/filecoin-project/lotus/lib/sigs/bls"
 )
 
 func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Wallet, bt *api.BlockTemplate) (*types.FullBlock, error) {
@@ -78,17 +78,17 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Wal
 		}
 	}
 
-	store := sm.ChainStore().Store(ctx)
-	blsmsgroot, err := toArray(store, blsMsgCids)
+	bs := cbor.NewCborStore(sm.ChainStore().Blockstore())
+	blsmsgroot, err := amt.FromArray(ctx, bs, toIfArr(blsMsgCids))
 	if err != nil {
 		return nil, xerrors.Errorf("building bls amt: %w", err)
 	}
-	secpkmsgroot, err := toArray(store, secpkMsgCids)
+	secpkmsgroot, err := amt.FromArray(ctx, bs, toIfArr(secpkMsgCids))
 	if err != nil {
 		return nil, xerrors.Errorf("building secpk amt: %w", err)
 	}
 
-	mmcid, err := store.Put(store.Context(), &types.MsgMeta{
+	mmcid, err := bs.Put(ctx, &types.MsgMeta{
 		BlsMessages:   blsmsgroot,
 		SecpkMessages: secpkmsgroot,
 	})
@@ -109,12 +109,6 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Wal
 	}
 	next.ParentWeight = pweight
 
-	baseFee, err := sm.ChainStore().ComputeBaseFee(ctx, pts)
-	if err != nil {
-		return nil, xerrors.Errorf("computing base fee: %w", err)
-	}
-	next.ParentBaseFee = baseFee
-
 	cst := cbor.NewCborStore(sm.ChainStore().Blockstore())
 	tree, err := state.LoadStateTree(cst, st)
 	if err != nil {
@@ -148,45 +142,36 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Wal
 }
 
 func aggregateSignatures(sigs []crypto.Signature) (*crypto.Signature, error) {
-	sigsS := make([][]byte, len(sigs))
-	for i := 0; i < len(sigs); i++ {
-		sigsS[i] = sigs[i].Data
+	var blsSigs []bls.Signature
+	for _, s := range sigs {
+		var bsig bls.Signature
+		copy(bsig[:], s.Data)
+		blsSigs = append(blsSigs, bsig)
 	}
 
-	aggregator := new(bls.AggregateSignature).AggregateCompressed(sigsS)
-	if aggregator == nil {
+	aggSig := bls.Aggregate(blsSigs)
+	if aggSig == nil {
 		if len(sigs) > 0 {
 			return nil, xerrors.Errorf("bls.Aggregate returned nil with %d signatures", len(sigs))
 		}
 
-		// Note: for blst this condition should not happen - nil should not
-		// be returned
 		return &crypto.Signature{
 			Type: crypto.SigTypeBLS,
-			Data: new(bls.Signature).Compress(),
+			Data: new(bls.Signature)[:],
 		}, nil
 	}
-	aggSigAff := aggregator.ToAffine()
-	if aggSigAff == nil {
-		return &crypto.Signature{
-			Type: crypto.SigTypeBLS,
-			Data: new(bls.Signature).Compress(),
-		}, nil
-	}
-	aggSig := aggSigAff.Compress()
+
 	return &crypto.Signature{
 		Type: crypto.SigTypeBLS,
-		Data: aggSig,
+		Data: aggSig[:],
 	}, nil
 }
 
-func toArray(store adt.Store, cids []cid.Cid) (cid.Cid, error) {
-	arr := adt.MakeEmptyArray(store)
-	for i, c := range cids {
+func toIfArr(cids []cid.Cid) []cbg.CBORMarshaler {
+	out := make([]cbg.CBORMarshaler, 0, len(cids))
+	for _, c := range cids {
 		oc := cbg.CborCid(c)
-		if err := arr.Set(uint64(i), &oc); err != nil {
-			return cid.Undef, err
-		}
+		out = append(out, &oc)
 	}
-	return arr.Root()
+	return out
 }

chain/gen/slashfilter/slashfilter.go

@@ -1,112 +0,0 @@
-package slashfilter
-
-import (
-	"fmt"
-
-	"golang.org/x/xerrors"
-
-	"github.com/ipfs/go-cid"
-	ds "github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-datastore/namespace"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-type SlashFilter struct {
-	byEpoch   ds.Datastore // double-fork mining faults, parent-grinding fault
-	byParents ds.Datastore // time-offset mining faults
-}
-
-func New(dstore ds.Batching) *SlashFilter {
-	return &SlashFilter{
-		byEpoch:   namespace.Wrap(dstore, ds.NewKey("/slashfilter/epoch")),
-		byParents: namespace.Wrap(dstore, ds.NewKey("/slashfilter/parents")),
-	}
-}
-
-func (f *SlashFilter) MinedBlock(bh *types.BlockHeader, parentEpoch abi.ChainEpoch) error {
-	epochKey := ds.NewKey(fmt.Sprintf("/%s/%d", bh.Miner, bh.Height))
-	{
-		// double-fork mining (2 blocks at one epoch)
-		if err := checkFault(f.byEpoch, epochKey, bh, "double-fork mining faults"); err != nil {
-			return err
-		}
-	}
-
-	parentsKey := ds.NewKey(fmt.Sprintf("/%s/%x", bh.Miner, types.NewTipSetKey(bh.Parents...).Bytes()))
-	{
-		// time-offset mining faults (2 blocks with the same parents)
-		if err := checkFault(f.byParents, parentsKey, bh, "time-offset mining faults"); err != nil {
-			return err
-		}
-	}
-
-	{
-		// parent-grinding fault (didn't mine on top of our own block)
-
-		// First check if we have mined a block on the parent epoch
-		parentEpochKey := ds.NewKey(fmt.Sprintf("/%s/%d", bh.Miner, parentEpoch))
-		have, err := f.byEpoch.Has(parentEpochKey)
-		if err != nil {
-			return err
-		}
-
-		if have {
-			// If we had, make sure it's in our parent tipset
-			cidb, err := f.byEpoch.Get(parentEpochKey)
-			if err != nil {
-				return xerrors.Errorf("getting other block cid: %w", err)
-			}
-
-			_, parent, err := cid.CidFromBytes(cidb)
-			if err != nil {
-				return err
-			}
-
-			var found bool
-			for _, c := range bh.Parents {
-				if c.Equals(parent) {
-					found = true
-				}
-			}
-
-			if !found {
-				return xerrors.Errorf("produced block would trigger 'parent-grinding fault' consensus fault; miner: %s; bh: %s, expected parent: %s", bh.Miner, bh.Cid(), parent)
-			}
-		}
-	}
-
-	if err := f.byParents.Put(parentsKey, bh.Cid().Bytes()); err != nil {
-		return xerrors.Errorf("putting byEpoch entry: %w", err)
-	}
-
-	if err := f.byEpoch.Put(epochKey, bh.Cid().Bytes()); err != nil {
-		return xerrors.Errorf("putting byEpoch entry: %w", err)
-	}
-
-	return nil
-}
-
-func checkFault(t ds.Datastore, key ds.Key, bh *types.BlockHeader, faultType string) error {
-	fault, err := t.Has(key)
-	if err != nil {
-		return err
-	}
-
-	if fault {
-		cidb, err := t.Get(key)
-		if err != nil {
-			return xerrors.Errorf("getting other block cid: %w", err)
-		}
-
-		_, other, err := cid.CidFromBytes(cidb)
-		if err != nil {
-			return err
-		}
-
-		return xerrors.Errorf("produced block would trigger '%s' consensus fault; miner: %s; bh: %s, other: %s", faultType, bh.Miner, bh.Cid(), other)
-	}
-
-	return nil
-}

chain/market/fundmgr.go

@@ -4,159 +4,76 @@ import (
 	"context"
 	"sync"
 
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
-	"go.uber.org/fx"
-
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/events"
-	"github.com/filecoin-project/lotus/chain/events/state"
+	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/node/impl/full"
 )
 
 var log = logging.Logger("market_adapter")
 
-// API is the dependencies need to run a fund manager
-type API struct {
-	fx.In
-
-	full.ChainAPI
-	full.StateAPI
-	full.MpoolAPI
-}
-
-// FundMgr monitors available balances and adds funds when EnsureAvailable is called
 type FundMgr struct {
-	api fundMgrAPI
+	sm    *stmgr.StateManager
+	mpool full.MpoolAPI
 
-	lk        sync.RWMutex
+	lk        sync.Mutex
 	available map[address.Address]types.BigInt
 }
 
-// StartFundManager creates a new fund manager and sets up event hooks to manage state changes
-func StartFundManager(lc fx.Lifecycle, api API) *FundMgr {
-	fm := newFundMgr(&api)
-	lc.Append(fx.Hook{
-		OnStart: func(ctx context.Context) error {
-			ev := events.NewEvents(ctx, &api)
-			preds := state.NewStatePredicates(&api)
-			dealDiffFn := preds.OnStorageMarketActorChanged(preds.OnBalanceChanged(preds.AvailableBalanceChangedForAddresses(fm.getAddresses)))
-			match := func(oldTs, newTs *types.TipSet) (bool, events.StateChange, error) {
-				return dealDiffFn(ctx, oldTs.Key(), newTs.Key())
-			}
-			return ev.StateChanged(fm.checkFunc, fm.stateChanged, fm.revert, 0, events.NoTimeout, match)
-		},
-	})
-	return fm
-}
-
-type fundMgrAPI interface {
-	StateMarketBalance(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error)
-	MpoolPushMessage(context.Context, *types.Message, *api.MessageSendSpec) (*types.SignedMessage, error)
-	StateLookupID(context.Context, address.Address, types.TipSetKey) (address.Address, error)
-}
-
-func newFundMgr(api fundMgrAPI) *FundMgr {
+func NewFundMgr(sm *stmgr.StateManager, mpool full.MpoolAPI) *FundMgr {
 	return &FundMgr{
-		api:       api,
+		sm:    sm,
+		mpool: mpool,
+
 		available: map[address.Address]types.BigInt{},
 	}
 }
 
-// checkFunc tells the events api to simply proceed (we always want to watch)
-func (fm *FundMgr) checkFunc(ts *types.TipSet) (done bool, more bool, err error) {
-	return false, true, nil
-}
-
-// revert handles reverts to balances
-func (fm *FundMgr) revert(ctx context.Context, ts *types.TipSet) error {
-	// TODO: Is it ok to just ignore this?
-	log.Warn("balance change reverted; TODO: actually handle this!")
-	return nil
-}
-
-// stateChanged handles balance changes monitored on the chain from one tipset to the next
-func (fm *FundMgr) stateChanged(ts *types.TipSet, ts2 *types.TipSet, states events.StateChange, h abi.ChainEpoch) (more bool, err error) {
-	changedBalances, ok := states.(state.ChangedBalances)
-	if !ok {
-		panic("Expected state.ChangedBalances")
-	}
-	// overwrite our in memory cache with new values from chain (chain is canonical)
+func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Address, amt types.BigInt) (cid.Cid, error) {
 	fm.lk.Lock()
-	for addr, balanceChange := range changedBalances {
-		if fm.available[addr].Int != nil {
-			log.Infof("State balance change recorded, prev: %s, new: %s", fm.available[addr].String(), balanceChange.To.String())
+	avail, ok := fm.available[addr]
+	if !ok {
+		bal, err := fm.sm.MarketBalance(ctx, addr, nil)
+		if err != nil {
+			fm.lk.Unlock()
+			return cid.Undef, err
 		}
 
-		fm.available[addr] = balanceChange.To
-	}
-	fm.lk.Unlock()
-	return true, nil
-}
-
-func (fm *FundMgr) getAddresses() []address.Address {
-	fm.lk.RLock()
-	defer fm.lk.RUnlock()
-	addrs := make([]address.Address, 0, len(fm.available))
-	for addr := range fm.available {
-		addrs = append(addrs, addr)
-	}
-	return addrs
-}
-
-// EnsureAvailable looks at the available balance in escrow for a given
-// address, and if less than the passed in amount, adds the difference
-func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Address, amt types.BigInt) (cid.Cid, error) {
-	idAddr, err := fm.api.StateLookupID(ctx, addr, types.EmptyTSK)
-	if err != nil {
-		return cid.Undef, err
-	}
-	fm.lk.Lock()
-	bal, err := fm.api.StateMarketBalance(ctx, addr, types.EmptyTSK)
-	if err != nil {
-		fm.lk.Unlock()
-		return cid.Undef, err
+		avail = types.BigSub(bal.Escrow, bal.Locked)
 	}
 
-	stateAvail := types.BigSub(bal.Escrow, bal.Locked)
-
-	avail, ok := fm.available[idAddr]
-	if !ok {
-		avail = stateAvail
+	toAdd := types.NewInt(0)
+	avail = types.BigSub(avail, amt)
+	if avail.LessThan(types.NewInt(0)) {
+		// TODO: some rules around adding more to avoid doing stuff on-chain
+		//  all the time
+		toAdd = types.BigSub(toAdd, avail)
+		avail = types.NewInt(0)
 	}
+	fm.available[addr] = avail
 
-	toAdd := types.BigSub(amt, avail)
-	if toAdd.LessThan(types.NewInt(0)) {
-		toAdd = types.NewInt(0)
-	}
-	fm.available[idAddr] = big.Add(avail, toAdd)
 	fm.lk.Unlock()
 
-	log.Infof("Funds operation w/ Expected Balance: %s, In State: %s, Requested: %s, Adding: %s", avail.String(), stateAvail.String(), amt.String(), toAdd.String())
-
-	if toAdd.LessThanEqual(big.Zero()) {
-		return cid.Undef, nil
-	}
-
+	var err error
 	params, err := actors.SerializeParams(&addr)
 	if err != nil {
 		return cid.Undef, err
 	}
 
-	smsg, err := fm.api.MpoolPushMessage(ctx, &types.Message{
-		To:     builtin.StorageMarketActorAddr,
-		From:   wallet,
-		Value:  toAdd,
-		Method: builtin.MethodsMarket.AddBalance,
-		Params: params,
-	}, nil)
+	smsg, err := fm.mpool.MpoolPushMessage(ctx, &types.Message{
+		To:       builtin.StorageMarketActorAddr,
+		From:     wallet,
+		Value:    toAdd,
+		GasPrice: types.NewInt(0),
+		GasLimit: 100_000_000,
+		Method:   builtin.MethodsMarket.AddBalance,
+		Params:   params,
+	})
 	if err != nil {
 		return cid.Undef, err
 	}

chain/market/fundmgr_test.go

@@ -1,188 +0,0 @@
-package market
-
-import (
-	"context"
-	"errors"
-	"math/rand"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/crypto"
-	"github.com/filecoin-project/specs-actors/actors/builtin"
-	tutils "github.com/filecoin-project/specs-actors/support/testing"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-type fakeAPI struct {
-	returnedBalance    api.MarketBalance
-	returnedBalanceErr error
-	signature          crypto.Signature
-	receivedMessage    *types.Message
-	pushMessageErr     error
-	lookupIDErr        error
-}
-
-func (fapi *fakeAPI) StateLookupID(_ context.Context, addr address.Address, _ types.TipSetKey) (address.Address, error) {
-	return addr, fapi.lookupIDErr
-}
-func (fapi *fakeAPI) StateMarketBalance(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error) {
-	return fapi.returnedBalance, fapi.returnedBalanceErr
-}
-
-func (fapi *fakeAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) {
-	fapi.receivedMessage = msg
-	return &types.SignedMessage{
-		Message:   *msg,
-		Signature: fapi.signature,
-	}, fapi.pushMessageErr
-}
-
-func addFundsMsg(toAdd abi.TokenAmount, addr address.Address, wallet address.Address) *types.Message {
-	params, _ := actors.SerializeParams(&addr)
-	return &types.Message{
-		To:     builtin.StorageMarketActorAddr,
-		From:   wallet,
-		Value:  toAdd,
-		Method: builtin.MethodsMarket.AddBalance,
-		Params: params,
-	}
-}
-
-type expectedResult struct {
-	addAmt    abi.TokenAmount
-	shouldAdd bool
-	err       error
-}
-
-func TestAddFunds(t *testing.T) {
-	ctx := context.Background()
-	testCases := map[string]struct {
-		returnedBalanceErr error
-		returnedBalance    api.MarketBalance
-		addAmounts         []abi.TokenAmount
-		pushMessageErr     error
-		expectedResults    []expectedResult
-		lookupIDErr        error
-	}{
-		"succeeds, trivial case": {
-			returnedBalance: api.MarketBalance{Escrow: abi.NewTokenAmount(0), Locked: abi.NewTokenAmount(0)},
-			addAmounts:      []abi.TokenAmount{abi.NewTokenAmount(100)},
-			expectedResults: []expectedResult{
-				{
-					addAmt:    abi.NewTokenAmount(100),
-					shouldAdd: true,
-					err:       nil,
-				},
-			},
-		},
-		"succeeds, money already present": {
-			returnedBalance: api.MarketBalance{Escrow: abi.NewTokenAmount(150), Locked: abi.NewTokenAmount(50)},
-			addAmounts:      []abi.TokenAmount{abi.NewTokenAmount(100)},
-			expectedResults: []expectedResult{
-				{
-					shouldAdd: false,
-					err:       nil,
-				},
-			},
-		},
-		"succeeds, multiple adds": {
-			returnedBalance: api.MarketBalance{Escrow: abi.NewTokenAmount(150), Locked: abi.NewTokenAmount(50)},
-			addAmounts:      []abi.TokenAmount{abi.NewTokenAmount(100), abi.NewTokenAmount(200), abi.NewTokenAmount(250), abi.NewTokenAmount(250)},
-			expectedResults: []expectedResult{
-				{
-					shouldAdd: false,
-					err:       nil,
-				},
-				{
-					addAmt:    abi.NewTokenAmount(100),
-					shouldAdd: true,
-					err:       nil,
-				},
-				{
-					addAmt:    abi.NewTokenAmount(50),
-					shouldAdd: true,
-					err:       nil,
-				},
-				{
-					shouldAdd: false,
-					err:       nil,
-				},
-			},
-		},
-		"error on market balance": {
-			returnedBalanceErr: errors.New("something went wrong"),
-			addAmounts:         []abi.TokenAmount{abi.NewTokenAmount(100)},
-			expectedResults: []expectedResult{
-				{
-					err: errors.New("something went wrong"),
-				},
-			},
-		},
-		"error on push message": {
-			returnedBalance: api.MarketBalance{Escrow: abi.NewTokenAmount(0), Locked: abi.NewTokenAmount(0)},
-			pushMessageErr:  errors.New("something went wrong"),
-			addAmounts:      []abi.TokenAmount{abi.NewTokenAmount(100)},
-			expectedResults: []expectedResult{
-				{
-					err: errors.New("something went wrong"),
-				},
-			},
-		},
-		"error looking up address": {
-			lookupIDErr: errors.New("something went wrong"),
-			addAmounts:  []abi.TokenAmount{abi.NewTokenAmount(100)},
-			expectedResults: []expectedResult{
-				{
-					err: errors.New("something went wrong"),
-				},
-			},
-		},
-	}
-
-	for testCase, data := range testCases {
-		//nolint:scopelint
-		t.Run(testCase, func(t *testing.T) {
-			ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
-			defer cancel()
-			sig := make([]byte, 100)
-			_, err := rand.Read(sig)
-			require.NoError(t, err)
-			fapi := &fakeAPI{
-				returnedBalance:    data.returnedBalance,
-				returnedBalanceErr: data.returnedBalanceErr,
-				signature: crypto.Signature{
-					Type: crypto.SigTypeUnknown,
-					Data: sig,
-				},
-				pushMessageErr: data.pushMessageErr,
-				lookupIDErr:    data.lookupIDErr,
-			}
-			fundMgr := newFundMgr(fapi)
-			addr := tutils.NewIDAddr(t, uint64(rand.Uint32()))
-			wallet := tutils.NewIDAddr(t, uint64(rand.Uint32()))
-			for i, amount := range data.addAmounts {
-				fapi.receivedMessage = nil
-				_, err := fundMgr.EnsureAvailable(ctx, addr, wallet, amount)
-				expected := data.expectedResults[i]
-				if expected.err == nil {
-					require.NoError(t, err)
-					if expected.shouldAdd {
-						expectedMessage := addFundsMsg(expected.addAmt, addr, wallet)
-						require.Equal(t, expectedMessage, fapi.receivedMessage)
-					} else {
-						require.Nil(t, fapi.receivedMessage)
-					}
-				} else {
-					require.EqualError(t, err, expected.err.Error())
-				}
-			}
-		})
-	}
-}

chain/messagepool/block_proba.go

@@ -1,102 +0,0 @@
-package messagepool
-
-import (
-	"math"
-	"sync"
-)
-
-var noWinnersProbCache []float64
-var noWinnersProbOnce sync.Once
-
-func noWinnersProb() []float64 {
-	noWinnersProbOnce.Do(func() {
-		poissPdf := func(x float64) float64 {
-			const Mu = 5
-			lg, _ := math.Lgamma(x + 1)
-			result := math.Exp((math.Log(Mu) * x) - lg - Mu)
-			return result
-		}
-
-		out := make([]float64, 0, MaxBlocks)
-		for i := 0; i < MaxBlocks; i++ {
-			out = append(out, poissPdf(float64(i)))
-		}
-		noWinnersProbCache = out
-	})
-	return noWinnersProbCache
-}
-
-var noWinnersProbAssumingCache []float64
-var noWinnersProbAssumingOnce sync.Once
-
-func noWinnersProbAssumingMoreThanOne() []float64 {
-	noWinnersProbAssumingOnce.Do(func() {
-		cond := math.Log(-1 + math.Exp(5))
-		poissPdf := func(x float64) float64 {
-			const Mu = 5
-			lg, _ := math.Lgamma(x + 1)
-			result := math.Exp((math.Log(Mu) * x) - lg - cond)
-			return result
-		}
-
-		out := make([]float64, 0, MaxBlocks)
-		for i := 0; i < MaxBlocks; i++ {
-			out = append(out, poissPdf(float64(i+1)))
-		}
-		noWinnersProbAssumingCache = out
-	})
-	return noWinnersProbAssumingCache
-}
-
-func binomialCoefficient(n, k float64) float64 {
-	if k > n {
-		return math.NaN()
-	}
-	r := 1.0
-	for d := 1.0; d <= k; d++ {
-		r *= n
-		r /= d
-		n--
-	}
-	return r
-}
-
-func (mp *MessagePool) blockProbabilities(tq float64) []float64 {
-	noWinners := noWinnersProbAssumingMoreThanOne()
-
-	p := 1 - tq
-	binoPdf := func(x, trials float64) float64 {
-		// based on https://github.com/atgjack/prob
-		if x > trials {
-			return 0
-		}
-		if p == 0 {
-			if x == 0 {
-				return 1.0
-			}
-			return 0.0
-		}
-		if p == 1 {
-			if x == trials {
-				return 1.0
-			}
-			return 0.0
-		}
-		coef := binomialCoefficient(trials, x)
-		pow := math.Pow(p, x) * math.Pow(1-p, trials-x)
-		if math.IsInf(coef, 0) {
-			return 0
-		}
-		return coef * pow
-	}
-
-	out := make([]float64, 0, MaxBlocks)
-	for place := 0; place < MaxBlocks; place++ {
-		var pPlace float64
-		for otherWinners, pCase := range noWinners {
-			pPlace += pCase * binoPdf(float64(place), float64(otherWinners))
-		}
-		out = append(out, pPlace)
-	}
-	return out
-}

chain/messagepool/block_proba_test.go

@@ -1,43 +0,0 @@
-package messagepool
-
-import (
-	"math"
-	"math/rand"
-	"testing"
-	"time"
-)
-
-func TestBlockProbability(t *testing.T) {
-	mp := &MessagePool{}
-	bp := mp.blockProbabilities(1 - 0.15)
-	t.Logf("%+v\n", bp)
-	for i := 0; i < len(bp)-1; i++ {
-		if bp[i] < bp[i+1] {
-			t.Fatalf("expected decreasing block probabilities for this quality: %d %f %f",
-				i, bp[i], bp[i+1])
-		}
-	}
-}
-
-func TestWinnerProba(t *testing.T) {
-	rand.Seed(time.Now().UnixNano())
-	const N = 1000000
-	winnerProba := noWinnersProb()
-	sum := 0
-	for i := 0; i < N; i++ {
-		minersRand := rand.Float64()
-		j := 0
-		for ; j < MaxBlocks; j++ {
-			minersRand -= winnerProba[j]
-			if minersRand < 0 {
-				break
-			}
-		}
-		sum += j
-	}
-
-	if avg := float64(sum) / N; math.Abs(avg-5) > 0.01 {
-		t.Fatalf("avg too far off: %f", avg)
-	}
-
-}

chain/messagepool/config.go

@@ -1,92 +0,0 @@
-package messagepool
-
-import (
-	"encoding/json"
-	"fmt"
-	"time"
-
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-	"github.com/ipfs/go-datastore"
-)
-
-var (
-	ReplaceByFeeRatioDefault  = 1.25
-	MemPoolSizeLimitHiDefault = 30000
-	MemPoolSizeLimitLoDefault = 20000
-	PruneCooldownDefault      = time.Minute
-	GasLimitOverestimation    = 1.25
-
-	ConfigKey = datastore.NewKey("/mpool/config")
-)
-
-func loadConfig(ds dtypes.MetadataDS) (*types.MpoolConfig, error) {
-	haveCfg, err := ds.Has(ConfigKey)
-	if err != nil {
-		return nil, err
-	}
-
-	if !haveCfg {
-		return DefaultConfig(), nil
-	}
-
-	cfgBytes, err := ds.Get(ConfigKey)
-	if err != nil {
-		return nil, err
-	}
-	cfg := new(types.MpoolConfig)
-	err = json.Unmarshal(cfgBytes, cfg)
-	return cfg, err
-}
-
-func saveConfig(cfg *types.MpoolConfig, ds dtypes.MetadataDS) error {
-	cfgBytes, err := json.Marshal(cfg)
-	if err != nil {
-		return err
-	}
-	return ds.Put(ConfigKey, cfgBytes)
-}
-
-func (mp *MessagePool) GetConfig() *types.MpoolConfig {
-	mp.cfgLk.Lock()
-	defer mp.cfgLk.Unlock()
-	return mp.cfg.Clone()
-}
-
-func validateConfg(cfg *types.MpoolConfig) error {
-	if cfg.ReplaceByFeeRatio < ReplaceByFeeRatioDefault {
-		return fmt.Errorf("'ReplaceByFeeRatio' is less than required %f < %f",
-			cfg.ReplaceByFeeRatio, ReplaceByFeeRatioDefault)
-	}
-	if cfg.GasLimitOverestimation < 1 {
-		return fmt.Errorf("'GasLimitOverestimation' cannot be less than 1")
-	}
-	return nil
-}
-
-func (mp *MessagePool) SetConfig(cfg *types.MpoolConfig) error {
-	if err := validateConfg(cfg); err != nil {
-		return err
-	}
-	cfg = cfg.Clone()
-
-	mp.cfgLk.Lock()
-	mp.cfg = cfg
-	err := saveConfig(cfg, mp.ds)
-	if err != nil {
-		log.Warnf("error persisting mpool config: %s", err)
-	}
-	mp.cfgLk.Unlock()
-
-	return nil
-}
-
-func DefaultConfig() *types.MpoolConfig {
-	return &types.MpoolConfig{
-		SizeLimitHigh:          MemPoolSizeLimitHiDefault,
-		SizeLimitLow:           MemPoolSizeLimitLoDefault,
-		ReplaceByFeeRatio:      ReplaceByFeeRatioDefault,
-		PruneCooldown:          PruneCooldownDefault,
-		GasLimitOverestimation: GasLimitOverestimation,
-	}
-}

chain/messagepool/gasguess/guessgas.go

@@ -1,79 +0,0 @@
-package gasguess
-
-import (
-	"context"
-
-	"github.com/ipfs/go-cid"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/lotus/chain/types"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/specs-actors/actors/builtin"
-)
-
-type ActorLookup func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error)
-
-const failedGasGuessRatio = 0.5
-const failedGasGuessMax = 25_000_000
-
-const MinGas = 1298450
-const MaxGas = 1600271356
-
-type CostKey struct {
-	Code cid.Cid
-	M    abi.MethodNum
-}
-
-var Costs = map[CostKey]int64{
-	{builtin.InitActorCodeID, 2}:          8916753,
-	{builtin.StorageMarketActorCodeID, 2}: 6955002,
-	{builtin.StorageMarketActorCodeID, 4}: 245436108,
-	{builtin.StorageMinerActorCodeID, 4}:  2315133,
-	{builtin.StorageMinerActorCodeID, 5}:  1600271356,
-	{builtin.StorageMinerActorCodeID, 6}:  22864493,
-	{builtin.StorageMinerActorCodeID, 7}:  142002419,
-	{builtin.StorageMinerActorCodeID, 10}: 23008274,
-	{builtin.StorageMinerActorCodeID, 11}: 19303178,
-	{builtin.StorageMinerActorCodeID, 14}: 566356835,
-	{builtin.StorageMinerActorCodeID, 16}: 5325185,
-	{builtin.StorageMinerActorCodeID, 18}: 2328637,
-	{builtin.StoragePowerActorCodeID, 2}:  23600956,
-}
-
-func failedGuess(msg *types.SignedMessage) int64 {
-	guess := int64(float64(msg.Message.GasLimit) * failedGasGuessRatio)
-	if guess > failedGasGuessMax {
-		guess = failedGasGuessMax
-	}
-	return guess
-}
-
-func GuessGasUsed(ctx context.Context, tsk types.TipSetKey, msg *types.SignedMessage, al ActorLookup) (int64, error) {
-	if msg.Message.Method == builtin.MethodSend {
-		switch msg.Message.From.Protocol() {
-		case address.BLS:
-			return 1298450, nil
-		case address.SECP256K1:
-			return 1385999, nil
-		default:
-			// who knows?
-			return 1298450, nil
-		}
-	}
-
-	to, err := al(ctx, msg.Message.To, tsk)
-	if err != nil {
-		return failedGuess(msg), xerrors.Errorf("could not lookup actor: %w", err)
-	}
-
-	guess, ok := Costs[CostKey{to.Code, msg.Message.Method}]
-	if !ok {
-		return failedGuess(msg), xerrors.Errorf("unknown code-method combo")
-	}
-	if guess > msg.Message.GasLimit {
-		guess = msg.Message.GasLimit
-	}
-	return guess, nil
-}

chain/messagepool/messagepool_test.go

@@ -3,57 +3,33 @@ package messagepool
 import (
 	"context"
 	"fmt"
-	"sort"
 	"testing"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/crypto"
-	"github.com/filecoin-project/lotus/chain/messagepool/gasguess"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/types/mock"
 	"github.com/filecoin-project/lotus/chain/wallet"
 	_ "github.com/filecoin-project/lotus/lib/sigs/bls"
 	_ "github.com/filecoin-project/lotus/lib/sigs/secp"
-	"github.com/filecoin-project/specs-actors/actors/builtin"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
-	logging "github.com/ipfs/go-log/v2"
 )
 
-func init() {
-	_ = logging.SetLogLevel("*", "INFO")
-}
-
 type testMpoolAPI struct {
 	cb func(rev, app []*types.TipSet) error
 
 	bmsgs      map[cid.Cid][]*types.SignedMessage
 	statenonce map[address.Address]uint64
-	balance    map[address.Address]types.BigInt
 
 	tipsets []*types.TipSet
-
-	published int
-
-	baseFee types.BigInt
 }
 
 func newTestMpoolAPI() *testMpoolAPI {
-	tma := &testMpoolAPI{
+	return &testMpoolAPI{
 		bmsgs:      make(map[cid.Cid][]*types.SignedMessage),
 		statenonce: make(map[address.Address]uint64),
-		balance:    make(map[address.Address]types.BigInt),
-		baseFee:    types.NewInt(100),
 	}
-	genesis := mock.MkBlock(nil, 1, 1)
-	tma.tipsets = append(tma.tipsets, mock.TipSet(genesis))
-	return tma
-}
-
-func (tma *testMpoolAPI) nextBlock() *types.BlockHeader {
-	newBlk := mock.MkBlock(tma.tipsets[len(tma.tipsets)-1], 1, 1)
-	tma.tipsets = append(tma.tipsets, mock.TipSet(newBlk))
-	return newBlk
 }
 
 func (tma *testMpoolAPI) applyBlock(t *testing.T, b *types.BlockHeader) {
@@ -74,21 +50,14 @@ func (tma *testMpoolAPI) setStateNonce(addr address.Address, v uint64) {
 	tma.statenonce[addr] = v
 }
 
-func (tma *testMpoolAPI) setBalance(addr address.Address, v uint64) {
-	tma.balance[addr] = types.FromFil(v)
-}
-
-func (tma *testMpoolAPI) setBalanceRaw(addr address.Address, v types.BigInt) {
-	tma.balance[addr] = v
-}
-
 func (tma *testMpoolAPI) setBlockMessages(h *types.BlockHeader, msgs ...*types.SignedMessage) {
 	tma.bmsgs[h.Cid()] = msgs
+	tma.tipsets = append(tma.tipsets, mock.TipSet(h))
 }
 
 func (tma *testMpoolAPI) SubscribeHeadChanges(cb func(rev, app []*types.TipSet) error) *types.TipSet {
 	tma.cb = cb
-	return tma.tipsets[0]
+	return nil
 }
 
 func (tma *testMpoolAPI) PutMessage(m types.ChainMsg) (cid.Cid, error) {
@@ -96,48 +65,13 @@ func (tma *testMpoolAPI) PutMessage(m types.ChainMsg) (cid.Cid, error) {
 }
 
 func (tma *testMpoolAPI) PubSubPublish(string, []byte) error {
-	tma.published++
 	return nil
 }
 
-func (tma *testMpoolAPI) GetActorAfter(addr address.Address, ts *types.TipSet) (*types.Actor, error) {
-	// regression check for load bug
-	if ts == nil {
-		panic("GetActorAfter called with nil tipset")
-	}
-
-	balance, ok := tma.balance[addr]
-	if !ok {
-		balance = types.NewInt(1000e6)
-		tma.balance[addr] = balance
-	}
-
-	msgs := make([]*types.SignedMessage, 0)
-	for _, b := range ts.Blocks() {
-		for _, m := range tma.bmsgs[b.Cid()] {
-			if m.Message.From == addr {
-				msgs = append(msgs, m)
-			}
-		}
-	}
-
-	sort.Slice(msgs, func(i, j int) bool {
-		return msgs[i].Message.Nonce < msgs[j].Message.Nonce
-	})
-
-	nonce := tma.statenonce[addr]
-
-	for _, m := range msgs {
-		if m.Message.Nonce != nonce {
-			break
-		}
-		nonce++
-	}
-
+func (tma *testMpoolAPI) StateGetActor(addr address.Address, ts *types.TipSet) (*types.Actor, error) {
 	return &types.Actor{
-		Code:    builtin.StorageMarketActorCodeID,
-		Nonce:   nonce,
-		Balance: balance,
+		Nonce:   tma.statenonce[addr],
+		Balance: types.NewInt(90000000),
 	}, nil
 }
 
@@ -184,10 +118,6 @@ func (tma *testMpoolAPI) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error)
 	return nil, fmt.Errorf("tipset not found")
 }
 
-func (tma *testMpoolAPI) ChainComputeBaseFee(ctx context.Context, ts *types.TipSet) (types.BigInt, error) {
-	return tma.baseFee, nil
-}
-
 func assertNonce(t *testing.T, mp *MessagePool, addr address.Address, val uint64) {
 	t.Helper()
 	n, err := mp.GetNonce(addr)
@@ -222,7 +152,7 @@ func TestMessagePool(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	a := tma.nextBlock()
+	a := mock.MkBlock(nil, 1, 1)
 
 	sender, err := w.GenerateKey(crypto.SigTypeBLS)
 	if err != nil {
@@ -248,56 +178,7 @@ func TestMessagePool(t *testing.T) {
 	assertNonce(t, mp, sender, 2)
 }
 
-func TestMessagePoolMessagesInEachBlock(t *testing.T) {
-	tma := newTestMpoolAPI()
-
-	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	ds := datastore.NewMapDatastore()
-
-	mp, err := New(tma, ds, "mptest")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	a := tma.nextBlock()
-
-	sender, err := w.GenerateKey(crypto.SigTypeBLS)
-	if err != nil {
-		t.Fatal(err)
-	}
-	target := mock.Address(1001)
-
-	var msgs []*types.SignedMessage
-	for i := 0; i < 5; i++ {
-		m := mock.MkMessage(sender, target, uint64(i), w)
-		msgs = append(msgs, m)
-		mustAdd(t, mp, m)
-	}
-
-	tma.setStateNonce(sender, 0)
-
-	tma.setBlockMessages(a, msgs[0], msgs[1])
-	tma.applyBlock(t, a)
-	tsa := mock.TipSet(a)
-
-	_, _ = mp.Pending()
-
-	selm, _ := mp.SelectMessages(tsa, 1)
-	if len(selm) == 0 {
-		t.Fatal("should have returned the rest of the messages")
-	}
-}
-
 func TestRevertMessages(t *testing.T) {
-	futureDebug = true
-	defer func() {
-		futureDebug = false
-	}()
-
 	tma := newTestMpoolAPI()
 
 	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
@@ -312,8 +193,8 @@ func TestRevertMessages(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	a := tma.nextBlock()
-	b := tma.nextBlock()
+	a := mock.MkBlock(nil, 1, 1)
+	b := mock.MkBlock(mock.TipSet(a), 1, 1)
 
 	sender, err := w.GenerateKey(crypto.SigTypeBLS)
 	if err != nil {
@@ -347,319 +228,8 @@ func TestRevertMessages(t *testing.T) {
 	assertNonce(t, mp, sender, 4)
 
 	p, _ := mp.Pending()
-	fmt.Printf("%+v\n", p)
 	if len(p) != 3 {
 		t.Fatal("expected three messages in mempool")
 	}
 
 }
-
-func TestPruningSimple(t *testing.T) {
-	oldMaxNonceGap := MaxNonceGap
-	MaxNonceGap = 1000
-	defer func() {
-		MaxNonceGap = oldMaxNonceGap
-	}()
-
-	tma := newTestMpoolAPI()
-
-	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	ds := datastore.NewMapDatastore()
-
-	mp, err := New(tma, ds, "mptest")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	a := tma.nextBlock()
-	tma.applyBlock(t, a)
-
-	sender, err := w.GenerateKey(crypto.SigTypeBLS)
-	if err != nil {
-		t.Fatal(err)
-	}
-	tma.setBalance(sender, 1) // in FIL
-	target := mock.Address(1001)
-
-	for i := 0; i < 5; i++ {
-		smsg := mock.MkMessage(sender, target, uint64(i), w)
-		if err := mp.Add(smsg); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	for i := 10; i < 50; i++ {
-		smsg := mock.MkMessage(sender, target, uint64(i), w)
-		if err := mp.Add(smsg); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	mp.cfg.SizeLimitHigh = 40
-	mp.cfg.SizeLimitLow = 10
-
-	mp.Prune()
-
-	msgs, _ := mp.Pending()
-	if len(msgs) != 5 {
-		t.Fatal("expected only 5 messages in pool, got: ", len(msgs))
-	}
-}
-
-func TestLoadLocal(t *testing.T) {
-	tma := newTestMpoolAPI()
-	ds := datastore.NewMapDatastore()
-
-	mp, err := New(tma, ds, "mptest")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// the actors
-	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	tma.setBalance(a1, 1) // in FIL
-	tma.setBalance(a2, 1) // in FIL
-	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
-	msgs := make(map[cid.Cid]struct{})
-	for i := 0; i < 10; i++ {
-		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
-		cid, err := mp.Push(m)
-		if err != nil {
-			t.Fatal(err)
-		}
-		msgs[cid] = struct{}{}
-	}
-	err = mp.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	mp, err = New(tma, ds, "mptest")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	pmsgs, _ := mp.Pending()
-	if len(msgs) != len(pmsgs) {
-		t.Fatalf("expected %d messages, but got %d", len(msgs), len(pmsgs))
-	}
-
-	for _, m := range pmsgs {
-		cid := m.Cid()
-		_, ok := msgs[cid]
-		if !ok {
-			t.Fatal("unknown message")
-		}
-
-		delete(msgs, cid)
-	}
-
-	if len(msgs) > 0 {
-		t.Fatalf("not all messages were laoded; missing %d messages", len(msgs))
-	}
-}
-
-func TestClearAll(t *testing.T) {
-	tma := newTestMpoolAPI()
-	ds := datastore.NewMapDatastore()
-
-	mp, err := New(tma, ds, "mptest")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// the actors
-	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	tma.setBalance(a1, 1) // in FIL
-	tma.setBalance(a2, 1) // in FIL
-	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
-	for i := 0; i < 10; i++ {
-		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
-		_, err := mp.Push(m)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	for i := 0; i < 10; i++ {
-		m := makeTestMessage(w2, a2, a1, uint64(i), gasLimit, uint64(i+1))
-		mustAdd(t, mp, m)
-	}
-
-	mp.Clear(true)
-
-	pending, _ := mp.Pending()
-	if len(pending) > 0 {
-		t.Fatalf("cleared the mpool, but got %d pending messages", len(pending))
-	}
-}
-
-func TestClearNonLocal(t *testing.T) {
-	tma := newTestMpoolAPI()
-	ds := datastore.NewMapDatastore()
-
-	mp, err := New(tma, ds, "mptest")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// the actors
-	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	tma.setBalance(a1, 1) // in FIL
-	tma.setBalance(a2, 1) // in FIL
-
-	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
-	for i := 0; i < 10; i++ {
-		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
-		_, err := mp.Push(m)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	for i := 0; i < 10; i++ {
-		m := makeTestMessage(w2, a2, a1, uint64(i), gasLimit, uint64(i+1))
-		mustAdd(t, mp, m)
-	}
-
-	mp.Clear(false)
-
-	pending, _ := mp.Pending()
-	if len(pending) != 10 {
-		t.Fatalf("expected 10 pending messages, but got %d instead", len(pending))
-	}
-
-	for _, m := range pending {
-		if m.Message.From != a1 {
-			t.Fatalf("expected message from %s but got one from %s instead", a1, m.Message.From)
-		}
-	}
-}
-
-func TestUpdates(t *testing.T) {
-	tma := newTestMpoolAPI()
-	ds := datastore.NewMapDatastore()
-
-	mp, err := New(tma, ds, "mptest")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// the actors
-	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	ctx, cancel := context.WithCancel(context.TODO())
-	defer cancel()
-
-	ch, err := mp.Updates(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
-
-	tma.setBalance(a1, 1) // in FIL
-	tma.setBalance(a2, 1) // in FIL
-
-	for i := 0; i < 10; i++ {
-		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
-		_, err := mp.Push(m)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		_, ok := <-ch
-		if !ok {
-			t.Fatal("expected update, but got a closed channel instead")
-		}
-	}
-
-	err = mp.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	_, ok := <-ch
-	if ok {
-		t.Fatal("expected closed channel, but got an update instead")
-	}
-}

chain/messagepool/provider.go

@@ -1,81 +0,0 @@
-package messagepool
-
-import (
-	"context"
-
-	"github.com/ipfs/go-cid"
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/lotus/chain/stmgr"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-type Provider interface {
-	SubscribeHeadChanges(func(rev, app []*types.TipSet) error) *types.TipSet
-	PutMessage(m types.ChainMsg) (cid.Cid, error)
-	PubSubPublish(string, []byte) error
-	GetActorAfter(address.Address, *types.TipSet) (*types.Actor, error)
-	StateAccountKey(context.Context, address.Address, *types.TipSet) (address.Address, error)
-	MessagesForBlock(*types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error)
-	MessagesForTipset(*types.TipSet) ([]types.ChainMsg, error)
-	LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error)
-	ChainComputeBaseFee(ctx context.Context, ts *types.TipSet) (types.BigInt, error)
-}
-
-type mpoolProvider struct {
-	sm *stmgr.StateManager
-	ps *pubsub.PubSub
-}
-
-func NewProvider(sm *stmgr.StateManager, ps *pubsub.PubSub) Provider {
-	return &mpoolProvider{sm: sm, ps: ps}
-}
-
-func (mpp *mpoolProvider) SubscribeHeadChanges(cb func(rev, app []*types.TipSet) error) *types.TipSet {
-	mpp.sm.ChainStore().SubscribeHeadChanges(cb)
-	return mpp.sm.ChainStore().GetHeaviestTipSet()
-}
-
-func (mpp *mpoolProvider) PutMessage(m types.ChainMsg) (cid.Cid, error) {
-	return mpp.sm.ChainStore().PutMessage(m)
-}
-
-func (mpp *mpoolProvider) PubSubPublish(k string, v []byte) error {
-	return mpp.ps.Publish(k, v) //nolint
-}
-
-func (mpp *mpoolProvider) GetActorAfter(addr address.Address, ts *types.TipSet) (*types.Actor, error) {
-	var act types.Actor
-	stcid, _, err := mpp.sm.TipSetState(context.TODO(), ts)
-	if err != nil {
-		return nil, xerrors.Errorf("computing tipset state for GetActor: %w", err)
-	}
-
-	return &act, mpp.sm.WithStateTree(stcid, mpp.sm.WithActor(addr, stmgr.GetActor(&act)))
-}
-
-func (mpp *mpoolProvider) StateAccountKey(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) {
-	return mpp.sm.ResolveToKeyAddress(ctx, addr, ts)
-}
-
-func (mpp *mpoolProvider) MessagesForBlock(h *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) {
-	return mpp.sm.ChainStore().MessagesForBlock(h)
-}
-
-func (mpp *mpoolProvider) MessagesForTipset(ts *types.TipSet) ([]types.ChainMsg, error) {
-	return mpp.sm.ChainStore().MessagesForTipset(ts)
-}
-
-func (mpp *mpoolProvider) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error) {
-	return mpp.sm.ChainStore().LoadTipSet(tsk)
-}
-
-func (mpp *mpoolProvider) ChainComputeBaseFee(ctx context.Context, ts *types.TipSet) (types.BigInt, error) {
-	baseFee, err := mpp.sm.ChainStore().ComputeBaseFee(ctx, ts)
-	if err != nil {
-		return types.NewInt(0), xerrors.Errorf("computing base fee at %s: %w", ts, err)
-	}
-	return baseFee, nil
-}

chain/messagepool/pruning.go

@@ -1,105 +0,0 @@
-package messagepool
-
-import (
-	"context"
-	"sort"
-	"time"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/ipfs/go-cid"
-	"golang.org/x/xerrors"
-)
-
-func (mp *MessagePool) pruneExcessMessages() error {
-	mp.curTsLk.Lock()
-	ts := mp.curTs
-	mp.curTsLk.Unlock()
-
-	mp.lk.Lock()
-	defer mp.lk.Unlock()
-
-	if mp.currentSize < mp.cfg.SizeLimitHigh {
-		return nil
-	}
-
-	select {
-	case <-mp.pruneCooldown:
-		err := mp.pruneMessages(context.TODO(), ts)
-		go func() {
-			time.Sleep(mp.cfg.PruneCooldown)
-			mp.pruneCooldown <- struct{}{}
-		}()
-		return err
-	default:
-		return xerrors.New("cannot prune before cooldown")
-	}
-}
-
-func (mp *MessagePool) pruneMessages(ctx context.Context, ts *types.TipSet) error {
-	start := time.Now()
-	defer func() {
-		log.Infof("message pruning took %s", time.Since(start))
-	}()
-
-	baseFee, err := mp.api.ChainComputeBaseFee(ctx, ts)
-	if err != nil {
-		return xerrors.Errorf("computing basefee: %w", err)
-	}
-
-	pending, _ := mp.getPendingMessages(ts, ts)
-
-	// priority actors -- not pruned
-	priority := make(map[address.Address]struct{})
-	for _, actor := range mp.cfg.PriorityAddrs {
-		priority[actor] = struct{}{}
-	}
-
-	// Collect all messages to track which ones to remove and create chains for block inclusion
-	pruneMsgs := make(map[cid.Cid]*types.SignedMessage, mp.currentSize)
-	keepCount := 0
-
-	var chains []*msgChain
-	for actor, mset := range pending {
-		// we never prune priority actors
-		_, keep := priority[actor]
-		if keep {
-			keepCount += len(mset)
-			continue
-		}
-
-		// not a priority actor, track the messages and create chains
-		for _, m := range mset {
-			pruneMsgs[m.Message.Cid()] = m
-		}
-		actorChains := mp.createMessageChains(actor, mset, baseFee, ts)
-		chains = append(chains, actorChains...)
-	}
-
-	// Sort the chains
-	sort.Slice(chains, func(i, j int) bool {
-		return chains[i].Before(chains[j])
-	})
-
-	// Keep messages (remove them from pruneMsgs) from chains while we are under the low water mark
-	loWaterMark := mp.cfg.SizeLimitLow
-keepLoop:
-	for _, chain := range chains {
-		for _, m := range chain.msgs {
-			if keepCount < loWaterMark {
-				delete(pruneMsgs, m.Message.Cid())
-				keepCount++
-			} else {
-				break keepLoop
-			}
-		}
-	}
-
-	// and remove all messages that are still in pruneMsgs after processing the chains
-	log.Infof("Pruning %d messages", len(pruneMsgs))
-	for _, m := range pruneMsgs {
-		mp.remove(m.Message.From, m.Message.Nonce, false)
-	}
-
-	return nil
-}

chain/messagepool/repub.go

@@ -1,161 +0,0 @@
-package messagepool
-
-import (
-	"context"
-	"sort"
-	"time"
-
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/messagepool/gasguess"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/ipfs/go-cid"
-)
-
-const repubMsgLimit = 30
-
-var RepublishBatchDelay = 100 * time.Millisecond
-
-func (mp *MessagePool) republishPendingMessages() error {
-	mp.curTsLk.Lock()
-	ts := mp.curTs
-
-	baseFee, err := mp.api.ChainComputeBaseFee(context.TODO(), ts)
-	if err != nil {
-		mp.curTsLk.Unlock()
-		return xerrors.Errorf("computing basefee: %w", err)
-	}
-	baseFeeLowerBound := types.BigDiv(baseFee, baseFeeLowerBoundFactor)
-
-	pending := make(map[address.Address]map[uint64]*types.SignedMessage)
-	mp.lk.Lock()
-	mp.republished = nil // clear this to avoid races triggering an early republish
-	for actor := range mp.localAddrs {
-		mset, ok := mp.pending[actor]
-		if !ok {
-			continue
-		}
-		if len(mset.msgs) == 0 {
-			continue
-		}
-		// we need to copy this while holding the lock to avoid races with concurrent modification
-		pend := make(map[uint64]*types.SignedMessage, len(mset.msgs))
-		for nonce, m := range mset.msgs {
-			pend[nonce] = m
-		}
-		pending[actor] = pend
-	}
-	mp.lk.Unlock()
-	mp.curTsLk.Unlock()
-
-	if len(pending) == 0 {
-		return nil
-	}
-
-	var chains []*msgChain
-	for actor, mset := range pending {
-		// We use the baseFee lower bound for createChange so that we optimistically include
-		// chains that might become profitable in the next 20 blocks.
-		// We still check the lowerBound condition for individual messages so that we don't send
-		// messages that will be rejected by the mpool spam protector, so this is safe to do.
-		next := mp.createMessageChains(actor, mset, baseFeeLowerBound, ts)
-		chains = append(chains, next...)
-	}
-
-	if len(chains) == 0 {
-		return nil
-	}
-
-	sort.Slice(chains, func(i, j int) bool {
-		return chains[i].Before(chains[j])
-	})
-
-	gasLimit := int64(build.BlockGasLimit)
-	minGas := int64(gasguess.MinGas)
-	var msgs []*types.SignedMessage
-loop:
-	for i := 0; i < len(chains); {
-		chain := chains[i]
-
-		// we can exceed this if we have picked (some) longer chain already
-		if len(msgs) > repubMsgLimit {
-			break
-		}
-
-		// there is not enough gas for any message
-		if gasLimit <= minGas {
-			break
-		}
-
-		// has the chain been invalidated?
-		if !chain.valid {
-			i++
-			continue
-		}
-
-		// does it fit in a block?
-		if chain.gasLimit <= gasLimit {
-			// check the baseFee lower bound -- only republish messages that can be included in the chain
-			// within the next 20 blocks.
-			for _, m := range chain.msgs {
-				if !allowNegativeChains(ts.Height()) && m.Message.GasFeeCap.LessThan(baseFeeLowerBound) {
-					chain.Invalidate()
-					continue loop
-				}
-				gasLimit -= m.Message.GasLimit
-				msgs = append(msgs, m)
-			}
-
-			// we processed the whole chain, advance
-			i++
-			continue
-		}
-
-		// we can't fit the current chain but there is gas to spare
-		// trim it and push it down
-		chain.Trim(gasLimit, mp, baseFee, true)
-		for j := i; j < len(chains)-1; j++ {
-			if chains[j].Before(chains[j+1]) {
-				break
-			}
-			chains[j], chains[j+1] = chains[j+1], chains[j]
-		}
-	}
-
-	count := 0
-	log.Infof("republishing %d messages", len(msgs))
-	for _, m := range msgs {
-		mb, err := m.Serialize()
-		if err != nil {
-			return xerrors.Errorf("cannot serialize message: %w", err)
-		}
-
-		err = mp.api.PubSubPublish(build.MessagesTopic(mp.netName), mb)
-		if err != nil {
-			return xerrors.Errorf("cannot publish: %w", err)
-		}
-
-		count++
-
-		if count < len(msgs) {
-			// this delay is here to encourage the pubsub subsystem to process the messages serially
-			// and avoid creating nonce gaps because of concurrent validation.
-			time.Sleep(RepublishBatchDelay)
-		}
-	}
-
-	// track most recently republished messages
-	republished := make(map[cid.Cid]struct{})
-	for _, m := range msgs[:count] {
-		republished[m.Cid()] = struct{}{}
-	}
-
-	mp.lk.Lock()
-	// update the republished set so that we can trigger early republish from head changes
-	mp.republished = republished
-	mp.lk.Unlock()
-
-	return nil
-}

chain/messagepool/repub_test.go

@@ -1,72 +0,0 @@
-package messagepool
-
-import (
-	"testing"
-	"time"
-
-	"github.com/filecoin-project/go-state-types/crypto"
-	"github.com/filecoin-project/lotus/chain/messagepool/gasguess"
-	"github.com/filecoin-project/lotus/chain/wallet"
-	"github.com/filecoin-project/specs-actors/actors/builtin"
-	"github.com/ipfs/go-datastore"
-)
-
-func TestRepubMessages(t *testing.T) {
-	oldRepublishBatchDelay := RepublishBatchDelay
-	RepublishBatchDelay = time.Microsecond
-	defer func() {
-		RepublishBatchDelay = oldRepublishBatchDelay
-	}()
-
-	tma := newTestMpoolAPI()
-	ds := datastore.NewMapDatastore()
-
-	mp, err := New(tma, ds, "mptest")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// the actors
-	w1, err := wallet.NewWallet(wallet.NewMemKeyStore())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	w2, err := wallet.NewWallet(wallet.NewMemKeyStore())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}]
-
-	tma.setBalance(a1, 1) // in FIL
-
-	for i := 0; i < 10; i++ {
-		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
-		_, err := mp.Push(m)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	if tma.published != 10 {
-		t.Fatalf("expected to have published 10 messages, but got %d instead", tma.published)
-	}
-
-	mp.repubTrigger <- struct{}{}
-	time.Sleep(100 * time.Millisecond)
-
-	if tma.published != 20 {
-		t.Fatalf("expected to have published 20 messages, but got %d instead", tma.published)
-	}
-}

chain/messagepool/selection.go

@@ -1,854 +0,0 @@
-package messagepool
-
-import (
-	"context"
-	"math/big"
-	"sort"
-	"time"
-
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	tbig "github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/messagepool/gasguess"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/chain/vm"
-)
-
-var bigBlockGasLimit = big.NewInt(build.BlockGasLimit)
-
-// this is *temporary* mutilation until we have implemented uncapped miner penalties -- it will go
-// away in the next fork.
-func allowNegativeChains(epoch abi.ChainEpoch) bool {
-	return epoch < build.UpgradeBreezeHeight+5
-}
-
-const MaxBlocks = 15
-
-type msgChain struct {
-	msgs         []*types.SignedMessage
-	gasReward    *big.Int
-	gasLimit     int64
-	gasPerf      float64
-	effPerf      float64
-	bp           float64
-	parentOffset float64
-	valid        bool
-	merged       bool
-	next         *msgChain
-	prev         *msgChain
-}
-
-func (mp *MessagePool) SelectMessages(ts *types.TipSet, tq float64) ([]*types.SignedMessage, error) {
-	mp.curTsLk.Lock()
-	defer mp.curTsLk.Unlock()
-
-	mp.lk.Lock()
-	defer mp.lk.Unlock()
-
-	// if the ticket quality is high enough that the first block has higher probability
-	// than any other block, then we don't bother with optimal selection because the
-	// first block will always have higher effective performance
-	if tq > 0.84 {
-		return mp.selectMessagesGreedy(mp.curTs, ts)
-	}
-
-	return mp.selectMessagesOptimal(mp.curTs, ts, tq)
-}
-
-func (mp *MessagePool) selectMessagesOptimal(curTs, ts *types.TipSet, tq float64) ([]*types.SignedMessage, error) {
-	start := time.Now()
-
-	baseFee, err := mp.api.ChainComputeBaseFee(context.TODO(), ts)
-	if err != nil {
-		return nil, xerrors.Errorf("computing basefee: %w", err)
-	}
-
-	// 0. Load messages from the target tipset; if it is the same as the current tipset in
-	//    the mpool, then this is just the pending messages
-	pending, err := mp.getPendingMessages(curTs, ts)
-	if err != nil {
-		return nil, err
-	}
-
-	if len(pending) == 0 {
-		return nil, nil
-	}
-
-	// defer only here so if we have no pending messages we don't spam
-	defer func() {
-		log.Infow("message selection done", "took", time.Since(start))
-	}()
-
-	// 0b. Select all priority messages that fit in the block
-	minGas := int64(gasguess.MinGas)
-	result, gasLimit := mp.selectPriorityMessages(pending, baseFee, ts)
-
-	// have we filled the block?
-	if gasLimit < minGas {
-		return result, nil
-	}
-
-	// 1. Create a list of dependent message chains with maximal gas reward per limit consumed
-	startChains := time.Now()
-	var chains []*msgChain
-	for actor, mset := range pending {
-		next := mp.createMessageChains(actor, mset, baseFee, ts)
-		chains = append(chains, next...)
-	}
-	if dt := time.Since(startChains); dt > time.Millisecond {
-		log.Infow("create message chains done", "took", dt)
-	}
-
-	// 2. Sort the chains
-	sort.Slice(chains, func(i, j int) bool {
-		return chains[i].Before(chains[j])
-	})
-
-	if !allowNegativeChains(curTs.Height()) && len(chains) != 0 && chains[0].gasPerf < 0 {
-		log.Warnw("all messages in mpool have non-positive gas performance", "bestGasPerf", chains[0].gasPerf)
-		return result, nil
-	}
-
-	// 3. Parition chains into blocks (without trimming)
-	//    we use the full blockGasLimit (as opposed to the residual gas limit from the
-	//    priority message selection) as we have to account for what other miners are doing
-	nextChain := 0
-	partitions := make([][]*msgChain, MaxBlocks)
-	for i := 0; i < MaxBlocks && nextChain < len(chains); i++ {
-		gasLimit := int64(build.BlockGasLimit)
-		for nextChain < len(chains) {
-			chain := chains[nextChain]
-			nextChain++
-			partitions[i] = append(partitions[i], chain)
-			gasLimit -= chain.gasLimit
-			if gasLimit < minGas {
-				break
-			}
-		}
-
-	}
-
-	// 4. Compute effective performance for each chain, based on the partition they fall into
-	//    The effective performance is the gasPerf of the chain * block probability
-	blockProb := mp.blockProbabilities(tq)
-	effChains := 0
-	for i := 0; i < MaxBlocks; i++ {
-		for _, chain := range partitions[i] {
-			chain.SetEffectivePerf(blockProb[i])
-		}
-		effChains += len(partitions[i])
-	}
-
-	// nullify the effective performance of chains that don't fit in any partition
-	for _, chain := range chains[effChains:] {
-		chain.SetNullEffectivePerf()
-	}
-
-	// 5. Resort the chains based on effective performance
-	sort.Slice(chains, func(i, j int) bool {
-		return chains[i].BeforeEffective(chains[j])
-	})
-
-	// 6. Merge the head chains to produce the list of messages selected for inclusion
-	//    subject to the residual gas limit
-	//    When a chain is merged in, all its previous dependent chains *must* also be
-	//    merged in or we'll have a broken block
-	startMerge := time.Now()
-	last := len(chains)
-	for i, chain := range chains {
-		// did we run out of performing chains?
-		if !allowNegativeChains(curTs.Height()) && chain.gasPerf < 0 {
-			break
-		}
-
-		// has it already been merged?
-		if chain.merged {
-			continue
-		}
-
-		// compute the dependencies that must be merged and the gas limit including deps
-		chainGasLimit := chain.gasLimit
-		var chainDeps []*msgChain
-		for curChain := chain.prev; curChain != nil && !curChain.merged; curChain = curChain.prev {
-			chainDeps = append(chainDeps, curChain)
-			chainGasLimit += curChain.gasLimit
-		}
-
-		// does it all fit in the block?
-		if chainGasLimit <= gasLimit {
-			// include it together with all dependencies
-			for i := len(chainDeps) - 1; i >= 0; i-- {
-				curChain := chainDeps[i]
-				curChain.merged = true
-				result = append(result, curChain.msgs...)
-			}
-
-			chain.merged = true
-			// adjust the effective pefromance for all subsequent chains
-			if next := chain.next; next != nil && next.effPerf > 0 {
-				next.effPerf += next.parentOffset
-				for next = next.next; next != nil && next.effPerf > 0; next = next.next {
-					next.setEffPerf()
-				}
-			}
-			result = append(result, chain.msgs...)
-			gasLimit -= chainGasLimit
-
-			// resort to account for already merged chains and effective performance adjustments
-			sort.Slice(chains[i+1:], func(i, j int) bool {
-				return chains[i].BeforeEffective(chains[j])
-			})
-			continue
-		}
-
-		// we can't fit this chain and its dependencies because of block gasLimit -- we are
-		// at the edge
-		last = i
-		break
-	}
-	if dt := time.Since(startMerge); dt > time.Millisecond {
-		log.Infow("merge message chains done", "took", dt)
-	}
-
-	// 7. We have reached the edge of what can fit wholesale; if we still hae available
-	//    gasLimit to pack some more chains, then trim the last chain and push it down.
-	//    Trimming invalidaates subsequent dependent chains so that they can't be selected
-	//    as their dependency cannot be (fully) included.
-	//    We do this in a loop because the blocker might have been inordinately large and
-	//    we might have to do it multiple times to satisfy tail packing
-	startTail := time.Now()
-tailLoop:
-	for gasLimit >= minGas && last < len(chains) {
-		// trim if necessary
-		if chains[last].gasLimit > gasLimit {
-			chains[last].Trim(gasLimit, mp, baseFee, allowNegativeChains(curTs.Height()))
-		}
-
-		// push down if it hasn't been invalidated
-		if chains[last].valid {
-			for i := last; i < len(chains)-1; i++ {
-				if chains[i].BeforeEffective(chains[i+1]) {
-					break
-				}
-				chains[i], chains[i+1] = chains[i+1], chains[i]
-			}
-		}
-
-		// select the next (valid and fitting) chain and its dependencies for inclusion
-		for i, chain := range chains[last:] {
-			// has the chain been invalidated?
-			if !chain.valid {
-				continue
-			}
-
-			// has it already been merged?
-			if chain.merged {
-				continue
-			}
-
-			// if gasPerf < 0 we have no more profitable chains
-			if !allowNegativeChains(curTs.Height()) && chain.gasPerf < 0 {
-				break tailLoop
-			}
-
-			// compute the dependencies that must be merged and the gas limit including deps
-			chainGasLimit := chain.gasLimit
-			depGasLimit := int64(0)
-			var chainDeps []*msgChain
-			for curChain := chain.prev; curChain != nil && !curChain.merged; curChain = curChain.prev {
-				chainDeps = append(chainDeps, curChain)
-				chainGasLimit += curChain.gasLimit
-				depGasLimit += curChain.gasLimit
-			}
-
-			// does it all fit in the bock
-			if chainGasLimit <= gasLimit {
-				// include it together with all dependencies
-				for i := len(chainDeps) - 1; i >= 0; i-- {
-					curChain := chainDeps[i]
-					curChain.merged = true
-					result = append(result, curChain.msgs...)
-				}
-
-				chain.merged = true
-				result = append(result, chain.msgs...)
-				gasLimit -= chainGasLimit
-				continue
-			}
-
-			// it doesn't all fit; now we have to take into account the dependent chains before
-			// making a decision about trimming or invalidating.
-			// if the dependencies exceed the gas limit, then we must invalidate the chain
-			// as it can never be included.
-			// Otherwise we can just trim and continue
-			if depGasLimit > gasLimit {
-				chain.Invalidate()
-				last += i + 1
-				continue tailLoop
-			}
-
-			// dependencies fit, just trim it
-			chain.Trim(gasLimit-depGasLimit, mp, baseFee, allowNegativeChains(curTs.Height()))
-			last += i
-			continue tailLoop
-		}
-
-		// the merge loop ended after processing all the chains and we we probably have still
-		// gas to spare; end the loop.
-		break
-	}
-	if dt := time.Since(startTail); dt > time.Millisecond {
-		log.Infow("pack tail chains done", "took", dt)
-	}
-
-	return result, nil
-}
-
-func (mp *MessagePool) selectMessagesGreedy(curTs, ts *types.TipSet) ([]*types.SignedMessage, error) {
-	start := time.Now()
-
-	baseFee, err := mp.api.ChainComputeBaseFee(context.TODO(), ts)
-	if err != nil {
-		return nil, xerrors.Errorf("computing basefee: %w", err)
-	}
-
-	// 0. Load messages for the target tipset; if it is the same as the current tipset in the mpool
-	//    then this is just the pending messages
-	pending, err := mp.getPendingMessages(curTs, ts)
-	if err != nil {
-		return nil, err
-	}
-
-	if len(pending) == 0 {
-		return nil, nil
-	}
-
-	// defer only here so if we have no pending messages we don't spam
-	defer func() {
-		log.Infow("message selection done", "took", time.Since(start))
-	}()
-
-	// 0b. Select all priority messages that fit in the block
-	minGas := int64(gasguess.MinGas)
-	result, gasLimit := mp.selectPriorityMessages(pending, baseFee, ts)
-
-	// have we filled the block?
-	if gasLimit < minGas {
-		return result, nil
-	}
-
-	// 1. Create a list of dependent message chains with maximal gas reward per limit consumed
-	startChains := time.Now()
-	var chains []*msgChain
-	for actor, mset := range pending {
-		next := mp.createMessageChains(actor, mset, baseFee, ts)
-		chains = append(chains, next...)
-	}
-	if dt := time.Since(startChains); dt > time.Millisecond {
-		log.Infow("create message chains done", "took", dt)
-	}
-
-	// 2. Sort the chains
-	sort.Slice(chains, func(i, j int) bool {
-		return chains[i].Before(chains[j])
-	})
-
-	if !allowNegativeChains(curTs.Height()) && len(chains) != 0 && chains[0].gasPerf < 0 {
-		log.Warnw("all messages in mpool have non-positive gas performance", "bestGasPerf", chains[0].gasPerf)
-		return result, nil
-	}
-
-	// 3. Merge the head chains to produce the list of messages selected for inclusion, subject to
-	//    the block gas limit.
-	startMerge := time.Now()
-	last := len(chains)
-	for i, chain := range chains {
-		// did we run out of performing chains?
-		if !allowNegativeChains(curTs.Height()) && chain.gasPerf < 0 {
-			break
-		}
-
-		// does it fit in the block?
-		if chain.gasLimit <= gasLimit {
-			gasLimit -= chain.gasLimit
-			result = append(result, chain.msgs...)
-			continue
-		}
-
-		// we can't fit this chain because of block gasLimit -- we are at the edge
-		last = i
-		break
-	}
-	if dt := time.Since(startMerge); dt > time.Millisecond {
-		log.Infow("merge message chains done", "took", dt)
-	}
-
-	// 4. We have reached the edge of what we can fit wholesale; if we still have available gasLimit
-	// to pack some more chains, then trim the last chain and push it down.
-	// Trimming invalidates subsequent dependent chains so that they can't be selected as their
-	// dependency cannot be (fully) included.
-	// We do this in a loop because the blocker might have been inordinately large and we might
-	// have to do it multiple times to satisfy tail packing.
-	startTail := time.Now()
-tailLoop:
-	for gasLimit >= minGas && last < len(chains) {
-		// trim
-		chains[last].Trim(gasLimit, mp, baseFee, allowNegativeChains(curTs.Height()))
-
-		// push down if it hasn't been invalidated
-		if chains[last].valid {
-			for i := last; i < len(chains)-1; i++ {
-				if chains[i].Before(chains[i+1]) {
-					break
-				}
-				chains[i], chains[i+1] = chains[i+1], chains[i]
-			}
-		}
-
-		// select the next (valid and fitting) chain for inclusion
-		for i, chain := range chains[last:] {
-			// has the chain been invalidated?
-			if !chain.valid {
-				continue
-			}
-
-			// if gasPerf < 0 we have no more profitable chains
-			if !allowNegativeChains(curTs.Height()) && chain.gasPerf < 0 {
-				break tailLoop
-			}
-
-			// does it fit in the bock?
-			if chain.gasLimit <= gasLimit {
-				gasLimit -= chain.gasLimit
-				result = append(result, chain.msgs...)
-				continue
-			}
-
-			// this chain needs to be trimmed
-			last += i
-			continue tailLoop
-		}
-
-		// the merge loop ended after processing all the chains and we probably still have
-		// gas to spare; end the loop
-		break
-	}
-	if dt := time.Since(startTail); dt > time.Millisecond {
-		log.Infow("pack tail chains done", "took", dt)
-	}
-
-	return result, nil
-}
-
-func (mp *MessagePool) selectPriorityMessages(pending map[address.Address]map[uint64]*types.SignedMessage, baseFee types.BigInt, ts *types.TipSet) ([]*types.SignedMessage, int64) {
-	start := time.Now()
-	defer func() {
-		if dt := time.Since(start); dt > time.Millisecond {
-			log.Infow("select priority messages done", "took", dt)
-		}
-	}()
-
-	result := make([]*types.SignedMessage, 0, mp.cfg.SizeLimitLow)
-	gasLimit := int64(build.BlockGasLimit)
-	minGas := int64(gasguess.MinGas)
-
-	// 1. Get priority actor chains
-	var chains []*msgChain
-	priority := mp.cfg.PriorityAddrs
-	for _, actor := range priority {
-		mset, ok := pending[actor]
-		if ok {
-			// remove actor from pending set as we are already processed these messages
-			delete(pending, actor)
-			// create chains for the priority actor
-			next := mp.createMessageChains(actor, mset, baseFee, ts)
-			chains = append(chains, next...)
-		}
-	}
-
-	if len(chains) == 0 {
-		return nil, gasLimit
-	}
-
-	// 2. Sort the chains
-	sort.Slice(chains, func(i, j int) bool {
-		return chains[i].Before(chains[j])
-	})
-
-	if !allowNegativeChains(ts.Height()) && len(chains) != 0 && chains[0].gasPerf < 0 {
-		log.Warnw("all priority messages in mpool have negative gas performance", "bestGasPerf", chains[0].gasPerf)
-		return nil, gasLimit
-	}
-
-	// 3. Merge chains until the block limit, as long as they have non-negative gas performance
-	last := len(chains)
-	for i, chain := range chains {
-		if !allowNegativeChains(ts.Height()) && chain.gasPerf < 0 {
-			break
-		}
-
-		if chain.gasLimit <= gasLimit {
-			gasLimit -= chain.gasLimit
-			result = append(result, chain.msgs...)
-			continue
-		}
-
-		// we can't fit this chain because of block gasLimit -- we are at the edge
-		last = i
-		break
-	}
-
-tailLoop:
-	for gasLimit >= minGas && last < len(chains) {
-		// trim, discarding negative performing messages
-		chains[last].Trim(gasLimit, mp, baseFee, allowNegativeChains(ts.Height()))
-
-		// push down if it hasn't been invalidated
-		if chains[last].valid {
-			for i := last; i < len(chains)-1; i++ {
-				if chains[i].Before(chains[i+1]) {
-					break
-				}
-				chains[i], chains[i+1] = chains[i+1], chains[i]
-			}
-		}
-
-		// select the next (valid and fitting) chain for inclusion
-		for i, chain := range chains[last:] {
-			// has the chain been invalidated
-			if !chain.valid {
-				continue
-			}
-
-			// if gasPerf < 0 we have no more profitable chains
-			if !allowNegativeChains(ts.Height()) && chain.gasPerf < 0 {
-				break tailLoop
-			}
-
-			// does it fit in the bock?
-			if chain.gasLimit <= gasLimit {
-				gasLimit -= chain.gasLimit
-				result = append(result, chain.msgs...)
-				continue
-			}
-
-			// this chain needs to be trimmed
-			last += i
-			continue tailLoop
-		}
-
-		// the merge loop ended after processing all the chains and we probably still have gas to spare;
-		// end the loop
-		break
-	}
-
-	return result, gasLimit
-}
-
-func (mp *MessagePool) getPendingMessages(curTs, ts *types.TipSet) (map[address.Address]map[uint64]*types.SignedMessage, error) {
-	start := time.Now()
-
-	result := make(map[address.Address]map[uint64]*types.SignedMessage)
-	defer func() {
-		if dt := time.Since(start); dt > time.Millisecond {
-			log.Infow("get pending messages done", "took", dt)
-		}
-	}()
-
-	// are we in sync?
-	inSync := false
-	if curTs.Height() == ts.Height() && curTs.Equals(ts) {
-		inSync = true
-	}
-
-	// first add our current pending messages
-	for a, mset := range mp.pending {
-		if inSync {
-			// no need to copy the map
-			result[a] = mset.msgs
-		} else {
-			// we need to copy the map to avoid clobbering it as we load more messages
-			msetCopy := make(map[uint64]*types.SignedMessage, len(mset.msgs))
-			for nonce, m := range mset.msgs {
-				msetCopy[nonce] = m
-			}
-			result[a] = msetCopy
-
-		}
-	}
-
-	// we are in sync, that's the happy path
-	if inSync {
-		return result, nil
-	}
-
-	if err := mp.runHeadChange(curTs, ts, result); err != nil {
-		return nil, xerrors.Errorf("failed to process difference between mpool head and given head: %w", err)
-	}
-
-	return result, nil
-}
-
-func (*MessagePool) getGasReward(msg *types.SignedMessage, baseFee types.BigInt) *big.Int {
-	maxPremium := types.BigSub(msg.Message.GasFeeCap, baseFee)
-
-	if types.BigCmp(maxPremium, msg.Message.GasPremium) > 0 {
-		maxPremium = msg.Message.GasPremium
-	}
-
-	gasReward := tbig.Mul(maxPremium, types.NewInt(uint64(msg.Message.GasLimit)))
-	return gasReward.Int
-}
-
-func (*MessagePool) getGasPerf(gasReward *big.Int, gasLimit int64) float64 {
-	// gasPerf = gasReward * build.BlockGasLimit / gasLimit
-	a := new(big.Rat).SetInt(new(big.Int).Mul(gasReward, bigBlockGasLimit))
-	b := big.NewRat(1, gasLimit)
-	c := new(big.Rat).Mul(a, b)
-	r, _ := c.Float64()
-	return r
-}
-
-func (mp *MessagePool) createMessageChains(actor address.Address, mset map[uint64]*types.SignedMessage, baseFee types.BigInt, ts *types.TipSet) []*msgChain {
-	// collect all messages
-	msgs := make([]*types.SignedMessage, 0, len(mset))
-	for _, m := range mset {
-		msgs = append(msgs, m)
-	}
-
-	// sort by nonce
-	sort.Slice(msgs, func(i, j int) bool {
-		return msgs[i].Message.Nonce < msgs[j].Message.Nonce
-	})
-
-	// sanity checks:
-	// - there can be no gaps in nonces, starting from the current actor nonce
-	//   if there is a gap, drop messages after the gap, we can't include them
-	// - all messages must have minimum gas and the total gas for the candidate messages
-	//   cannot exceed the block limit; drop all messages that exceed the limit
-	// - the total gasReward cannot exceed the actor's balance; drop all messages that exceed
-	//   the balance
-	a, err := mp.api.GetActorAfter(actor, ts)
-	if err != nil {
-		log.Errorf("failed to load actor state, not building chain for %s: %w", actor, err)
-		return nil
-	}
-
-	curNonce := a.Nonce
-	balance := a.Balance.Int
-	gasLimit := int64(0)
-	skip := 0
-	i := 0
-	rewards := make([]*big.Int, 0, len(msgs))
-	for i = 0; i < len(msgs); i++ {
-		m := msgs[i]
-
-		if m.Message.Nonce < curNonce {
-			log.Warnf("encountered message from actor %s with nonce (%d) less than the current nonce (%d)",
-				actor, m.Message.Nonce, curNonce)
-			skip++
-			continue
-		}
-
-		if m.Message.Nonce != curNonce {
-			break
-		}
-		curNonce++
-
-		minGas := vm.PricelistByEpoch(ts.Height()).OnChainMessage(m.ChainLength()).Total()
-		if m.Message.GasLimit < minGas {
-			break
-		}
-
-		gasLimit += m.Message.GasLimit
-		if gasLimit > build.BlockGasLimit {
-			break
-		}
-
-		required := m.Message.RequiredFunds().Int
-		if balance.Cmp(required) < 0 {
-			break
-		}
-		balance = new(big.Int).Sub(balance, required)
-
-		value := m.Message.Value.Int
-		if balance.Cmp(value) >= 0 {
-			// Note: we only account for the value if the balance doesn't drop below 0
-			//       otherwise the message will fail and the miner can reap the gas rewards
-			balance = new(big.Int).Sub(balance, value)
-		}
-
-		gasReward := mp.getGasReward(m, baseFee)
-		rewards = append(rewards, gasReward)
-	}
-
-	// check we have a sane set of messages to construct the chains
-	if i > skip {
-		msgs = msgs[skip:i]
-	} else {
-		return nil
-	}
-
-	// ok, now we can construct the chains using the messages we have
-	// invariant: each chain has a bigger gasPerf than the next -- otherwise they can be merged
-	// and increase the gasPerf of the first chain
-	// We do this in two passes:
-	// - in the first pass we create chains that aggreagate messages with non-decreasing gasPerf
-	// - in the second pass we merge chains to maintain the invariant.
-	var chains []*msgChain
-	var curChain *msgChain
-
-	newChain := func(m *types.SignedMessage, i int) *msgChain {
-		chain := new(msgChain)
-		chain.msgs = []*types.SignedMessage{m}
-		chain.gasReward = rewards[i]
-		chain.gasLimit = m.Message.GasLimit
-		chain.gasPerf = mp.getGasPerf(chain.gasReward, chain.gasLimit)
-		chain.valid = true
-		return chain
-	}
-
-	// create the individual chains
-	for i, m := range msgs {
-		if curChain == nil {
-			curChain = newChain(m, i)
-			continue
-		}
-
-		gasReward := new(big.Int).Add(curChain.gasReward, rewards[i])
-		gasLimit := curChain.gasLimit + m.Message.GasLimit
-		gasPerf := mp.getGasPerf(gasReward, gasLimit)
-
-		// try to add the message to the current chain -- if it decreases the gasPerf, then make a
-		// new chain
-		if gasPerf < curChain.gasPerf {
-			chains = append(chains, curChain)
-			curChain = newChain(m, i)
-		} else {
-			curChain.msgs = append(curChain.msgs, m)
-			curChain.gasReward = gasReward
-			curChain.gasLimit = gasLimit
-			curChain.gasPerf = gasPerf
-		}
-	}
-	chains = append(chains, curChain)
-
-	// merge chains to maintain the invariant
-	for {
-		merged := 0
-
-		for i := len(chains) - 1; i > 0; i-- {
-			if chains[i].gasPerf >= chains[i-1].gasPerf {
-				chains[i-1].msgs = append(chains[i-1].msgs, chains[i].msgs...)
-				chains[i-1].gasReward = new(big.Int).Add(chains[i-1].gasReward, chains[i].gasReward)
-				chains[i-1].gasLimit += chains[i].gasLimit
-				chains[i-1].gasPerf = mp.getGasPerf(chains[i-1].gasReward, chains[i-1].gasLimit)
-				chains[i].valid = false
-				merged++
-			}
-		}
-
-		if merged == 0 {
-			break
-		}
-
-		// drop invalidated chains
-		newChains := make([]*msgChain, 0, len(chains)-merged)
-		for _, c := range chains {
-			if c.valid {
-				newChains = append(newChains, c)
-			}
-		}
-		chains = newChains
-	}
-
-	// link dependent chains
-	for i := 0; i < len(chains)-1; i++ {
-		chains[i].next = chains[i+1]
-	}
-
-	for i := len(chains) - 1; i > 0; i-- {
-		chains[i].prev = chains[i-1]
-	}
-
-	return chains
-}
-
-func (mc *msgChain) Before(other *msgChain) bool {
-	return mc.gasPerf > other.gasPerf ||
-		(mc.gasPerf == other.gasPerf && mc.gasReward.Cmp(other.gasReward) > 0)
-}
-
-func (mc *msgChain) Trim(gasLimit int64, mp *MessagePool, baseFee types.BigInt, allowNegative bool) {
-	i := len(mc.msgs) - 1
-	for i >= 0 && (mc.gasLimit > gasLimit || (!allowNegative && mc.gasPerf < 0)) {
-		gasReward := mp.getGasReward(mc.msgs[i], baseFee)
-		mc.gasReward = new(big.Int).Sub(mc.gasReward, gasReward)
-		mc.gasLimit -= mc.msgs[i].Message.GasLimit
-		if mc.gasLimit > 0 {
-			mc.gasPerf = mp.getGasPerf(mc.gasReward, mc.gasLimit)
-			if mc.bp != 0 {
-				mc.setEffPerf()
-			}
-		} else {
-			mc.gasPerf = 0
-			mc.effPerf = 0
-		}
-		i--
-	}
-
-	if i < 0 {
-		mc.msgs = nil
-		mc.valid = false
-	} else {
-		mc.msgs = mc.msgs[:i+1]
-	}
-
-	if mc.next != nil {
-		mc.next.Invalidate()
-		mc.next = nil
-	}
-}
-
-func (mc *msgChain) Invalidate() {
-	mc.valid = false
-	mc.msgs = nil
-	if mc.next != nil {
-		mc.next.Invalidate()
-		mc.next = nil
-	}
-}
-
-func (mc *msgChain) SetEffectivePerf(bp float64) {
-	mc.bp = bp
-	mc.setEffPerf()
-}
-
-func (mc *msgChain) setEffPerf() {
-	effPerf := mc.gasPerf * mc.bp
-	if effPerf > 0 && mc.prev != nil {
-		effPerfWithParent := (effPerf*float64(mc.gasLimit) + mc.prev.effPerf*float64(mc.prev.gasLimit)) / float64(mc.gasLimit+mc.prev.gasLimit)
-		mc.parentOffset = effPerf - effPerfWithParent
-		effPerf = effPerfWithParent
-	}
-	mc.effPerf = effPerf
-
-}
-
-func (mc *msgChain) SetNullEffectivePerf() {
-	if mc.gasPerf < 0 {
-		mc.effPerf = mc.gasPerf
-	} else {
-		mc.effPerf = 0
-	}
-}
-
-func (mc *msgChain) BeforeEffective(other *msgChain) bool {
-	// move merged chains to the front so we can discard them earlier
-	return (mc.merged && !other.merged) || mc.effPerf > other.effPerf ||
-		(mc.effPerf == other.effPerf && mc.gasPerf > other.gasPerf) ||
-		(mc.effPerf == other.effPerf && mc.gasPerf == other.gasPerf && mc.gasReward.Cmp(other.gasReward) > 0)
-}

chain/metrics/consensus.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"encoding/json"
 
-	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log/v2"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
@@ -44,7 +44,7 @@ func SendHeadNotifs(nickname string) func(mctx helpers.MetricsCtx, lc fx.Lifecyc
 					}
 				}()
 				go func() {
-					sub, err := ps.Subscribe(topic) //nolint
+					sub, err := ps.Subscribe(topic)
 					if err != nil {
 						return
 					}
@@ -116,7 +116,6 @@ func sendHeadNotifs(ctx context.Context, ps *pubsub.PubSub, topic string, chain
 				return err
 			}
 
-			//nolint
 			if err := ps.Publish(topic, b); err != nil {
 				return err
 			}

chain/state/statetree.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
 	"github.com/ipfs/go-cid"
+	hamt "github.com/ipfs/go-hamt-ipld"
 	cbor "github.com/ipfs/go-ipld-cbor"
 	logging "github.com/ipfs/go-log/v2"
 	"go.opencensus.io/trace"
@@ -22,7 +23,7 @@ var log = logging.Logger("statetree")
 
 // StateTree stores actors state by their ID.
 type StateTree struct {
-	root  *adt.Map
+	root  *hamt.Node
 	Store cbor.IpldStore
 
 	snaps *stateSnaps
@@ -116,16 +117,15 @@ func (ss *stateSnaps) deleteActor(addr address.Address) {
 }
 
 func NewStateTree(cst cbor.IpldStore) (*StateTree, error) {
-
 	return &StateTree{
-		root:  adt.MakeEmptyMap(adt.WrapStore(context.TODO(), cst)),
+		root:  hamt.NewNode(cst, hamt.UseTreeBitWidth(5)),
 		Store: cst,
 		snaps: newStateSnaps(),
 	}, nil
 }
 
 func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) {
-	nd, err := adt.AsMap(adt.WrapStore(context.TODO(), cst), c)
+	nd, err := hamt.LoadNode(context.Background(), cst, c, hamt.UseTreeBitWidth(5))
 	if err != nil {
 		log.Errorf("loading hamt node %s failed: %s", c, err)
 		return nil, err
@@ -170,10 +170,7 @@ func (st *StateTree) LookupID(addr address.Address) (address.Address, error) {
 		return address.Undef, xerrors.Errorf("loading init actor state: %w", err)
 	}
 
-	a, found, err := ias.ResolveAddress(&AdtStore{st.Store}, addr)
-	if err == nil && !found {
-		err = types.ErrActorNotFound
-	}
+	a, err := ias.ResolveAddress(&AdtStore{st.Store}, addr)
 	if err != nil {
 		return address.Undef, xerrors.Errorf("resolve address %s: %w", addr, err)
 	}
@@ -192,7 +189,7 @@ func (st *StateTree) GetActor(addr address.Address) (*types.Actor, error) {
 	// Transform `addr` to its ID format.
 	iaddr, err := st.LookupID(addr)
 	if err != nil {
-		if xerrors.Is(err, types.ErrActorNotFound) {
+		if xerrors.Is(err, init_.ErrAddressNotFound) {
 			return nil, xerrors.Errorf("resolution lookup failed (%s): %w", addr, err)
 		}
 		return nil, xerrors.Errorf("address resolution: %w", err)
@@ -209,10 +206,12 @@ func (st *StateTree) GetActor(addr address.Address) (*types.Actor, error) {
 	}
 
 	var act types.Actor
-	if found, err := st.root.Get(adt.AddrKey(addr), &act); err != nil {
+	err = st.root.Find(context.TODO(), string(addr.Bytes()), &act)
+	if err != nil {
+		if err == hamt.ErrNotFound {
+			return nil, types.ErrActorNotFound
+		}
 		return nil, xerrors.Errorf("hamt find failed: %w", err)
-	} else if !found {
-		return nil, types.ErrActorNotFound
 	}
 
 	st.snaps.setActor(addr, &act)
@@ -227,7 +226,7 @@ func (st *StateTree) DeleteActor(addr address.Address) error {
 
 	iaddr, err := st.LookupID(addr)
 	if err != nil {
-		if xerrors.Is(err, types.ErrActorNotFound) {
+		if xerrors.Is(err, init_.ErrAddressNotFound) {
 			return xerrors.Errorf("resolution lookup failed (%s): %w", addr, err)
 		}
 		return xerrors.Errorf("address resolution: %w", err)
@@ -246,7 +245,7 @@ func (st *StateTree) DeleteActor(addr address.Address) error {
 }
 
 func (st *StateTree) Flush(ctx context.Context) (cid.Cid, error) {
-	ctx, span := trace.StartSpan(ctx, "stateTree.Flush") //nolint:staticcheck
+	ctx, span := trace.StartSpan(ctx, "stateTree.Flush")
 	defer span.End()
 	if len(st.snaps.layers) != 1 {
 		return cid.Undef, xerrors.Errorf("tried to flush state tree with snapshots on the stack")
@@ -254,21 +253,25 @@ func (st *StateTree) Flush(ctx context.Context) (cid.Cid, error) {
 
 	for addr, sto := range st.snaps.layers[0].actors {
 		if sto.Delete {
-			if err := st.root.Delete(adt.AddrKey(addr)); err != nil {
+			if err := st.root.Delete(ctx, string(addr.Bytes())); err != nil {
 				return cid.Undef, err
 			}
 		} else {
-			if err := st.root.Put(adt.AddrKey(addr), &sto.Act); err != nil {
+			if err := st.root.Set(ctx, string(addr.Bytes()), &sto.Act); err != nil {
 				return cid.Undef, err
 			}
 		}
 	}
 
-	return st.root.Root()
+	if err := st.root.Flush(ctx); err != nil {
+		return cid.Undef, err
+	}
+
+	return st.Store.Put(ctx, st.root)
 }
 
 func (st *StateTree) Snapshot(ctx context.Context) error {
-	ctx, span := trace.StartSpan(ctx, "stateTree.SnapShot") //nolint:staticcheck
+	ctx, span := trace.StartSpan(ctx, "stateTree.SnapShot")
 	defer span.End()
 
 	st.snaps.addLayer()
@@ -336,15 +339,3 @@ func (st *StateTree) MutateActor(addr address.Address, f func(*types.Actor) erro
 
 	return st.SetActor(addr, act)
 }
-
-func (st *StateTree) ForEach(f func(address.Address, *types.Actor) error) error {
-	var act types.Actor
-	return st.root.ForEach(&act, func(k string) error {
-		addr, err := address.NewFromBytes([]byte(k))
-		if err != nil {
-			return xerrors.Errorf("invalid address (%x) found in state tree key: %w", []byte(k), err)
-		}
-
-		return f(addr, &act)
-	})
-}

chain/state/statetree_test.go

@@ -272,7 +272,7 @@ func TestStateTreeConsistency(t *testing.T) {
 	}
 
 	fmt.Println("root is: ", root)
-	if root.String() != "bafy2bzaceb2bhqw75pqp44efoxvlnm73lnctq6djair56bfn5x3gw56epcxbi" {
+	if root.String() != "bafy2bzaceadyjnrv3sbjvowfl3jr4pdn5p2bf3exjjie2f3shg4oy5sub7h34" {
 		t.Fatal("MISMATCH!")
 	}
 }

chain/stmgr/call.go

@@ -4,9 +4,7 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/ipfs/go-cid"
 	"go.opencensus.io/trace"
 	"golang.org/x/xerrors"
@@ -22,18 +20,7 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate
 	ctx, span := trace.StartSpan(ctx, "statemanager.CallRaw")
 	defer span.End()
 
-	vmopt := &vm.VMOpts{
-		StateBase:      bstate,
-		Epoch:          bheight,
-		Rand:           r,
-		Bstore:         sm.cs.Blockstore(),
-		Syscalls:       sm.cs.VMSys(),
-		CircSupplyCalc: sm.GetCirculatingSupply,
-		NtwkVersion:    sm.GetNtwkVersion,
-		BaseFee:        types.NewInt(0),
-	}
-
-	vmi, err := vm.NewVM(vmopt)
+	vmi, err := vm.NewVM(bstate, bheight, r, sm.cs.Blockstore(), sm.cs.VMSys())
 	if err != nil {
 		return nil, xerrors.Errorf("failed to set up vm: %w", err)
 	}
@@ -41,13 +28,9 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate
 	if msg.GasLimit == 0 {
 		msg.GasLimit = build.BlockGasLimit
 	}
-	if msg.GasFeeCap == types.EmptyInt {
-		msg.GasFeeCap = types.NewInt(0)
+	if msg.GasPrice == types.EmptyInt {
+		msg.GasPrice = types.NewInt(0)
 	}
-	if msg.GasPremium == types.EmptyInt {
-		msg.GasPremium = types.NewInt(0)
-	}
-
 	if msg.Value == types.EmptyInt {
 		msg.Value = types.NewInt(0)
 	}
@@ -55,7 +38,7 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate
 	if span.IsRecordingEvents() {
 		span.AddAttributes(
 			trace.Int64Attribute("gas_limit", msg.GasLimit),
-			trace.StringAttribute("gas_feecap", msg.GasFeeCap.String()),
+			trace.Int64Attribute("gas_price", int64(msg.GasPrice.Uint64())),
 			trace.StringAttribute("value", msg.Value.String()),
 		)
 	}
@@ -96,109 +79,18 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
 
 	state := ts.ParentState()
 
-	r := store.NewChainRand(sm.cs, ts.Cids())
+	r := store.NewChainRand(sm.cs, ts.Cids(), ts.Height())
 
 	return sm.CallRaw(ctx, msg, state, r, ts.Height())
 }
 
-func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, priorMsgs []types.ChainMsg, ts *types.TipSet) (*api.InvocResult, error) {
-	ctx, span := trace.StartSpan(ctx, "statemanager.CallWithGas")
-	defer span.End()
-
-	if ts == nil {
-		ts = sm.cs.GetHeaviestTipSet()
-	}
-
-	state, _, err := sm.TipSetState(ctx, ts)
-	if err != nil {
-		return nil, xerrors.Errorf("computing tipset state: %w", err)
-	}
-
-	r := store.NewChainRand(sm.cs, ts.Cids())
-
-	if span.IsRecordingEvents() {
-		span.AddAttributes(
-			trace.Int64Attribute("gas_limit", msg.GasLimit),
-			trace.StringAttribute("gas_feecap", msg.GasFeeCap.String()),
-			trace.StringAttribute("value", msg.Value.String()),
-		)
-	}
-
-	vmopt := &vm.VMOpts{
-		StateBase:      state,
-		Epoch:          ts.Height() + 1,
-		Rand:           r,
-		Bstore:         sm.cs.Blockstore(),
-		Syscalls:       sm.cs.VMSys(),
-		CircSupplyCalc: sm.GetCirculatingSupply,
-		NtwkVersion:    sm.GetNtwkVersion,
-		BaseFee:        ts.Blocks()[0].ParentBaseFee,
-	}
-	vmi, err := vm.NewVM(vmopt)
-	if err != nil {
-		return nil, xerrors.Errorf("failed to set up vm: %w", err)
-	}
-	for i, m := range priorMsgs {
-		_, err := vmi.ApplyMessage(ctx, m)
-		if err != nil {
-			return nil, xerrors.Errorf("applying prior message (%d, %s): %w", i, m.Cid(), err)
-		}
-	}
-
-	fromActor, err := vmi.StateTree().GetActor(msg.From)
-	if err != nil {
-		return nil, xerrors.Errorf("call raw get actor: %s", err)
-	}
-
-	msg.Nonce = fromActor.Nonce
-
-	fromKey, err := sm.ResolveToKeyAddress(ctx, msg.From, ts)
-	if err != nil {
-		return nil, xerrors.Errorf("could not resolve key: %w", err)
-	}
-
-	var msgApply types.ChainMsg
-
-	switch fromKey.Protocol() {
-	case address.BLS:
-		msgApply = msg
-	case address.SECP256K1:
-		msgApply = &types.SignedMessage{
-			Message: *msg,
-			Signature: crypto.Signature{
-				Type: crypto.SigTypeSecp256k1,
-				Data: make([]byte, 65),
-			},
-		}
-
-	}
-
-	ret, err := vmi.ApplyMessage(ctx, msgApply)
-	if err != nil {
-		return nil, xerrors.Errorf("apply message failed: %w", err)
-	}
-
-	var errs string
-	if ret.ActorErr != nil {
-		errs = ret.ActorErr.Error()
-	}
-
-	return &api.InvocResult{
-		Msg:            msg,
-		MsgRct:         &ret.MessageReceipt,
-		ExecutionTrace: ret.ExecutionTrace,
-		Error:          errs,
-		Duration:       ret.Duration,
-	}, nil
-}
-
 var errHaltExecution = fmt.Errorf("halt")
 
 func (sm *StateManager) Replay(ctx context.Context, ts *types.TipSet, mcid cid.Cid) (*types.Message, *vm.ApplyRet, error) {
 	var outm *types.Message
 	var outr *vm.ApplyRet
 
-	_, _, err := sm.computeTipSetState(ctx, ts, func(c cid.Cid, m *types.Message, ret *vm.ApplyRet) error {
+	_, _, err := sm.computeTipSetState(ctx, ts.Blocks(), func(c cid.Cid, m *types.Message, ret *vm.ApplyRet) error {
 		if c == mcid {
 			outm = m
 			outr = ret