Merge pull request #8457 from filecoin-project/release/v1.15.1

build: release: v1.15.1

.circleci/config.yml

@@ -2,7 +2,6 @@ version: 2.1
 orbs:
   go: gotest/tools@0.0.13
   aws-cli: circleci/aws-cli@1.3.2
-  packer: salaxander/packer@0.0.3
 
 executors:
   golang:
@@ -12,6 +11,16 @@ executors:
   ubuntu:
     docker:
       - image: ubuntu:20.04
+  packer:
+    description: |
+      The HashiCorp provided Packer container
+    parameters:
+      packer-version:
+        type: string
+        default: "1.8"
+    docker:
+      - image: hashicorp/packer:<< parameters.packer-version >>
+
 
 commands:
   install-deps:
@@ -75,6 +84,26 @@ commands:
           command: |
             git fetch --all
 
+  packer_build:
+    description: "Run a packer build"
+    parameters:
+      template:
+        description: |
+          The name of the packer template file
+        type: string
+        default: packer.json
+      args:
+        description: |
+          Arguments to pass to the packer build command
+        type: string
+        default: ""
+
+    steps:
+      - run:
+          name: "Run a packer build"
+          command: packer build << parameters.args >> << parameters.template >>
+          no_output_timeout: 30m
+
 jobs:
   mod-tidy-check:
     executor: golang
@@ -390,7 +419,7 @@ jobs:
 
   build-appimage:
     machine:
-      image: ubuntu-2004:202104-01
+      image: ubuntu-2004:202111-02
     steps:
       - checkout
       - attach_workspace:
@@ -398,6 +427,16 @@ jobs:
       - run:
           name: install appimage-builder
           command: |
+            # appimage-builder requires /dev/snd to exist. It creates containers during the testing phase
+            # that pass sound devices from the host to the testing container. (hard coded!)
+            # https://github.com/AppImageCrafters/appimage-builder/blob/master/appimagebuilder/modules/test/execution_test.py#L54
+            # Circleci doesn't provide a working sound device; this is enough to fake it.
+            if [ ! -e /dev/snd ]
+            then
+              sudo mkdir /dev/snd
+              sudo mknod /dev/snd/ControlC0 c 1 2
+            fi
+
             # docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html
             sudo apt update
             sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace
@@ -618,6 +657,11 @@ jobs:
         default: "latest"
         description: A comma-separated string containing docker image tags to build and push (default = latest)
 
+      target:
+        type: string
+        default: "lotus-all-in-one"
+        description: Docker target to build
+
     steps:
       - run:
           name: Confirm that environment variables are set
@@ -657,6 +701,7 @@ jobs:
 
             docker build \
               <<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
+              --target <<parameters.target>> \
               -f <<parameters.path>>/<<parameters.dockerfile>> \
               $docker_tag_args \
               <<parameters.path>>
@@ -669,42 +714,16 @@ jobs:
               docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
             done
 
-  publish-packer-mainnet:
-    description: build and push AWS IAM and DigitalOcean droplet.
+  publish-packer-snap:
+    description: build packer image with snap. mainnet only.
     executor:
-      name: packer/default
-      packer-version: 1.6.6
+      name: packer
     steps:
       - checkout
       - attach_workspace:
           at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
-  publish-packer-calibrationnet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
-  publish-packer-butterflynet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG"
+      - packer_build:
+          template: tools/packer/lotus-snap.pkr.hcl
   publish-dockerhub:
     description: publish to dockerhub
     machine:
@@ -785,6 +804,11 @@ workflows:
           suite: itest-deals_512mb
           target: "./itests/deals_512mb_test.go"
       
+      - test:
+          name: test-itest-deals_anycid
+          suite: itest-deals_anycid
+          target: "./itests/deals_anycid_test.go"
+      
       - test:
           name: test-itest-deals_concurrent
           suite: itest-deals_concurrent
@@ -850,6 +874,11 @@ workflows:
           suite: itest-get_messages_in_ts
           target: "./itests/get_messages_in_ts_test.go"
       
+      - test:
+          name: test-itest-mempool
+          suite: itest-mempool
+          target: "./itests/mempool_test.go"
+      
       - test:
           name: test-itest-multisig
           suite: itest-multisig
@@ -981,10 +1010,19 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-appimage:
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - publish:
           requires:
             - build-all
             - build-macos
+            - build-appimage
           filters:
             branches:
               ignore:
@@ -997,36 +1035,13 @@ workflows:
           path: .
           repo: lotus-dev
           tag: '${CIRCLE_SHA1:0:8}'
-      - publish-packer-mainnet:
-          requires:
-            - build-all
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-calibrationnet:
-          requires:
-            - build-ntwk-calibration
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-butterflynet:
-          requires:
-            - build-ntwk-butterfly
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+          target: lotus-all-in-one
+      - build-and-push-image:
+          dockerfile: Dockerfile.lotus
+          path: .
+          repo: lotus-test
+          tag: '${CIRCLE_SHA1:0:8}'
+          target: lotus-test
       - publish-snapcraft:
           name: publish-snapcraft-stable
           channel: stable
@@ -1063,3 +1078,13 @@ workflows:
       - publish-dockerhub:
           name: publish-dockerhub-nightly
           tag: nightly
+  monthly:
+    triggers:
+      - schedule:
+          cron: "0 0 1 * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - publish-packer-snap

.circleci/template.yml

@@ -2,7 +2,6 @@ version: 2.1
 orbs:
   go: gotest/tools@0.0.13
   aws-cli: circleci/aws-cli@1.3.2
-  packer: salaxander/packer@0.0.3
 
 executors:
   golang:
@@ -12,6 +11,16 @@ executors:
   ubuntu:
     docker:
       - image: ubuntu:20.04
+  packer:
+    description: |
+      The HashiCorp provided Packer container
+    parameters:
+      packer-version:
+        type: string
+        default: "1.8"
+    docker:
+      - image: hashicorp/packer:<< parameters.packer-version >>
+
 
 commands:
   install-deps:
@@ -75,6 +84,26 @@ commands:
           command: |
             git fetch --all
 
+  packer_build:
+    description: "Run a packer build"
+    parameters:
+      template:
+        description: |
+          The name of the packer template file
+        type: string
+        default: packer.json
+      args:
+        description: |
+          Arguments to pass to the packer build command
+        type: string
+        default: ""
+
+    steps:
+      - run:
+          name: "Run a packer build"
+          command: packer build << parameters.args >> << parameters.template >>
+          no_output_timeout: 30m
+
 jobs:
   mod-tidy-check:
     executor: golang
@@ -390,7 +419,7 @@ jobs:
 
   build-appimage:
     machine:
-      image: ubuntu-2004:202104-01
+      image: ubuntu-2004:202111-02
     steps:
       - checkout
       - attach_workspace:
@@ -398,6 +427,16 @@ jobs:
       - run:
           name: install appimage-builder
           command: |
+            # appimage-builder requires /dev/snd to exist. It creates containers during the testing phase
+            # that pass sound devices from the host to the testing container. (hard coded!)
+            # https://github.com/AppImageCrafters/appimage-builder/blob/master/appimagebuilder/modules/test/execution_test.py#L54
+            # Circleci doesn't provide a working sound device; this is enough to fake it.
+            if [ ! -e /dev/snd ]
+            then
+              sudo mkdir /dev/snd
+              sudo mknod /dev/snd/ControlC0 c 1 2
+            fi
+
             # docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html
             sudo apt update
             sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace
@@ -618,6 +657,11 @@ jobs:
         default: "latest"
         description: A comma-separated string containing docker image tags to build and push (default = latest)
 
+      target:
+        type: string
+        default: "lotus-all-in-one"
+        description: Docker target to build
+
     steps:
       - run:
           name: Confirm that environment variables are set
@@ -657,6 +701,7 @@ jobs:
 
             docker build \
               <<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
+              --target <<parameters.target>> \
               -f <<parameters.path>>/<<parameters.dockerfile>> \
               $docker_tag_args \
               <<parameters.path>>
@@ -669,42 +714,16 @@ jobs:
               docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
             done
 
-  publish-packer-mainnet:
-    description: build and push AWS IAM and DigitalOcean droplet.
+  publish-packer-snap:
+    description: build packer image with snap. mainnet only.
     executor:
-      name: packer/default
-      packer-version: 1.6.6
+      name: packer
     steps:
       - checkout
       - attach_workspace:
           at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
-  publish-packer-calibrationnet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
-  publish-packer-butterflynet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG"
+      - packer_build:
+          template: tools/packer/lotus-snap.pkr.hcl
   publish-dockerhub:
     description: publish to dockerhub
     machine:
@@ -816,10 +835,19 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-appimage:
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - publish:
           requires:
             - build-all
             - build-macos
+            - build-appimage
           filters:
             branches:
               ignore:
@@ -832,36 +860,13 @@ workflows:
           path: .
           repo: lotus-dev
           tag: '${CIRCLE_SHA1:0:8}'
-      - publish-packer-mainnet:
-          requires:
-            - build-all
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-calibrationnet:
-          requires:
-            - build-ntwk-calibration
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-butterflynet:
-          requires:
-            - build-ntwk-butterfly
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+          target: lotus-all-in-one
+      - build-and-push-image:
+          dockerfile: Dockerfile.lotus
+          path: .
+          repo: lotus-test
+          tag: '${CIRCLE_SHA1:0:8}'
+          target: lotus-test
       - publish-snapcraft:
           name: publish-snapcraft-stable
           channel: stable
@@ -898,3 +903,13 @@ workflows:
       - publish-dockerhub:
           name: publish-dockerhub-nightly
           tag: nightly
+  monthly:
+    triggers:
+      - schedule:
+          cron: "0 0 1 * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - publish-packer-snap

.github/pull_request_template.md

@@ -12,10 +12,10 @@
 
 Before you mark the PR ready for review, please make sure that:
 - [ ] All commits have a clear commit message.
-- [ ] The PR title is in the form of `<PR type>: <area>: <change being made>`
+- [ ] The PR title is in the form of of `<PR type>: <area>: <change being made>`
     - example: ` fix: mempool: Introduce a cache for valid signatures`
-    - `PR type`: _fix_, _feat_, _INTERFACE BREAKING CHANGE_, _CONSENSUS BREAKING_, _build_, _chore_, _ci_, _docs_, _misc_,_perf_, _refactor_, _revert_, _style_, _test_
-    - `area`: _api_, _chain_, _state_, _vm_, _data transfer_, _market_, _mempool_, _message_, _block production_, _multisig_, _networking_, _paychan_, _proving_, _sealing_, _wallet_
+    - `PR type`: _fix_, _feat_, _INTERFACE BREAKING CHANGE_, _CONSENSUS BREAKING_, _build_, _chore_, _ci_, _docs_,_perf_, _refactor_, _revert_, _style_, _test_
+    - `area`: _api_, _chain_, _state_, _vm_, _data transfer_, _market_, _mempool_, _message_, _block production_, _multisig_, _networking_, _paychan_, _proving_, _sealing_, _wallet_, _deps_
 - [ ] This PR has tests for new functionality or change in behaviour
 - [ ] If new user-facing features are introduced, clear usage guidelines and / or documentation updates should be included in https://lotus.filecoin.io or [Discussion Tutorials.](https://github.com/filecoin-project/lotus/discussions/categories/tutorials)
 - [ ] CI is green

.gitignore

@@ -40,6 +40,7 @@ build/paramfetch.sh
 /bundle
 /darwin
 /linux
+*.snap
 
 *-fuzz.zip
 /chain/types/work_msg/

AppImageBuilder.yml

@@ -49,23 +49,23 @@ AppDir:
     fedora:
       image: appimagecrafters/tests-env:fedora-30
       command: ./AppRun
-      use_host_x: true
+      use_host_x: false
     debian:
       image: appimagecrafters/tests-env:debian-stable
       command: ./AppRun
-      use_host_x: true
+      use_host_x: false
     arch:
       image: appimagecrafters/tests-env:archlinux-latest
       command: ./AppRun
-      use_host_x: true
+      use_host_x: false
     centos:
       image: appimagecrafters/tests-env:centos-7
       command: ./AppRun
-      use_host_x: true
+      use_host_x: false
     ubuntu:
       image: appimagecrafters/tests-env:ubuntu-xenial
       command: ./AppRun
-      use_host_x: true
+      use_host_x: false
 AppImage:
   arch: x86_64
   update-information: guess

CHANGELOG.md

@@ -1,12 +1,265 @@
 # Lotus changelog
 
+# 1.15.1 / 2022-04-07
+
+This is a *HIGHLY recommended* feature release v1.15.1, especially for node operators and storage providers who want to be a part of the content addressing network of Filecoin and IPFS.
+This feature release introduces Index Provider, GraphSync v2, and many other latest functionalities, improvements and bug fixes. More importantly, node operator can now enable the FVM(experimental) to sync mainnet!!
+
+## Highlights
+
+### 🔥🔥🔥 FVM (Experimental) 🔥🔥🔥
+- feat: fvm: FVM integration  ([filecoin-project/lotus#8332](https://github.com/filecoin-project/lotus/pull/8332))
+The lotus team is excited to announce the launch of experimental non-programmable FVM on mainnet. By enabling `"LOTUS_USE_FVM_EXPERIMENTAL=1` envvar, the lotus daemon will be running the [WASM-compiled built-in actors](https://github.com/filecoin-project/builtin-actors) that is compatible with the existing chain(Network v15 OhSnap). If you are trying it out and having any questions or feedbacks, please leave a comment [here](https://github.com/filecoin-project/lotus/discussions/8334)!
+  - chore: FVM: log when fvm is used([filecoin-project/lotus#8363](https://github.com/filecoin-project/lotus/pull/8363))
+  - chore: ffi: the latest fvm release([filecoin-project/lotus#8382](https://github.com/filecoin-project/lotus/pull/8382))
+  
+### 🌟🌟🌟 Index Provider (Production Ready!) 🌟🌟🌟
+- feat: markets: Integrate index ingest protocol and retrieve by any CID ([filecoin-project/lotus#7313](https://github.com/filecoin-project/lotus/pull/7313))
+
+More and more useful data is being stored on Filecoin via deals made by clients to Storage Providers. The goal is that this content is discoverable when people need them. To achieve that goal, one of the projects [the Bedrock team](https://www.notion.so/pl-strflt/Bedrock-2e956d5d8143432080a1d84435ccf0ff) is working on is building an Indexer Ecosystem, a project that's focus on content addressing on Filecoin, then potentially have interoperability with IPFS in the future and eventually serve the retrieval market. The Indexer Ecosystem high level architecture overview diagram can be found [here](https://github.com/filecoin-project/storetheindex/blob/main/doc/indexer_ecosys.png) and a detailed write up about can be found [here](https://www.notion.so/pl-strflt/Introducing-Indexer-to-SP-90bf296794174a8281c121d4ce6747a0).  
+
+That being said, with this release, lotus Storage Providers can easily become an Index Provider and serve the Indexer Content Addressing System. Index Providers generate advertisements from the deals made by a storage provider and announces the data to the indexer nodes for further processing:
+- To learn more about *what is an Index Provider and how to be an Index Provider*, read it [here](https://lotus.filecoin.io/storage-providers/operate/index-provider/) in lotus docuementation.
+- An [one-off migration](https://lotus.filecoin.io/storage-providers/operate/index-provider/#first-time-migration) is needed in order for a Storage Provider to become an Index Provider and announce the proper formatted index. It's *highly recommended* for all Index Provider to do a [force bulk initialization](https://lotus.filecoin.io/storage-providers/configure/dagstore/#forcing-bulk-initialization) to enable index announcement on all existing deals.
+  - Note that the Initialization places IO workload on your storage system. SP should set a proper `concurrency` based on your hardware or can stop/start initialization at their wish/convenience as proving deadlines approach and elapse, to avoid IOPS starvation or competition with window PoSt.
+- After the first one-time migration, being an Index Provider barely puts any extra usage on SP's market system.  
+  - You can find the testing result by SPX fellows [here](https://github.com/filecoin-project/lotus/discussions/8087).
+ 
+We recommend all Storage Providers that are serving deals in the Filecoin network to become a Index Provider, make the data you are storing discoverable for the retrieval market and retrieval clients! 
+  - If you have any questions about becoming an index provider, or the indexer system in general, leave a comment [here](https://github.com/filecoin-project/lotus/discussions/8341).
+  - Follow the indexer project at https://github.com/filecoin-project/go-indexer-core.
+  - If you have any feature request or bug reports of running an index provider, create an issue in https://github.com/filecoin-project/index-provider.
+  - You may also join the #storetheindex channel in the Filecoin Slack to engage with the team & the community!
+
+### ❗️❗️❗️ Dag Migration For New CAR index format in DagStore ❗️❗️❗️
+The index provider leverages the latest CARv2 indexing format `MultihashIndexSorted`, which stores the multihash code as well as the digest of all CIDs in a CAR file. Thus, all Storage Providers SHOULD perform an one-off DAG mirgation to regenerate DagStore CARv2 indices. You have to do it to become an index provider, failing to do so may also impact your future deal making.
+Follow the instruction [here](https://lotus.filecoin.io/storage-providers/operate/index-provider/) to perform the migration. 
+
+## New Features
+- feat: sealing: Sector upgrade queue ([filecoin-project/lotus#8333](https://github.com/filecoin-project/lotus/pull/8333))
+  - see more details in docs: [here](https://lotus.filecoin.io/storage-providers/operate/snap-deals/#snap-deal-queue)
+- feat: market utils: Support unixfsnode in TraverseDag ([filecoin-project/lotus#8168](https://github.com/filecoin-project/lotus/pull/8168))
+- feat: config: enable indexer providing by default ([filecoin-project/lotus#8314](https://github.com/filecoin-project/lotus/pull/8314))
+- feat: api: Make ClientCalcCommP multithreaded ([filecoin-project/lotus#8276](https://github.com/filecoin-project/lotus/pull/8276))
+- feat: config: Persistent subsystem log level config ([filecoin-project/lotus#8283](https://github.com/filecoin-project/lotus/pull/8283))
+- feat: shed: blockstore/vlog to car export cmds ([filecoin-project/lotus#8265](https://github.com/filecoin-project/lotus/pull/8265))
+- feat: shed: ItestD ([filecoin-project/lotus#8290](https://github.com/filecoin-project/lotus/pull/8290))
+- feat: Make add piece idempotent ([filecoin-project/lotus#8160](https://github.com/filecoin-project/lotus/pull/8160))
+- feat: paychmgr: Support paych funding (a.k.a. fast paid retrieval) ([filecoin-project/lotus#7883](https://github.com/filecoin-project/lotus/pull/7883))
+- feat: ci: packer snap ([filecoin-project/lotus#7819](https://github.com/filecoin-project/lotus/pull/7819))
+- feat: #6147: Include worker name in sealing errors ([filecoin-project/lotus#7844](https://github.com/filecoin-project/lotus/pull/7844))
+- Feat: cli: Remove verified data cap ([filecoin-project/lotus#8175](https://github.com/filecoin-project/lotus/pull/8175))
+- feat: gateway: add MsigGetVestingSchedule to gateway api ([filecoin-project/lotus#8104](https://github.com/filecoin-project/lotus/pull/8104))
+- feat: itests: add itests ensemble mocknet getter ([filecoin-project/lotus#8157](https://github.com/filecoin-project/lotus/pull/8157))
+- feat: lotus-miner sectors list --initial-pledge ([filecoin-project/lotus#8098](https://github.com/filecoin-project/lotus/pull/8098))
+- Resource Manager Metrics ([filecoin-project/lotus#8089](https://github.com/filecoin-project/lotus/pull/8089))
+- feat: cli: set current network version from params ([filecoin-project/lotus#8111](https://github.com/filecoin-project/lotus/pull/8111))
+- feat: Snapdeals support in `storage find` CLI ([filecoin-project/lotus#8130](https://github.com/filecoin-project/lotus/pull/8130))
+
+## Improvements
+- improve resource manager integration ([filecoin-project/lotus#8318](https://github.com/filecoin-project/lotus/pull/8318))
+- add check manual-stateless-deal with interactive deal making ([filecoin-project/lotus#7560](https://github.com/filecoin-project/lotus/pull/7560))
+- test: cli: adding wallet tests ([filecoin-project/lotus#8079](https://github.com/filecoin-project/lotus/pull/8079))
+- test: chain: unit tests for the syncer & sync manager ([filecoin-project/lotus#8072](https://github.com/filecoin-project/lotus/pull/8072))
+- test: cli: unit tests for sync related commands ([filecoin-project/lotus#8080](https://github.com/filecoin-project/lotus/pull/8080))
+- misc: wallet: wallet tests with annotations for system test matrix ([filecoin-project/lotus#7928](https://github.com/filecoin-project/lotus/pull/7928))
+- test: Cli: add mempool tests ([filecoin-project/lotus#8162](https://github.com/filecoin-project/lotus/pull/8162))
+- add a state-tree diff command to lotus shed ([filecoin-project/lotus#8081](https://github.com/filecoin-project/lotus/pull/8081))
+- test: mempool: Add unit and integration tests ([filecoin-project/lotus#8017](https://github.com/filecoin-project/lotus/pull/8017))
+- splistore cold object reification redux ([filecoin-project/lotus#8029](https://github.com/filecoin-project/lotus/pull/8029))
+- test: cli: chain category unit tests ([filecoin-project/lotus#8048](https://github.com/filecoin-project/lotus/pull/8048))
+- feat: config: Move MakeNewSectorForDeals config into the Sealing section([filecoin-project/lotus#8382](https://github.com/filecoin-project/lotus/pull/8382))
+
+## Bug Fixes
+- fix: FVM: add finality check for consensus faults #8452
+- fix: market: Reuse the market PubSub in index provider #8451
+- fix: market: set all index provider options based on lotus config #8444  
+- fix: sealing: Fix PR1 worker selection (#8421)
+- fix: miner: dead loop on removing sector (#8421)
+- fix: sealing: Remove sector copies from workers after snapdeals ([filecoin-project/lotus#8331](https://github.com/filecoin-project/lotus/pull/8331))
+- fix: storagefsm: Fix error loop on bad event ([filecoin-project/lotus#8339](https://github.com/filecoin-project/lotus/pull/8339))
+- fix: sealing: FinalizeSector doesn't need sealed replica access ([filecoin-project/lotus#8339](https://github.com/filecoin-project/lotus/pull/8339))
+- fix: sealing: always do cooldown in handleSubmitReplicaUpdateFailed ([filecoin-project/lotus#8353](https://github.com/filecoin-project/lotus/pull/8353))
+- fix: storage cli: Output primary sector status correctly ([filecoin-project/lotus#8320](https://github.com/filecoin-project/lotus/pull/8320))
+- fix: sealing fsm: Handle inputLk correctly ([filecoin-project/lotus#8291](https://github.com/filecoin-project/lotus/pull/8291))
+- fix: piece provider: Don't log CIDs as binary ([filecoin-project/lotus#8287](https://github.com/filecoin-project/lotus/pull/8287))
+- fix:sealing:Log instead of error normal shutdown of state machine ([filecoin-project/lotus#8232](https://github.com/filecoin-project/lotus/pull/8232))
+- fix:sealing:Handle finalize replica update failures in fsm ([filecoin-project/lotus#8229](https://github.com/filecoin-project/lotus/pull/8229))
+- ci: appimage: re-install appimage CI ([filecoin-project/lotus#7943](https://github.com/filecoin-project/lotus/pull/7943))
+- fix: sealing: PRU insufficient collateral  ([filecoin-project/lotus#8219](https://github.com/filecoin-project/lotus/pull/8219))
+- fix: shed: diff command ([filecoin-project/lotus#8202](https://github.com/filecoin-project/lotus/pull/8202))
+- Make `--lite` option visible in the lotus daemon help text  ([filecoin-project/lotus#8207](https://github.com/filecoin-project/lotus/pull/8207))
+- fix:sealing:Less verbose sector manager logging ([filecoin-project/lotus#8213](https://github.com/filecoin-project/lotus/pull/8213))
+- avoid panic ([filecoin-project/lotus#8205](https://github.com/filecoin-project/lotus/pull/8205))
+- A package is vulnerable to Exposure of Sensitive Information  ([filecoin-project/lotus#8204](https://github.com/filecoin-project/lotus/pull/8204))
+- fix: sealing: add flag usage ([filecoin-project/lotus#8190](https://github.com/filecoin-project/lotus/pull/8190))
+- Fix the epoch used for gas in the message pool & validation ([filecoin-project/lotus#8163](https://github.com/filecoin-project/lotus/pull/8163))
+- fix:sealing:really-do-it flag for abort upgrade ([filecoin-project/lotus#8181](https://github.com/filecoin-project/lotus/pull/8181))
+- fix:proving:post check sector handles snap deals replica faults ([filecoin-project/lotus#8177](https://github.com/filecoin-project/lotus/pull/8177))
+- fix: client: calculate commps for pieces bigger than 32GB ([filecoin-project/lotus#8179](https://github.com/filecoin-project/lotus/pull/8179))
+- fix:cli:Continue instead of return error if no valid value is filled ([filecoin-project/lotus#8131](https://github.com/filecoin-project/lotus/pull/8131))
+- fix: limit reification sizes ([filecoin-project/lotus#8149](https://github.com/filecoin-project/lotus/pull/8149))
+- fix: state: Allow lotus-miner info to complete without admin permission ([filecoin-project/lotus#8057](https://github.com/filecoin-project/lotus/pull/8057))
+- fix:paychan:deflake integration test ([filecoin-project/lotus#8088](https://github.com/filecoin-project/lotus/pull/8088))
+- fix: worker: allow enable/disabling ReplicaUpdate tasks ([filecoin-project/lotus#8090](https://github.com/filecoin-project/lotus/pull/8090))
+- don't fail reification on missing references ([filecoin-project/lotus#8128](https://github.com/filecoin-project/lotus/pull/8128))
+- sealer: fix error message ([filecoin-project/lotus#8121](https://github.com/filecoin-project/lotus/pull/8121))
+- don't track peer ids in rcmgr metrics ([filecoin-project/lotus#8099](https://github.com/filecoin-project/lotus/pull/8099))
+- temporarily disable reification ([filecoin-project/lotus#8132](https://github.com/filecoin-project/lotus/pull/8132))
+- [Describe]: when excute cmd "lotus-bench sealing" without "benchmark-… ([filecoin-project/lotus#8173](https://github.com/filecoin-project/lotus/pull/8173))
+
+## Dependency Updates
+- deps: update go-libp2p and go-libp2p-resource-manager ([filecoin-project/lotus#8289](https://github.com/filecoin-project/lotus/pull/8289))
+- feat(deps): update to graphsync v0.13.0 with 2.0 protocol ([filecoin-project/lotus#8273](https://github.com/filecoin-project/lotus/pull/8273))
+- dep: actor: get v7 ([filecoin-project/lotus#8194](https://github.com/filecoin-project/lotus/pull/8194))
+- deps: update go-libp2p to v0.18 release  ([filecoin-project/lotus#8355](https://github.com/filecoin-project/lotus/pull/8355))
+- github.com/filecoin-project/go-data-transfer (v1.14.1 -> v1.15.0):
+- github.com/filecoin-project/go-fil-markets (v1.19.2 -> v1.20.1):
+- deps: update go-libp2p to v0.18.0-rc5 ([filecoin-project/lotus#8169](https://github.com/filecoin-project/lotus/pull/8169))
+
+## Others
+- chore: build: backport releases ([filecoin-project/lotus#8192](https://github.com/filecoin-project/lotus/pull/8192))
+- feat: build: bump the version to v1.15.1-dev ([filecoin-project/lotus#8073](https://github.com/filecoin-project/lotus/pull/8073))
+- makefile: add make jen ([filecoin-project/lotus#8122](https://github.com/filecoin-project/lotus/pull/8122))
+- chore: Merge releases into master ([filecoin-project/lotus#8156](https://github.com/filecoin-project/lotus/pull/8156))
+- chore: ci: disable publish-packer #8451
+
+# 1.15.0 / 2022-03-09
+
+This is an optional release with retrieval improvements(client side), SP ux with unsealing, snap deals and regular deal making and many other new features, improvements and bug fixes.
+
+## Highlights
+- feat:sealing: StartEpochSealingBuffer triggers packing on time([filecoin-project/lotus#7905](https://github.com/filecoin-project/lotus/pull/7905))
+  - use the `StartEpochSealingBuffer` configuration variable as a way to enforce that sectors are packed for sealing / updating no matter how many deals they have if the nearest deal start date is close enough to the present.
+- feat: #6017 market: retrieval ask CLI command ([filecoin-project/lotus#7814](https://github.com/filecoin-project/lotus/pull/7814))
+- feat(graphsync): allow setting of per-peer incoming requests for miners ([filecoin-project/lotus#7578](https://github.com/filecoin-project/lotus/pull/7578))
+  - by setting `SimultaneousTransfersForStoragePerClient` in deal making configuration.
+- Make retrieval even faster ([filecoin-project/lotus#7746](https://github.com/filecoin-project/lotus/pull/7746))
+- feat: #7747 sealing: Adding conf variable for capping number of concurrent unsealing jobs (#7884) ([filecoin-project/lotus#7884](https://github.com/filecoin-project/lotus/pull/7884))
+  - by setting `MaxConcurrentUnseals` in `DAGStoreConfig`
+
+## New Features
+- feat: mpool: Cache state nonces ([filecoin-project/lotus#8005](https://github.com/filecoin-project/lotus/pull/8005))
+- chore: build: make the OhSnap epoch configurable by an envvar for devnets ([filecoin-project/lotus#7995](https://github.com/filecoin-project/lotus/pull/7995))
+- Shed: Add a util to send a batch of messages ([filecoin-project/lotus#7667](https://github.com/filecoin-project/lotus/pull/7667))
+- Add api for transfer diagnostics ([filecoin-project/lotus#7759](https://github.com/filecoin-project/lotus/pull/7759))
+- Shed: Add a util to list terminated deals ([filecoin-project/lotus#7774](https://github.com/filecoin-project/lotus/pull/7774))
+- Expose EnableGasTracing as an env_var ([filecoin-project/lotus#7750](https://github.com/filecoin-project/lotus/pull/7750))
+- Command to list active sector locks ([filecoin-project/lotus#7735](https://github.com/filecoin-project/lotus/pull/7735))
+- Initial switch to OpenTelemetry ([filecoin-project/lotus#7725](https://github.com/filecoin-project/lotus/pull/7725))
+
+## Improvements
+- splitstore sortless compaction ([filecoin-project/lotus#8008](https://github.com/filecoin-project/lotus/pull/8008))
+- perf: chain: Make drand logs in daemon less noisy (#7955) ([filecoin-project/lotus#7955](https://github.com/filecoin-project/lotus/pull/7955))
+- chore: shed: storage stats 2.0 ([filecoin-project/lotus#7941](https://github.com/filecoin-project/lotus/pull/7941))
+- misc: api: Annotate lotus tests according to listed behaviors ([filecoin-project/lotus#7835](https://github.com/filecoin-project/lotus/pull/7835))
+- some basic splitstore refactors ([filecoin-project/lotus#7999](https://github.com/filecoin-project/lotus/pull/7999))
+- chore: sealer: quieten a log ([filecoin-project/lotus#7998](https://github.com/filecoin-project/lotus/pull/7998))
+- tvx: supply network version when extracting messages. ([filecoin-project/lotus#7996](https://github.com/filecoin-project/lotus/pull/7996))
+- chore: remove inaccurate comment in sealtasks ([filecoin-project/lotus#7977](https://github.com/filecoin-project/lotus/pull/7977))
+- Refactor: VM: Remove the NetworkVersionGetter ([filecoin-project/lotus#7818](https://github.com/filecoin-project/lotus/pull/7818))
+- refactor: state: Move randomness versioning out of the VM ([filecoin-project/lotus#7816](https://github.com/filecoin-project/lotus/pull/7816))
+- updating to new datastore/blockstore code with contexts ([filecoin-project/lotus#7646](https://github.com/filecoin-project/lotus/pull/7646))
+- Mempool msg selection should respect block message limits ([filecoin-project/lotus#7321](https://github.com/filecoin-project/lotus/pull/7321))
+- Minor improvement for OpenTelemetry ([filecoin-project/lotus#7760](https://github.com/filecoin-project/lotus/pull/7760))
+- Sort lotus-miner retrieval-deals by dealId ([filecoin-project/lotus#7749](https://github.com/filecoin-project/lotus/pull/7749))
+- dagstore pieceReader: Always read full in ReadAt ([filecoin-project/lotus#7737](https://github.com/filecoin-project/lotus/pull/7737))
+
+## Bug Fixes
+- fix: sealing: Stop recovery attempts after fault ([filecoin-project/lotus#8014](https://github.com/filecoin-project/lotus/pull/8014))
+- fix:snap: pay for the collateral difference needed if the miner available balance is insufficient ([filecoin-project/lotus#8234](https://github.com/filecoin-project/lotus/pull/8234))
+- sealer: fix error message ([filecoin-project/lotus#8136](https://github.com/filecoin-project/lotus/pull/8136))
+- typo in  variable name ([filecoin-project/lotus#8134](https://github.com/filecoin-project/lotus/pull/8134))
+- fix: sealer: allow enable/disabling ReplicaUpdate tasks ([filecoin-project/lotus#8093](https://github.com/filecoin-project/lotus/pull/8093))
+- chore: chain: fix log ([filecoin-project/lotus#7993](https://github.com/filecoin-project/lotus/pull/7993))
+- Fix: chain: create a new VM for each epoch ([filecoin-project/lotus#7966](https://github.com/filecoin-project/lotus/pull/7966))
+- fix: doc generation struct slice example value ([filecoin-project/lotus#7851](https://github.com/filecoin-project/lotus/pull/7851))
+- fix: returned error not be accept correctly ([filecoin-project/lotus#7852](https://github.com/filecoin-project/lotus/pull/7852))
+- fix: #7577 markets: When retrying Add Piece, first seek to start of reader ([filecoin-project/lotus#7812](https://github.com/filecoin-project/lotus/pull/7812))
+- misc: n/a sealing: Fix grammatical error in a log warning message ([filecoin-project/lotus#7831](https://github.com/filecoin-project/lotus/pull/7831))
+- sectors update-state checks if sector exists before changing its state ([filecoin-project/lotus#7762](https://github.com/filecoin-project/lotus/pull/7762))
+- SplitStore: supress compaction near upgrades ([filecoin-project/lotus#7734](https://github.com/filecoin-project/lotus/pull/7734))
+
+## Dependency Updates
+- github.com/filecoin-project/go-commp-utils (v0.1.2 -> v0.1.3):
+- github.com/filecoin-project/dagstore (v0.4.3 -> v0.4.4):
+- github.com/filecoin-project/go-fil-markets (v1.13.4 -> v1.19.2):
+- github.com/filecoin-project/go-statestore (v0.1.1 -> v0.2.0):
+- github.com/filecoin-project/go-storedcounter (v0.0.0-20200421200003-1c99c62e8a5b -> v0.1.0):
+- github.com/filecoin-project/specs-actors/v2 (v2.3.5 -> v2.3.6):
+  - feat(deps): update markets stack ([filecoin-project/lotus#7959](https://github.com/filecoin-project/lotus/pull/7959))
+  - Use go-libp2p-connmgr v0.3.1 ([filecoin-project/lotus#7957](https://github.com/filecoin-project/lotus/pull/7957))
+  - dep/fix 7701 Dependency: update to ipld-legacy to v0.1.1 ([filecoin-project/lotus#7751](https://github.com/filecoin-project/lotus/pull/7751))
+
+## Others
+- chore: backport: release ([filecoin-project/lotus#8245](https://github.com/filecoin-project/lotus/pull/8245))
+- Lotus release v1.15.0-rc3 ([filecoin-project/lotus#8236](https://github.com/filecoin-project/lotus/pull/8236))
+- Lotus release v1.15.0-rc2 ([filecoin-project/lotus#8211](https://github.com/filecoin-project/lotus/pull/8211))
+- Merge branch 'releases' into release/v1.15.0
+- chore: build: backport releases ([filecoin-project/lotus#8193](https://github.com/filecoin-project/lotus/pull/8193))
+- Merge branch 'releases' into release/v1.15.0
+- bump the version to v1.15.0-rc1
+- chore: build: v1.14.0 -> master ([filecoin-project/lotus#8053](https://github.com/filecoin-project/lotus/pull/8053))
+- chore: merge release/v1.14.0 PRs into master ([filecoin-project/lotus#7979](https://github.com/filecoin-project/lotus/pull/7979))
+- chore: update PR template ([filecoin-project/lotus#7918](https://github.com/filecoin-project/lotus/pull/7918))
+- build: release: bump master version to v1.15.0-dev ([filecoin-project/lotus#7922](https://github.com/filecoin-project/lotus/pull/7922))
+- misc: docs: remove issue number from the pr title ([filecoin-project/lotus#7902](https://github.com/filecoin-project/lotus/pull/7902))
+- Snapcraft grade no develgrade ([filecoin-project/lotus#7802](https://github.com/filecoin-project/lotus/pull/7802))
+- chore: create pull_request_template.md ([filecoin-project/lotus#7726](https://github.com/filecoin-project/lotus/pull/7726))
+- Disable appimage ([filecoin-project/lotus#7707](https://github.com/filecoin-project/lotus/pull/7707))
+
+## Contributors
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| @arajasek | 73 | +7232/-2778 | 386 |
+| @zenground0 | 27 | +5604/-1049 | 219 |
+| @vyzo | 118 | +4356/-1470 | 253 |
+| @zl | 1 | +3725/-309 | 8 |
+| @dirkmc | 7 | +1392/-1110 | 61 |
+| arajasek | 37 | +221/-1329 | 90 |
+| @magik6k | 33 | +1138/-336 | 101 |
+| @whyrusleeping | 2 | +483/-585 | 28 |
+| Darko Brdareski | 14 | +725/-276 | 154 |
+| @rvagg | 2 | +43/-947 | 10 |
+| @hannahhoward | 5 | +436/-335 | 31 |
+| @hannahhoward | 12 | +507/-133 | 37 |
+| @jennijuju | 27 | +333/-178 | 54 |
+| @TheMenko | 8 | +237/-179 | 17 |
+| c r | 2 | +227/-45 | 12 |
+| @dirkmck | 12 | +188/-40 | 27 |
+| @ribasushi | 3 | +128/-62 | 3 |
+| @raulk | 6 | +128/-49 | 9 |
+| @Whyrusleeping | 1 | +76/-70 | 8 |
+| @Stebalien | 1 | +55/-37 | 1 |
+| @jennijuju | 11 | +29/-16 | 11 |
+| @aarshkshah1992 | 1 | +23/-19 | 5 |
+| @travisperson | 1 | +0/-18 | 2 |
+| @gstuart | 3 | +12/-1 | 3 |
+| @coryschwartz | 4 | +5/-6 | 4 |
+| @pefish | 1 | +4/-3 | 1 |
+| @Kubuxu | 1 | +5/-2 | 2 |
+| Colin Kennedy | 1 | +4/-2 | 1 |
+| Rob Quist | 1 | +2/-2 | 1 |
+| @shotcollin | 1 | +1/-1 | 1 |
+
+
+# 1.14.4 / 2022-03-03
+
+This is a *highly recommended* optional release for storage providers that are doing snap deals. This fix the bug
+that causes some snap deal sectors are stuck in `FinalizeReplicaUpdate`. In addition, SPs should be able to force
+update sectors status without getting blocked by `normal shutdown of state machine`.
+
+# v1.14.3 / 2022-02-28
+
+This is an **optional** release, that includes a fix to properly register the `--really-do-it` flag for abort-upgrade.
+
 # 1.14.2 / 2022-02-24
 
-This is an **optional** release of lotus, that's had a couple more improvements w.r.t Snap experience for storage providers in preparation of the[upcoming OhSnap upgrade](https://github.com/filecoin-project/community/discussions/74?sort=new#discussioncomment-1922550). 
+This is an **optional** release of lotus, that's had a couple more improvements w.r.t Snap experience for storage providers in preparation of the[upcoming OhSnap upgrade](https://github.com/filecoin-project/community/discussions/74?sort=new#discussioncomment-1922550).
 
 Note that the network is STILL scheduled to upgrade to v15 on March 1st at 2022-03-01T15:00:00Z. All node operators, including storage providers, must upgrade to at least Lotus v1.14.0 before that time. Storage providers must update their daemons, miners, and worker(s).
 
-Wanna know how to Snap your deal? Check [this](https://github.com/filecoin-project/lotus/discussions/8141) out! 
+Wanna know how to Snap your deal? Check [this](https://github.com/filecoin-project/lotus/discussions/8141) out!
 
 ## Bug Fixes
 - fix lotus-bench for sealing jobs (#8173)
@@ -15,8 +268,8 @@ Wanna know how to Snap your deal? Check [this](https://github.com/filecoin-proje
 - fix: sealing: missing file type (#8180)
 
 ## Others
-- Retract force-pushed v1.14.0 to work around stale gomod caches (#8159): We originally tagged v1.14.0 off the wrong 
-  commit and fixed that by a force push, in which is a really bad practise since it messes up the go mod. Therefore, 
+- Retract force-pushed v1.14.0 to work around stale gomod caches (#8159): We originally tagged v1.14.0 off the wrong
+  commit and fixed that by a force push, in which is a really bad practise since it messes up the go mod. Therefore,
   we want to retract it and users may use v1.14.1&^.
 
 ## Contributors
@@ -35,7 +288,7 @@ This is an **optional** release of lotus, that fixes the incorrect *comment* of
 
 # 1.14.0 / 2022-02-17
 
-This is a MANDATORY release of Lotus that introduces [Filecoin network v15, 
+This is a MANDATORY release of Lotus that introduces [Filecoin network v15,
 codenamed the OhSnap upgrade](https://github.com/filecoin-project/community/discussions/74?sort=new#discussioncomment-1922550).
 
 The network is scheduled to upgrade to v15 on March 1st at 2022-03-01T15:00:00Z. All node operators, including storage providers, must upgrade to this release (or a later release) before that time. Storage providers must update their daemons, miners, and worker(s).
@@ -52,7 +305,7 @@ It is recommended that storage providers download the new params before updating
 - Upgrade the Lotus daemon and miner **when the previous step is complete**
 
 All node operators, including storage providers, should be aware that a pre-migration will begin at 2022-03-01T13:30:00Z (90 minutes before the real upgrade). The pre-migration will take between 20 and 50 minutes, depending on hardware specs. During this time, expect slower block validation times, increased CPU and memory usage, and longer delays for API queries.
-  
+
 ## New Features and Changes
 - Integrate actor v7-rc1:
   - Integrate v7 actors ([#7617](https://github.com/filecoin-project/lotus/pull/7617))
@@ -78,7 +331,7 @@ All node operators, including storage providers, should be aware that a pre-migr
 - Fix: state: circsuypply calc around null blocks ([#7890](https://github.com/filecoin-project/lotus/pull/7890))
 - Mempool msg selection should respect block message limits ([#7321](https://github.com/filecoin-project/lotus/pull/7321))
   SplitStore: supress compaction near upgrades ([#7734](https://github.com/filecoin-project/lotus/pull/7734))
-  
+
 ## Others
 - chore: create pull_request_template.md ([#7726](https://github.com/filecoin-project/lotus/pull/7726))
 
@@ -101,17 +354,15 @@ All node operators, including storage providers, should be aware that a pre-migr
 | Travis Person | 1 | +2/-2 | 2 |
 | Rod Vagg | 1 | +2/-2 | 2 |
 
-
-
 # v1.13.2 /  2022-01-09
 
-Lotus v1.13.2 is a *highly recommended* feature release with remarkable retrieval improvements, new features like 
-worker management, schedule enhancements and so on. 
+Lotus v1.13.2 is a *highly recommended* feature release with remarkable retrieval improvements, new features like
+worker management, schedule enhancements and so on.
 
 ## Highlights
 - 🚀🚀🚀Improve retrieval deal experience
-  - Testing result with MinerX.3 shows the retrieval deal success rate has increased dramatically with faster transfer 
-    speed, you can join or follow along furthur performance testings [here](https://github.com/filecoin-project/lotus/discussions/7874). We recommend application developers to integrate with the new 
+  - Testing result with MinerX.3 shows the retrieval deal success rate has increased dramatically with faster transfer
+    speed, you can join or follow along furthur performance testings [here](https://github.com/filecoin-project/lotus/discussions/7874). We recommend application developers to integrate with the new
     retrieval APIs to provide a better client experience.
   - 🌟🌟🌟 Reduce retrieval Time-To-First-Byte over 100x ([#7693](https://github.com/filecoin-project/lotus/pull/7693))
     - This change makes most free, small retrievals sub-second
@@ -203,7 +454,7 @@ worker management, schedule enhancements and so on.
 | @jennijuju | 1 | +1/-1 | 1 |
 | @hunjixin | 1 | +1/-0 | 1 |
 
-    
+
 
 # v1.13.1 / 2021-11-26
 
@@ -286,32 +537,32 @@ Contributors
 | @hannahhoward | 1 | +3/-2 | 2 |
 | Marten Seemann | 1 | +3/-0 | 1 |
 | @ZenGround0 | 1 | +1/-1 | 1 |
- 
+
 
 # v1.13.0 / 2021-10-18
 
-Lotus v1.13.0 is a *highly recommended* feature release for all lotus users(i.e: storage providers, data brokers, application developers and so on) that supports the upcoming 
+Lotus v1.13.0 is a *highly recommended* feature release for all lotus users(i.e: storage providers, data brokers, application developers and so on) that supports the upcoming
 [Network v14 Chocolate upgrade](https://github.com/filecoin-project/lotus/discussions/7431).
 This feature release includes the latest functionalities and improvements, like data transfer rate-limiting for both storage and retrieval deals, proof v10 with CUDA support, etc. You can find more details in the Changelog below.
 
 ## Highlights
 - Enable separate storage and retrieval transfer limits ([filecoin-project/lotus#7405](https://github.com/filecoin-project/lotus/pull/7405))
-    - `SimultaneousTransfer` is now replaced by `SimultaneousTransfersForStorage` and `SimultaneousTransfersForRetrieval`, where users may set the amount of ongoing data transfer for storage and retrieval deals in parallel separately. The default value for both is set to 20. 
-    - If you are using the lotus client, these two configuration variables are under the `Client` section in `./lotus/config.toml`.
-    - If you are a service provider, these two configuration variables should be set under the `Dealmaking` section in `/.lotusminer/config.toml`.
+  - `SimultaneousTransfer` is now replaced by `SimultaneousTransfersForStorage` and `SimultaneousTransfersForRetrieval`, where users may set the amount of ongoing data transfer for storage and retrieval deals in parallel separately. The default value for both is set to 20.
+  - If you are using the lotus client, these two configuration variables are under the `Client` section in `./lotus/config.toml`.
+  - If you are a service provider, these two configuration variables should be set under the `Dealmaking` section in `/.lotusminer/config.toml`.
 - Update proofs to v10.0.0 ([filecoin-project/lotus#7420](https://github.com/filecoin-project/lotus/pull/7420))
-    - This version supports CUDA. To enable CUDA instead of openCL, build lotus with `FFI_USE_CUDA=1 FFI_BUILD_FROM_SOURCE=1 ...`.
-    - You can find additional Nvidia driver installation instructions written by MinerX fellows [here](https://github.com/filecoin-project/lotus/discussions/7443#discussioncomment-1425274) and perf improvements result on PC2/C2/WindowPoSt computation on different profiles [here](https://github.com/filecoin-project/lotus/discussions/7443), most people observe a 30-50% decrease in computation time.
+  - This version supports CUDA. To enable CUDA instead of openCL, build lotus with `FFI_USE_CUDA=1 FFI_BUILD_FROM_SOURCE=1 ...`.
+  - You can find additional Nvidia driver installation instructions written by MinerX fellows [here](https://github.com/filecoin-project/lotus/discussions/7443#discussioncomment-1425274) and perf improvements result on PC2/C2/WindowPoSt computation on different profiles [here](https://github.com/filecoin-project/lotus/discussions/7443), most people observe a 30-50% decrease in computation time.
 
 ## New Features
 - Feat/datamodel selector retrieval ([filecoin-project/lotus#6393](https://github.com/filecoin-project/lotus/pull/66393393))
-    - This introduces a new RetrievalOrder-struct field and a CLI option that takes a string representation as understood by [https://pkg.go.dev/github.com/ipld/go-ipld-selector-text-lite#SelectorSpecFromPath](https://pkg.go.dev/github.com/ipld/go-ipld-selector-text-lite#SelectorSpecFromPath). This allows for partial retrieval of any sub-DAG of a deal provided the user knows the exact low-level shape of the deal contents.
-        - For example, to retrieve the first entry of a UnixFS directory by executing, run `lotus client retrieve --miner f0XXXXX --datamodel-path-selector 'Links/0/Hash' bafyROOTCID ~/output`
+  - This introduces a new RetrievalOrder-struct field and a CLI option that takes a string representation as understood by [https://pkg.go.dev/github.com/ipld/go-ipld-selector-text-lite#SelectorSpecFromPath](https://pkg.go.dev/github.com/ipld/go-ipld-selector-text-lite#SelectorSpecFromPath). This allows for partial retrieval of any sub-DAG of a deal provided the user knows the exact low-level shape of the deal contents.
+    - For example, to retrieve the first entry of a UnixFS directory by executing, run `lotus client retrieve --miner f0XXXXX --datamodel-path-selector 'Links/0/Hash' bafyROOTCID ~/output`
 - Expose storage stats on the metrics endpoint ([filecoin-project/lotus#7418](https://github.com/filecoin-project/lotus/pull/7418))
 - feat: Catch panic to generate report and reraise ([filecoin-project/lotus#7341](https://github.com/filecoin-project/lotus/pull/7341))
-    - Set `LOTUS_PANIC_REPORT_PATH` and `LOTUS_PANIC_JOURNAL_LOOKBACK` to get reports generated when a panic occurs on your daemon miner or workers.
+  - Set `LOTUS_PANIC_REPORT_PATH` and `LOTUS_PANIC_JOURNAL_LOOKBACK` to get reports generated when a panic occurs on your daemon miner or workers.
 - Add envconfig docs to the config ([filecoin-project/lotus#7412](https://github.com/filecoin-project/lotus/pull/7412))
-    - You can now find supported env vars in [default-lotus-miner-config.toml](https://github.com/filecoin-project/lotus/blob/master/documentation/en/default-lotus-miner-config.toml).
+  - You can now find supported env vars in [default-lotus-miner-config.toml](https://github.com/filecoin-project/lotus/blob/master/documentation/en/default-lotus-miner-config.toml).
 - lotus shed: fr32 utils ([filecoin-project/lotus#7355](https://github.com/filecoin-project/lotus/pull/7355))
 - Miner CLI: Allow trying to change owners of any miner actor ([filecoin-project/lotus#7328](https://github.com/filecoin-project/lotus/pull/7328))
 - Add --unproven flag to the sectors list command ([filecoin-project/lotus#7308](https://github.com/filecoin-project/lotus/pull/7308))
@@ -328,7 +579,7 @@ This feature release includes the latest functionalities and improvements, like
 - Prep retrieval for selectors: no functional changes ([filecoin-project/lotus#7306](https://github.com/filecoin-project/lotus/pull/7306))
 - Seed: improve helptext ([filecoin-project/lotus#7304](https://github.com/filecoin-project/lotus/pull/7304))
 - Mempool: reduce size of sigValCache ([filecoin-project/lotus#7305](https://github.com/filecoin-project/lotus/pull/7305))
- - Stop indirectly depending on deprecated github.com/prometheus/common ([filecoin-project/lotus#7474](https://github.com/filecoin-project/lotus/pull/7474))
+- Stop indirectly depending on deprecated github.com/prometheus/common ([filecoin-project/lotus#7474](https://github.com/filecoin-project/lotus/pull/7474))
 
 ## Bug Fixes
 - StateSearchMsg: Correct usage of the allowReplaced flag ([filecoin-project/lotus#7450](https://github.com/filecoin-project/lotus/pull/7450))
@@ -383,7 +634,7 @@ This feature release includes the latest functionalities and improvements, like
 
 # v1.12.0 / 2021-10-12
 
-This is a mandatory release of Lotus that introduces [Filecoin Network v14](https://github.com/filecoin-project/community/discussions/74#discussioncomment-1398542), codenamed the Chocolate upgrade. The Filecoin mainnet will upgrade at epoch 1231620, on 2021-10-26T13:30:00Z. 
+This is a mandatory release of Lotus that introduces [Filecoin Network v14](https://github.com/filecoin-project/community/discussions/74#discussioncomment-1398542), codenamed the Chocolate upgrade. The Filecoin mainnet will upgrade at epoch 1231620, on 2021-10-26T13:30:00Z.
 
 The Chocolate upgrade introduces the following FIPs, delivered in [v6 actors](https://github.com/filecoin-project/specs-actors/releases/tag/v6.0.0)
 
@@ -397,7 +648,7 @@ The Chocolate upgrade introduces the following FIPs, delivered in [v6 actors](ht
 Note that this release is built on top of lotus v1.11.3. Enterprising users like storage providers, data brokers and others are recommended to use lotus v1.13.0 for latest new features, improvements and bug fixes.
 
 ## New Features and Changes
-- Implement and support [FIP-0024](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0024.md) BatchBalancer & BatchDiscount Post-HyperDrive Adjustment: 
+- Implement and support [FIP-0024](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0024.md) BatchBalancer & BatchDiscount Post-HyperDrive Adjustment:
   - Precommit batch balancer support/config ([filecoin-project/lotus#7410](https://github.com/filecoin-project/lotus/pull/7410))
     - Set `BatchPreCommitAboveBaseFee` to decide whether sending out a PreCommits in individual messages or in a batch.
     - The default value of `BatchPreCommitAboveBaseFee` and `AggregateAboveBaseFee` are now updated to 0.32nanoFIL.
@@ -414,17 +665,17 @@ Note that this release is built on top of lotus v1.11.3. Enterprising users like
 ## Dependency Updates
 - Add [v6 actors](https://github.com/filecoin-project/specs-actors/releases/tag/v6.0.0)
   - **Protocol changes**
-     - Multisig Approve only hashes when hash in params
-     - FIP 0020 WithdrawBalance methods return withdrawn value
-     - FIP 0021 Fix bug in power calculation when extending verified deals sectors
-     - FIP 0022 PublishStorageDeals drops errors in batch
-     - FIP 0024 BatchBalancer update and burn added to PreCommitBatch
-     - FIP 0026 Add FaultMaxAge extension
-     - Reduce calls to power and reward actors by passing values from power cron
-     - Defensive programming hardening power cron against programmer error
+    - Multisig Approve only hashes when hash in params
+    - FIP 0020 WithdrawBalance methods return withdrawn value
+    - FIP 0021 Fix bug in power calculation when extending verified deals sectors
+    - FIP 0022 PublishStorageDeals drops errors in batch
+    - FIP 0024 BatchBalancer update and burn added to PreCommitBatch
+    - FIP 0026 Add FaultMaxAge extension
+    - Reduce calls to power and reward actors by passing values from power cron
+    - Defensive programming hardening power cron against programmer error
   - **Implementation changes**
-     - Move to xerrors
-     - Improved logging: burn events are not logged with reasons and burned value.
+    - Move to xerrors
+    - Improved logging: burn events are not logged with reasons and burned value.
 - github.com/filecoin-project/go-state-types (v0.1.1-0.20210810190654-139e0e79e69e -> v0.1.1-0.20210915140513-d354ccf10379):
 
 ## Others
@@ -449,20 +700,20 @@ Note that this release is built on top of lotus v1.11.3. Enterprising users like
 
 # v1.11.3 / 2021-09-29
 
-lotus v1.11.3 is a feature release that's **highly recommended to ALL lotus users to upgrade**, including node 
-operators, storage providers and clients. It includes many improvements and bug fixes that result in perf 
+lotus v1.11.3 is a feature release that's **highly recommended to ALL lotus users to upgrade**, including node
+operators, storage providers and clients. It includes many improvements and bug fixes that result in perf
 improvements in different area, like deal making, sealing and so on.
 
 ## Highlights
 
 - 🌟🌟Introduce `MaxStagingDealsBytes - reject new deals if our staging deals area is full ([filecoin-project/lotus#7276](https://github.com/filecoin-project/lotus/pull/7276))
-    - Set `MaxStagingDealsBytes` under the [Dealmaking] section of the markets' subsystem's `config.toml` to reject new incoming deals when the `deal-staging` directory of market subsystem's repo gets too large. 
+  - Set `MaxStagingDealsBytes` under the [Dealmaking] section of the markets' subsystem's `config.toml` to reject new incoming deals when the `deal-staging` directory of market subsystem's repo gets too large.
 - 🌟🌟miner: Command to list/remove expired sectors locally ([filecoin-project/lotus#7140](https://github.com/filecoin-project/lotus/pull/7140))
-    - run `./lotus-miner sectors expired -h` for more details.
+  - run `./lotus-miner sectors expired -h` for more details.
 - 🚀update to ffi to update-bellperson-proofs-v9-0-2 ([filecoin-project/lotus#7369](https://github.com/filecoin-project/lotus/pull/7369))
-    - MinerX fellows(early testers of lotus releases) have reported faster WindowPoSt computation!
+  - MinerX fellows(early testers of lotus releases) have reported faster WindowPoSt computation!
 - 🌟dealpublisher: Fully validate deals before publishing ([filecoin-project/lotus#7234](https://github.com/filecoin-project/lotus/pull/7234))
-    - This excludes the expired deals before sending out a PSD message which reduces the chances of PSD message failure due to invalid deals. 
+  - This excludes the expired deals before sending out a PSD message which reduces the chances of PSD message failure due to invalid deals.
 - 🌟Simple alert system; FD limit alerts ([filecoin-project/lotus#7108](https://github.com/filecoin-project/lotus/pull/7108))
 
 ## New Features
@@ -533,7 +784,7 @@ improvements in different area, like deal making, sealing and so on.
 - Turn off patch ([filecoin-project/lotus#7172](https://github.com/filecoin-project/lotus/pull/7172))
 - test: disable flaky TestSimultaneousTransferLimit ([filecoin-project/lotus#7153](https://github.com/filecoin-project/lotus/pull/7153))
 
- 
+
 ## Contributors
 
 | Contributor | Commits | Lines ± | Files Changed |
@@ -565,46 +816,46 @@ improvements in different area, like deal making, sealing and so on.
 
 # v1.11.2 / 2021-09-06
 
-lotus v1.11.2 is a feature release that's **highly recommended ALL lotus users to upgrade**, including node operators, 
-storage providers and clients. 
+lotus v1.11.2 is a feature release that's **highly recommended ALL lotus users to upgrade**, including node operators,
+storage providers and clients.
 
 ## Highlights
 - 🌟🌟🌟 Introduce Dagstore and CARv2 for deal-making (#6671) ([filecoin-project/lotus#6671](https://github.com/filecoin-project/lotus/pull/6671))
-  - **[lotus miner markets' Dagstore](https://docs.filecoin.io/mine/lotus/dagstore/#conceptual-overview)** is a 
+  - **[lotus miner markets' Dagstore](https://docs.filecoin.io/mine/lotus/dagstore/#conceptual-overview)** is a
     component of the `markets` subsystem in lotus-miner. It is a sharded store to hold large IPLD graphs efficiently,
-    packaged as  location-transparent attachable CAR files and it replaces the former Badger staging blockstore. It 
+    packaged as  location-transparent attachable CAR files and it replaces the former Badger staging blockstore. It
     is designed to provide high efficiency and throughput, and minimize resource utilization during deal-making operations.  
-    The dagstore also leverages the indexing features of [CARv2](https://github.com/ipld/ipld/blob/master/specs/transport/car/carv2/index.md) to enable plan CAR files to act as read and write 
-    blockstores, which are served as the direct medium for data exchanges in markets for both storage and retrieval 
+    The dagstore also leverages the indexing features of [CARv2](https://github.com/ipld/ipld/blob/master/specs/transport/car/carv2/index.md) to enable plan CAR files to act as read and write
+    blockstores, which are served as the direct medium for data exchanges in markets for both storage and retrieval
     deal making without requiring intermediate buffers.
-  - In the future, lotus will leverage and interact with Dagstore a lot for new features and improvements for deal 
-    making, therefore, it's highly recommended to lotus users to go through [Lotus Miner: About the markets dagstore](https://docs.filecoin.io/mine/lotus/dagstore/#conceptual-overview) thoroughly to learn more about Dagstore's 
+  - In the future, lotus will leverage and interact with Dagstore a lot for new features and improvements for deal
+    making, therefore, it's highly recommended to lotus users to go through [Lotus Miner: About the markets dagstore](https://docs.filecoin.io/mine/lotus/dagstore/#conceptual-overview) thoroughly to learn more about Dagstore's
     conceptual overview, terminology, directory structure, configuration and so on.
-  - **Note**: 
-    - When you first start your lotus-miner or market subsystem with this release, a one-time/first-time **dagstore migration** will be triggered which replaces the former Badger staging blockstore with dagstore. We highly 
+  - **Note**:
+    - When you first start your lotus-miner or market subsystem with this release, a one-time/first-time **dagstore migration** will be triggered which replaces the former Badger staging blockstore with dagstore. We highly
       recommend storage providers to read this [section](https://docs.filecoin.io/mine/lotus/dagstore/#first-time-migration) to learn more about
       what the process does, what to expect and how monitor it.
-    - It is highly recommended to **wait all ongoing data transfer to finish or cancel inbound storage deals that 
+    - It is highly recommended to **wait all ongoing data transfer to finish or cancel inbound storage deals that
       are still transferring**, using the `lotus-miner data-transfers cancel` command before upgrade your market nodes. Reason being that the new dagstore changes attributes in the internal deal state objects, and the paths to the staging CARs where the deal data was being placed will be lost.
     - ‼️Having your dags initialized will become important in the near feature for you to provide a better storage
-          and retrieval service. We'd suggest you to start [forced bulk initialization] soon if possible as this process
-          places relatively high IP workload on your storage system and is better to be carried out gradually and over a
-          longer timeframe. Read how to do properly perform a force bulk initialization [here](https://docs.filecoin.io/mine/lotus/dagstore/#forcing-bulk-initialization).
+      and retrieval service. We'd suggest you to start [forced bulk initialization] soon if possible as this process
+      places relatively high IP workload on your storage system and is better to be carried out gradually and over a
+      longer timeframe. Read how to do properly perform a force bulk initialization [here](https://docs.filecoin.io/mine/lotus/dagstore/#forcing-bulk-initialization).
     - ⏮ Rollback Alert(from v1.11.2-rcX to any version lower): If a storages deal is initiated with M1/v1.11.2(-rcX)
       release, it needs to get to the `StorageDealAwaitingPrecommit` state before you can do a version rollback or the markets process may panic.
-  - 💙 **Special thanks to [MinerX fellows for testing and providing valuable feedbacks](https://github.com/filecoin-project/lotus/discussions/6852) for Dagstore in the past month!** 
+  - 💙 **Special thanks to [MinerX fellows for testing and providing valuable feedbacks](https://github.com/filecoin-project/lotus/discussions/6852) for Dagstore in the past month!**
 - 🌟🌟 rpcenc: Support reader redirect ([filecoin-project/lotus#6952](https://github.com/filecoin-project/lotus/pull/6952))
   - This allows market processes to send piece bytes directly to workers involved on `AddPiece`.
 - Extending sectors: more practical and flexible tools ([filecoin-project/lotus#6097](https://github.com/filecoin-project/lotus/pull/6097))
-    - `lotus-miner sectors check-expire` to inspect expiring sectors.
-    - `lotus-miner sectors renew` for renewing expiring sectors, see the command help menu for customizable option 
-      like `extension`, `new-expiration` and so on.
+  - `lotus-miner sectors check-expire` to inspect expiring sectors.
+  - `lotus-miner sectors renew` for renewing expiring sectors, see the command help menu for customizable option
+    like `extension`, `new-expiration` and so on.
 - ‼️ MpoolReplaceCmd ( lotus mpool replace`) now takes FIL for fee-limit ([filecoin-project/lotus#6927](https://github.com/filecoin-project/lotus/pull/6927))
 - Drop townhall/chainwatch ([filecoin-project/lotus#6912](https://github.com/filecoin-project/lotus/pull/6912))
-    - ChainWatch is no longer supported by lotus.
+  - ChainWatch is no longer supported by lotus.
 - Configurable CC Sector Expiration ([filecoin-project/lotus#6803](https://github.com/filecoin-project/lotus/pull/6803))
-    - Set `CommittedCapacitySectorLifetime` in lotus-miner/config.toml to specify the default expiration for a new CC 
-      sector, value must be between 180-540 days inclusive.
+  - Set `CommittedCapacitySectorLifetime` in lotus-miner/config.toml to specify the default expiration for a new CC
+    sector, value must be between 180-540 days inclusive.
 
 ## New Features
 - api/command for encoding actor params ([filecoin-project/lotus#7150](https://github.com/filecoin-project/lotus/pull/7150))
@@ -671,10 +922,10 @@ storage providers and clients.
 - remove m1 templates and make area selection multi-optionable ([filecoin-project/lotus#7121](https://github.com/filecoin-project/lotus/pull/7121))
 - release -> master ([filecoin-project/lotus#7105](https://github.com/filecoin-project/lotus/pull/7105))
 - Lotus release process  - how we make releases ([filecoin-project/lotus#6944](https://github.com/filecoin-project/lotus/pull/6944))
-- codecov: fix mock name ([filecoin-project/lotus#7039](https://github.com/filecoin-project/lotus/pull/7039))  
+- codecov: fix mock name ([filecoin-project/lotus#7039](https://github.com/filecoin-project/lotus/pull/7039))
 - codecov: fix regexes ([filecoin-project/lotus#7037](https://github.com/filecoin-project/lotus/pull/7037))
-- chore: disable flaky test ([filecoin-project/lotus#6957](https://github.com/filecoin-project/lotus/pull/6957))  
-- set buildtype in nerpa and butterfly ([filecoin-project/lotus#6085](https://github.com/filecoin-project/lotus/pull/6085))  
+- chore: disable flaky test ([filecoin-project/lotus#6957](https://github.com/filecoin-project/lotus/pull/6957))
+- set buildtype in nerpa and butterfly ([filecoin-project/lotus#6085](https://github.com/filecoin-project/lotus/pull/6085))
 - release v1.11.1 backport -> master ([filecoin-project/lotus#6929](https://github.com/filecoin-project/lotus/pull/6929))
 - chore: fixup issue templates ([filecoin-project/lotus#6899](https://github.com/filecoin-project/lotus/pull/6899))
 - bump master version to v1.11.2-dev ([filecoin-project/lotus#6903](https://github.com/filecoin-project/lotus/pull/6903))
@@ -716,15 +967,15 @@ Contributors
 
 > Note: for discussion about this release, please comment [here](https://github.com/filecoin-project/lotus/discussions/6904)
 
-This is a  **highly recommended** but optional Lotus v1.11.1 release that introduces many deal making and datastore improvements and new features along with other bug fixes. 
+This is a  **highly recommended** but optional Lotus v1.11.1 release that introduces many deal making and datastore improvements and new features along with other bug fixes.
 
 ## Highlights
 - ⭐️⭐️⭐️[**lotus-miner market subsystem**](https://docs.filecoin.io/mine/lotus/split-markets-miners/#frontmatter-title) is introduced in this release! It is **highly recommended** for storage providers to run markets processes on a separate machine! Doing so, only this machine needs to exposes public ports for deal making. This also means that the other miner operations can now be completely isolated by from the deal making processes and storage providers can stop and restarts the markets process without affecting an ongoing Winning/Window PoSt!
-  - More details on the concepts, architecture and how to split the market process can be found [here](https://docs.filecoin.io/mine/lotus/split-markets-miners/#concepts). 
+  - More details on the concepts, architecture and how to split the market process can be found [here](https://docs.filecoin.io/mine/lotus/split-markets-miners/#concepts).
   - Base on your system setup(running on separate machines, same machine and so on), please see the suggested practice by community members [here](https://github.com/filecoin-project/lotus/discussions/7047#discussion-3515335).
     - Note: if you are running lotus-worker on a different machine, you will need to set `MARKETS_API_INFO` for certain CLI to work properly. This will be improved by #7072.
   - Huge thanks to MinerX fellows for [helping testing the implementation, reporting the issues so they were fixed by now and providing feedbacks](https://github.com/filecoin-project/lotus/discussions/6861) to user docs in the past three weeks!
-- Config for collateral from miner available balance ([filecoin-project/lotus#6629](https://github.com/filecoin-project/lotus/pull/6629)) 
+- Config for collateral from miner available balance ([filecoin-project/lotus#6629](https://github.com/filecoin-project/lotus/pull/6629))
   - Better control your sector collateral payment by setting `CollateralFromMinerBalance`, `AvailableBalanceBuffer` and `DisableCollateralFallback`.
     - `CollateralFromMinerBalance`:  whether to use available miner balance for sector collateral instead of sending it with each message, default is `false`.
     - `AvailableBalanceBuffer`: minimum available balance to keep in the miner actor before sending it with messages, default is 0FIL.
@@ -732,114 +983,114 @@ This is a  **highly recommended** but optional Lotus v1.11.1 release that introd
 - Config for deal publishing control addresses ([filecoin-project/lotus#6697](https://github.com/filecoin-project/lotus/pull/6697))
   - Set `DealPublishControl` to set the wallet used for sending `PublishStorageDeals` messages, instructions [here](https://docs.filecoin.io/mine/lotus/miner-addresses/#control-addresses).
 - Config UX improvements ([filecoin-project/lotus#6848](https://github.com/filecoin-project/lotus/pull/6848))
-  - You can now preview the the default and updated node config by running `lotus/lotus-miner config default/updated`  
-  
+  - You can now preview the the default and updated node config by running `lotus/lotus-miner config default/updated`
+
 ## New Features
-  - ⭐️⭐️⭐️ Support standalone miner-market process ([filecoin-project/lotus#6356](https://github.com/filecoin-project/lotus/pull/6356))
-  - **⭐️⭐️ Experimental** [Splitstore]((https://github.com/filecoin-project/lotus/blob/master/blockstore/splitstore/README.md)) (more details coming in v1.11.2! Stay tuned! Join the discussion [here](https://github.com/filecoin-project/lotus/discussions/5788) if you have questions!) :
-    - Improve splitstore warmup ([filecoin-project/lotus#6867](https://github.com/filecoin-project/lotus/pull/6867))
-    - Moving GC for badger ([filecoin-project/lotus#6854](https://github.com/filecoin-project/lotus/pull/6854))
-    - splitstore shed utils ([filecoin-project/lotus#6811](https://github.com/filecoin-project/lotus/pull/6811))
-    - fix warmup by decoupling state from message receipt walk ([filecoin-project/lotus#6841](https://github.com/filecoin-project/lotus/pull/6841))
-    - Splitstore: support on-disk marksets using badger ([filecoin-project/lotus#6833](https://github.com/filecoin-project/lotus/pull/6833)) 
-    - cache loaded block messages ([filecoin-project/lotus#6760](https://github.com/filecoin-project/lotus/pull/6760))
-    - Splitstore: add retention policy option for keeping messages in the hotstore ([filecoin-project/lotus#6775](https://github.com/filecoin-project/lotus/pull/6775))
-    - Introduce the LOTUS_CHAIN_BADGERSTORE_DISABLE_FSYNC envvar ([filecoin-project/lotus#6817](https://github.com/filecoin-project/lotus/pull/6817))
-    - Splitstore: add support for protecting out of chain references in the blockstore ([filecoin-project/lotus#6777](https://github.com/filecoin-project/lotus/pull/6777))
-    - Implement exposed splitstore ([filecoin-project/lotus#6762](https://github.com/filecoin-project/lotus/pull/6762))
-    - Splitstore code reorg ([filecoin-project/lotus#6756](https://github.com/filecoin-project/lotus/pull/6756))
-    - Splitstore: Some small fixes ([filecoin-project/lotus#6754](https://github.com/filecoin-project/lotus/pull/6754))
-    - Splitstore Enhanchements ([filecoin-project/lotus#6474](https://github.com/filecoin-project/lotus/pull/6474))
-  - lotus-shed: initial export cmd for markets related metadata ([filecoin-project/lotus#6840](https://github.com/filecoin-project/lotus/pull/6840))
-  - add a very verbose -vv flag to lotus and lotus-miner. ([filecoin-project/lotus#6888](https://github.com/filecoin-project/lotus/pull/6888))
-  - Add allocated sectorid vis ([filecoin-project/lotus#4638](https://github.com/filecoin-project/lotus/pull/4638))
-  - add a command for compacting sector numbers bitfield ([filecoin-project/lotus#4640](https://github.com/filecoin-project/lotus/pull/4640))  
-    - Run `lotus-miner actor compact-allocated` to compact sector number allocations to reduce the size of the allocated sector number bitfield.
-  - Add ChainGetMessagesInTipset API ([filecoin-project/lotus#6642](https://github.com/filecoin-project/lotus/pull/6642))
-  - Handle the --color flag via proper global state ([filecoin-project/lotus#6743](https://github.com/filecoin-project/lotus/pull/6743))
-     - Enable color by default only if os.Stdout is a TTY ([filecoin-project/lotus#6696](https://github.com/filecoin-project/lotus/pull/6696))
-     - Stop outputing ANSI color on non-TTY ([filecoin-project/lotus#6694](https://github.com/filecoin-project/lotus/pull/6694))
-  - Envvar to disable slash filter ([filecoin-project/lotus#6620](https://github.com/filecoin-project/lotus/pull/6620))
-  - commit batch: AggregateAboveBaseFee config ([filecoin-project/lotus#6650](https://github.com/filecoin-project/lotus/pull/6650))
-  - shed tool to estimate aggregate network fees ([filecoin-project/lotus#6631](https://github.com/filecoin-project/lotus/pull/6631))
-  
+- ⭐️⭐️⭐️ Support standalone miner-market process ([filecoin-project/lotus#6356](https://github.com/filecoin-project/lotus/pull/6356))
+- **⭐️⭐️ Experimental** [Splitstore]((https://github.com/filecoin-project/lotus/blob/master/blockstore/splitstore/README.md)) (more details coming in v1.11.2! Stay tuned! Join the discussion [here](https://github.com/filecoin-project/lotus/discussions/5788) if you have questions!) :
+  - Improve splitstore warmup ([filecoin-project/lotus#6867](https://github.com/filecoin-project/lotus/pull/6867))
+  - Moving GC for badger ([filecoin-project/lotus#6854](https://github.com/filecoin-project/lotus/pull/6854))
+  - splitstore shed utils ([filecoin-project/lotus#6811](https://github.com/filecoin-project/lotus/pull/6811))
+  - fix warmup by decoupling state from message receipt walk ([filecoin-project/lotus#6841](https://github.com/filecoin-project/lotus/pull/6841))
+  - Splitstore: support on-disk marksets using badger ([filecoin-project/lotus#6833](https://github.com/filecoin-project/lotus/pull/6833))
+  - cache loaded block messages ([filecoin-project/lotus#6760](https://github.com/filecoin-project/lotus/pull/6760))
+  - Splitstore: add retention policy option for keeping messages in the hotstore ([filecoin-project/lotus#6775](https://github.com/filecoin-project/lotus/pull/6775))
+  - Introduce the LOTUS_CHAIN_BADGERSTORE_DISABLE_FSYNC envvar ([filecoin-project/lotus#6817](https://github.com/filecoin-project/lotus/pull/6817))
+  - Splitstore: add support for protecting out of chain references in the blockstore ([filecoin-project/lotus#6777](https://github.com/filecoin-project/lotus/pull/6777))
+  - Implement exposed splitstore ([filecoin-project/lotus#6762](https://github.com/filecoin-project/lotus/pull/6762))
+  - Splitstore code reorg ([filecoin-project/lotus#6756](https://github.com/filecoin-project/lotus/pull/6756))
+  - Splitstore: Some small fixes ([filecoin-project/lotus#6754](https://github.com/filecoin-project/lotus/pull/6754))
+  - Splitstore Enhanchements ([filecoin-project/lotus#6474](https://github.com/filecoin-project/lotus/pull/6474))
+- lotus-shed: initial export cmd for markets related metadata ([filecoin-project/lotus#6840](https://github.com/filecoin-project/lotus/pull/6840))
+- add a very verbose -vv flag to lotus and lotus-miner. ([filecoin-project/lotus#6888](https://github.com/filecoin-project/lotus/pull/6888))
+- Add allocated sectorid vis ([filecoin-project/lotus#4638](https://github.com/filecoin-project/lotus/pull/4638))
+- add a command for compacting sector numbers bitfield ([filecoin-project/lotus#4640](https://github.com/filecoin-project/lotus/pull/4640))
+  - Run `lotus-miner actor compact-allocated` to compact sector number allocations to reduce the size of the allocated sector number bitfield.
+- Add ChainGetMessagesInTipset API ([filecoin-project/lotus#6642](https://github.com/filecoin-project/lotus/pull/6642))
+- Handle the --color flag via proper global state ([filecoin-project/lotus#6743](https://github.com/filecoin-project/lotus/pull/6743))
+  - Enable color by default only if os.Stdout is a TTY ([filecoin-project/lotus#6696](https://github.com/filecoin-project/lotus/pull/6696))
+  - Stop outputing ANSI color on non-TTY ([filecoin-project/lotus#6694](https://github.com/filecoin-project/lotus/pull/6694))
+- Envvar to disable slash filter ([filecoin-project/lotus#6620](https://github.com/filecoin-project/lotus/pull/6620))
+- commit batch: AggregateAboveBaseFee config ([filecoin-project/lotus#6650](https://github.com/filecoin-project/lotus/pull/6650))
+- shed tool to estimate aggregate network fees ([filecoin-project/lotus#6631](https://github.com/filecoin-project/lotus/pull/6631))
+
 ## Bug Fixes
-  - Fix padding of deals, which only partially shipped in #5988 ([filecoin-project/lotus#6683](https://github.com/filecoin-project/lotus/pull/6683))
-  - fix deal concurrency test failures by upgrading graphsync and others ([filecoin-project/lotus#6724](https://github.com/filecoin-project/lotus/pull/6724))
-  - fix: on randomness change, use new rand ([filecoin-project/lotus#6805](https://github.com/filecoin-project/lotus/pull/6805))  - fix: always check if StateSearchMessage returns nil ([filecoin-project/lotus#6802](https://github.com/filecoin-project/lotus/pull/6802))  
-  - test: fix flaky window post tests ([filecoin-project/lotus#6804](https://github.com/filecoin-project/lotus/pull/6804))
-  -  wrap close(wait) with sync.Once to avoid panic ([filecoin-project/lotus#6800](https://github.com/filecoin-project/lotus/pull/6800))
-  - fixes #6786 segfault ([filecoin-project/lotus#6787](https://github.com/filecoin-project/lotus/pull/6787))
-  -  ClientRetrieve stops on cancel([filecoin-project/lotus#6739](https://github.com/filecoin-project/lotus/pull/6739))
-  - Fix bugs in sectors extend --v1-sectors ([filecoin-project/lotus#6066](https://github.com/filecoin-project/lotus/pull/6066))
-  - fix "lotus-seed genesis car" error "merkledag: not found" ([filecoin-project/lotus#6688](https://github.com/filecoin-project/lotus/pull/6688))
-  - Get retrieval pricing input should not error out on a deal state fetch ([filecoin-project/lotus#6679](https://github.com/filecoin-project/lotus/pull/6679))
-  - Fix more CID double-encoding as hex ([filecoin-project/lotus#6680](https://github.com/filecoin-project/lotus/pull/6680))  
-  - storage: Fix FinalizeSector with sectors in stoage paths ([filecoin-project/lotus#6653](https://github.com/filecoin-project/lotus/pull/6653))
-  - Fix tiny error in check-client-datacap ([filecoin-project/lotus#6664](https://github.com/filecoin-project/lotus/pull/6664))
-  - Fix: precommit_batch method used the wrong cfg.CommitBatchWait ([filecoin-project/lotus#6658](https://github.com/filecoin-project/lotus/pull/6658))
-  - fix ticket expiration check ([filecoin-project/lotus#6635](https://github.com/filecoin-project/lotus/pull/6635))
-  - remove precommit check in handleCommitFailed ([filecoin-project/lotus#6634](https://github.com/filecoin-project/lotus/pull/6634))
-  - fix prove commit aggregate send token amount ([filecoin-project/lotus#6625](https://github.com/filecoin-project/lotus/pull/6625))
-  
+- Fix padding of deals, which only partially shipped in #5988 ([filecoin-project/lotus#6683](https://github.com/filecoin-project/lotus/pull/6683))
+- fix deal concurrency test failures by upgrading graphsync and others ([filecoin-project/lotus#6724](https://github.com/filecoin-project/lotus/pull/6724))
+- fix: on randomness change, use new rand ([filecoin-project/lotus#6805](https://github.com/filecoin-project/lotus/pull/6805))  - fix: always check if StateSearchMessage returns nil ([filecoin-project/lotus#6802](https://github.com/filecoin-project/lotus/pull/6802))
+- test: fix flaky window post tests ([filecoin-project/lotus#6804](https://github.com/filecoin-project/lotus/pull/6804))
+-  wrap close(wait) with sync.Once to avoid panic ([filecoin-project/lotus#6800](https://github.com/filecoin-project/lotus/pull/6800))
+- fixes #6786 segfault ([filecoin-project/lotus#6787](https://github.com/filecoin-project/lotus/pull/6787))
+-  ClientRetrieve stops on cancel([filecoin-project/lotus#6739](https://github.com/filecoin-project/lotus/pull/6739))
+- Fix bugs in sectors extend --v1-sectors ([filecoin-project/lotus#6066](https://github.com/filecoin-project/lotus/pull/6066))
+- fix "lotus-seed genesis car" error "merkledag: not found" ([filecoin-project/lotus#6688](https://github.com/filecoin-project/lotus/pull/6688))
+- Get retrieval pricing input should not error out on a deal state fetch ([filecoin-project/lotus#6679](https://github.com/filecoin-project/lotus/pull/6679))
+- Fix more CID double-encoding as hex ([filecoin-project/lotus#6680](https://github.com/filecoin-project/lotus/pull/6680))
+- storage: Fix FinalizeSector with sectors in stoage paths ([filecoin-project/lotus#6653](https://github.com/filecoin-project/lotus/pull/6653))
+- Fix tiny error in check-client-datacap ([filecoin-project/lotus#6664](https://github.com/filecoin-project/lotus/pull/6664))
+- Fix: precommit_batch method used the wrong cfg.CommitBatchWait ([filecoin-project/lotus#6658](https://github.com/filecoin-project/lotus/pull/6658))
+- fix ticket expiration check ([filecoin-project/lotus#6635](https://github.com/filecoin-project/lotus/pull/6635))
+- remove precommit check in handleCommitFailed ([filecoin-project/lotus#6634](https://github.com/filecoin-project/lotus/pull/6634))
+- fix prove commit aggregate send token amount ([filecoin-project/lotus#6625](https://github.com/filecoin-project/lotus/pull/6625))
+
 ## Improvements
-  - Eliminate inefficiency in markets logging ([filecoin-project/lotus#6895](https://github.com/filecoin-project/lotus/pull/6895))
-  - rename `cmd/lotus{-storage=>}-miner` to match binary. ([filecoin-project/lotus#6886](https://github.com/filecoin-project/lotus/pull/6886))
-  - fix racy TestSimultanenousTransferLimit. ([filecoin-project/lotus#6862](https://github.com/filecoin-project/lotus/pull/6862))
-  - ValidateBlock: Assert that block header height's are greater than parents ([filecoin-project/lotus#6872](https://github.com/filecoin-project/lotus/pull/6872))
-  - feat: Don't panic when api impl is nil ([filecoin-project/lotus#6857](https://github.com/filecoin-project/lotus/pull/6857))
-  - add docker-compose file ([filecoin-project/lotus#6544](https://github.com/filecoin-project/lotus/pull/6544))
-  - easy way to make install app ([filecoin-project/lotus#5183](https://github.com/filecoin-project/lotus/pull/5183))
-  - api: Separate the Net interface from Common ([filecoin-project/lotus#6627](https://github.com/filecoin-project/lotus/pull/6627))  - add StateReadState to gateway api ([filecoin-project/lotus#6818](https://github.com/filecoin-project/lotus/pull/6818))  
-  - add SealProof in SectorBuilder ([filecoin-project/lotus#6815](https://github.com/filecoin-project/lotus/pull/6815))
-  - sealing: Handle preCommitParams errors more correctly ([filecoin-project/lotus#6763](https://github.com/filecoin-project/lotus/pull/6763)) 
-  - ClientFindData: always fetch peer id from chain ([filecoin-project/lotus#6807](https://github.com/filecoin-project/lotus/pull/6807))
-  - test: handle null blocks in TestForkRefuseCall ([filecoin-project/lotus#6758](https://github.com/filecoin-project/lotus/pull/6758))
-  - Add more deal details to lotus-miner info ([filecoin-project/lotus#6708](https://github.com/filecoin-project/lotus/pull/6708))
-  - add election backtest ([filecoin-project/lotus#5950](https://github.com/filecoin-project/lotus/pull/5950))
-  - add dollar sign ([filecoin-project/lotus#6690](https://github.com/filecoin-project/lotus/pull/6690))
-  - get-actor cli spelling fix ([filecoin-project/lotus#6681](https://github.com/filecoin-project/lotus/pull/6681))
-  - polish(statetree): accept a context in statetree diff for timeouts ([filecoin-project/lotus#6639](https://github.com/filecoin-project/lotus/pull/6639))
-  - Add helptext to lotus chain export ([filecoin-project/lotus#6672](https://github.com/filecoin-project/lotus/pull/6672))
-  - add an incremental nonce itest. ([filecoin-project/lotus#6663](https://github.com/filecoin-project/lotus/pull/6663))
-  - commit batch: Initialize the FailedSectors map ([filecoin-project/lotus#6647](https://github.com/filecoin-project/lotus/pull/6647))
-  - Fast-path retry submitting commit aggregate if commit is still valid  ([filecoin-project/lotus#6638](https://github.com/filecoin-project/lotus/pull/6638))
-  - Reuse timers in sealing batch logic ([filecoin-project/lotus#6636](https://github.com/filecoin-project/lotus/pull/6636))
-  
+- Eliminate inefficiency in markets logging ([filecoin-project/lotus#6895](https://github.com/filecoin-project/lotus/pull/6895))
+- rename `cmd/lotus{-storage=>}-miner` to match binary. ([filecoin-project/lotus#6886](https://github.com/filecoin-project/lotus/pull/6886))
+- fix racy TestSimultanenousTransferLimit. ([filecoin-project/lotus#6862](https://github.com/filecoin-project/lotus/pull/6862))
+- ValidateBlock: Assert that block header height's are greater than parents ([filecoin-project/lotus#6872](https://github.com/filecoin-project/lotus/pull/6872))
+- feat: Don't panic when api impl is nil ([filecoin-project/lotus#6857](https://github.com/filecoin-project/lotus/pull/6857))
+- add docker-compose file ([filecoin-project/lotus#6544](https://github.com/filecoin-project/lotus/pull/6544))
+- easy way to make install app ([filecoin-project/lotus#5183](https://github.com/filecoin-project/lotus/pull/5183))
+- api: Separate the Net interface from Common ([filecoin-project/lotus#6627](https://github.com/filecoin-project/lotus/pull/6627))  - add StateReadState to gateway api ([filecoin-project/lotus#6818](https://github.com/filecoin-project/lotus/pull/6818))
+- add SealProof in SectorBuilder ([filecoin-project/lotus#6815](https://github.com/filecoin-project/lotus/pull/6815))
+- sealing: Handle preCommitParams errors more correctly ([filecoin-project/lotus#6763](https://github.com/filecoin-project/lotus/pull/6763))
+- ClientFindData: always fetch peer id from chain ([filecoin-project/lotus#6807](https://github.com/filecoin-project/lotus/pull/6807))
+- test: handle null blocks in TestForkRefuseCall ([filecoin-project/lotus#6758](https://github.com/filecoin-project/lotus/pull/6758))
+- Add more deal details to lotus-miner info ([filecoin-project/lotus#6708](https://github.com/filecoin-project/lotus/pull/6708))
+- add election backtest ([filecoin-project/lotus#5950](https://github.com/filecoin-project/lotus/pull/5950))
+- add dollar sign ([filecoin-project/lotus#6690](https://github.com/filecoin-project/lotus/pull/6690))
+- get-actor cli spelling fix ([filecoin-project/lotus#6681](https://github.com/filecoin-project/lotus/pull/6681))
+- polish(statetree): accept a context in statetree diff for timeouts ([filecoin-project/lotus#6639](https://github.com/filecoin-project/lotus/pull/6639))
+- Add helptext to lotus chain export ([filecoin-project/lotus#6672](https://github.com/filecoin-project/lotus/pull/6672))
+- add an incremental nonce itest. ([filecoin-project/lotus#6663](https://github.com/filecoin-project/lotus/pull/6663))
+- commit batch: Initialize the FailedSectors map ([filecoin-project/lotus#6647](https://github.com/filecoin-project/lotus/pull/6647))
+- Fast-path retry submitting commit aggregate if commit is still valid  ([filecoin-project/lotus#6638](https://github.com/filecoin-project/lotus/pull/6638))
+- Reuse timers in sealing batch logic ([filecoin-project/lotus#6636](https://github.com/filecoin-project/lotus/pull/6636))
+
 ## Dependency Updates
-  - Update to proof v8.0.3 ([filecoin-project/lotus#6890](https://github.com/filecoin-project/lotus/pull/6890))
-  - update to go-fil-market v1.6.0 ([filecoin-project/lotus#6885](https://github.com/filecoin-project/lotus/pull/6885)) 
-  - Bump go-multihash, adjust test for supported version ([filecoin-project/lotus#6674](https://github.com/filecoin-project/lotus/pull/6674))
-  - github.com/filecoin-project/go-data-transfer (v1.6.0 -> v1.7.2):
-  - github.com/filecoin-project/go-fil-markets (v1.5.0 -> v1.6.2):
-  - github.com/filecoin-project/go-padreader (v0.0.0-20200903213702-ed5fae088b20 -> v0.0.0-20210723183308-812a16dc01b1)
-  - github.com/filecoin-project/go-state-types (v0.1.1-0.20210506134452-99b279731c48 -> v0.1.1-0.20210810190654-139e0e79e69e)
-  - github.com/filecoin-project/go-statemachine (v0.0.0-20200925024713-05bd7c71fbfe -> v1.0.1)
-  - update go-libp2p-pubsub to v0.5.0 ([filecoin-project/lotus#6764](https://github.com/filecoin-project/lotus/pull/6764))
-  
+- Update to proof v8.0.3 ([filecoin-project/lotus#6890](https://github.com/filecoin-project/lotus/pull/6890))
+- update to go-fil-market v1.6.0 ([filecoin-project/lotus#6885](https://github.com/filecoin-project/lotus/pull/6885))
+- Bump go-multihash, adjust test for supported version ([filecoin-project/lotus#6674](https://github.com/filecoin-project/lotus/pull/6674))
+- github.com/filecoin-project/go-data-transfer (v1.6.0 -> v1.7.2):
+- github.com/filecoin-project/go-fil-markets (v1.5.0 -> v1.6.2):
+- github.com/filecoin-project/go-padreader (v0.0.0-20200903213702-ed5fae088b20 -> v0.0.0-20210723183308-812a16dc01b1)
+- github.com/filecoin-project/go-state-types (v0.1.1-0.20210506134452-99b279731c48 -> v0.1.1-0.20210810190654-139e0e79e69e)
+- github.com/filecoin-project/go-statemachine (v0.0.0-20200925024713-05bd7c71fbfe -> v1.0.1)
+- update go-libp2p-pubsub to v0.5.0 ([filecoin-project/lotus#6764](https://github.com/filecoin-project/lotus/pull/6764))
+
 ## Others
-  - Master->v1.11.1 ([filecoin-project/lotus#7051](https://github.com/filecoin-project/lotus/pull/7051))
-  - v1.11.1-rc2 ([filecoin-project/lotus#6966](https://github.com/filecoin-project/lotus/pull/6966))
-  - Backport master -> v1.11.1 ([filecoin-project/lotus#6965](https://github.com/filecoin-project/lotus/pull/6965))
-  - Fixes in master -> release ([filecoin-project/lotus#6933](https://github.com/filecoin-project/lotus/pull/6933))
-  - Add changelog  for v1.11.1-rc1 and bump the version ([filecoin-project/lotus#6900](https://github.com/filecoin-project/lotus/pull/6900))
-  - Fix  merge release -> v1.11.1 ([filecoin-project/lotus#6897](https://github.com/filecoin-project/lotus/pull/6897)) 
-  - Update RELEASE_ISSUE_TEMPLATE.md ([filecoin-project/lotus#6880](https://github.com/filecoin-project/lotus/pull/6880))
-  - Add github actions for staled pr ([filecoin-project/lotus#6879](https://github.com/filecoin-project/lotus/pull/6879))
-  - Update issue templates and add templates for M1 ([filecoin-project/lotus#6856](https://github.com/filecoin-project/lotus/pull/6856))
-  - Fix links in issue templates
-  - Update issue templates to forms ([filecoin-project/lotus#6798](https://github.com/filecoin-project/lotus/pull/6798)
-  - Nerpa v13 upgrade ([filecoin-project/lotus#6837](https://github.com/filecoin-project/lotus/pull/6837))
-  - add docker-compose file ([filecoin-project/lotus#6544](https://github.com/filecoin-project/lotus/pull/6544))
-  - release -> master ([filecoin-project/lotus#6828](https://github.com/filecoin-project/lotus/pull/6828))
-  - Resurrect CODEOWNERS, but for maintainers group ([filecoin-project/lotus#6773](https://github.com/filecoin-project/lotus/pull/6773))
-  - Master disclaimer  ([filecoin-project/lotus#6757](https://github.com/filecoin-project/lotus/pull/6757))
-  - Create stale.yml ([filecoin-project/lotus#6747](https://github.com/filecoin-project/lotus/pull/6747))
-  - Release template: Update all testnet infra at once ([filecoin-project/lotus#6710](https://github.com/filecoin-project/lotus/pull/6710))
-  - Release Template: remove binary validation step ([filecoin-project/lotus#6709](https://github.com/filecoin-project/lotus/pull/6709))
-  - Reset of the interop network ([filecoin-project/lotus#6689](https://github.com/filecoin-project/lotus/pull/6689))  
-  - Update version.go to 1.11.1 ([filecoin-project/lotus#6621](https://github.com/filecoin-project/lotus/pull/6621))
-  
+- Master->v1.11.1 ([filecoin-project/lotus#7051](https://github.com/filecoin-project/lotus/pull/7051))
+- v1.11.1-rc2 ([filecoin-project/lotus#6966](https://github.com/filecoin-project/lotus/pull/6966))
+- Backport master -> v1.11.1 ([filecoin-project/lotus#6965](https://github.com/filecoin-project/lotus/pull/6965))
+- Fixes in master -> release ([filecoin-project/lotus#6933](https://github.com/filecoin-project/lotus/pull/6933))
+- Add changelog  for v1.11.1-rc1 and bump the version ([filecoin-project/lotus#6900](https://github.com/filecoin-project/lotus/pull/6900))
+- Fix  merge release -> v1.11.1 ([filecoin-project/lotus#6897](https://github.com/filecoin-project/lotus/pull/6897))
+- Update RELEASE_ISSUE_TEMPLATE.md ([filecoin-project/lotus#6880](https://github.com/filecoin-project/lotus/pull/6880))
+- Add github actions for staled pr ([filecoin-project/lotus#6879](https://github.com/filecoin-project/lotus/pull/6879))
+- Update issue templates and add templates for M1 ([filecoin-project/lotus#6856](https://github.com/filecoin-project/lotus/pull/6856))
+- Fix links in issue templates
+- Update issue templates to forms ([filecoin-project/lotus#6798](https://github.com/filecoin-project/lotus/pull/6798)
+- Nerpa v13 upgrade ([filecoin-project/lotus#6837](https://github.com/filecoin-project/lotus/pull/6837))
+- add docker-compose file ([filecoin-project/lotus#6544](https://github.com/filecoin-project/lotus/pull/6544))
+- release -> master ([filecoin-project/lotus#6828](https://github.com/filecoin-project/lotus/pull/6828))
+- Resurrect CODEOWNERS, but for maintainers group ([filecoin-project/lotus#6773](https://github.com/filecoin-project/lotus/pull/6773))
+- Master disclaimer  ([filecoin-project/lotus#6757](https://github.com/filecoin-project/lotus/pull/6757))
+- Create stale.yml ([filecoin-project/lotus#6747](https://github.com/filecoin-project/lotus/pull/6747))
+- Release template: Update all testnet infra at once ([filecoin-project/lotus#6710](https://github.com/filecoin-project/lotus/pull/6710))
+- Release Template: remove binary validation step ([filecoin-project/lotus#6709](https://github.com/filecoin-project/lotus/pull/6709))
+- Reset of the interop network ([filecoin-project/lotus#6689](https://github.com/filecoin-project/lotus/pull/6689))
+- Update version.go to 1.11.1 ([filecoin-project/lotus#6621](https://github.com/filecoin-project/lotus/pull/6621))
+
 ## Contributors
 
 | Contributor | Commits | Lines ± | Files Changed |
@@ -880,7 +1131,7 @@ This is a  **highly recommended** but optional Lotus v1.11.1 release that introd
 | dependabot[bot] | 1 | +3/-3 | 1 |
 | zhoutian527 | 1 | +2/-2 | 1 |
 | xloem | 1 | +4/-0 | 1 |
-| @travisperson| 2 | +2/-2 | 3 |
+| | 2 | +2/-2 | 3 |
 | Liviu Damian | 2 | +2/-2 | 2 |
 | @jimpick | 2 | +2/-2 | 2 |
 | Frank | 1 | +3/-0 | 1 |
@@ -889,9 +1140,10 @@ This is a  **highly recommended** but optional Lotus v1.11.1 release that introd
 
 # 1.11.0 / 2021-07-22
 
-This is a **highly recommended** release of Lotus that have many bug fixes, improvements and new features. 
+This is a **highly recommended** release of Lotus that have many bug fixes, improvements and new features.
 
 ## Highlights
+- Miner SimultaneousTransfers config ([filecoin-project/lotus#6612](https://github.com/filecoin-project/lotus/pull/6612))
 - Miner SimultaneousTransfers config ([filecoin-project/lotus#6612](https://github.com/filecoin-project/lotus/pull/6612))
   - Set `SimultaneousTransfers` in lotus miner config to configure the maximum number of parallel online data transfers, including both storage and retrieval deals.
 - Dynamic Retrieval pricing ([filecoin-project/lotus#6175](https://github.com/filecoin-project/lotus/pull/6175))
@@ -900,7 +1152,7 @@ This is a **highly recommended** release of Lotus that have many bug fixes, impr
   - run `lotus mpool manage and follow the instructions!
   - Demo available at https://www.youtube.com/watch?v=QDocpLQjZgQ.
 - Add utils to use multisigs as miner owners ([filecoin-project/lotus#6490](https://github.com/filecoin-project/lotus/pull/6490))
-  
+
 ## More New Features
 -  feat: implement lotus-sim ([filecoin-project/lotus#6406](https://github.com/filecoin-project/lotus/pull/6406))
 - implement a command to export a car ([filecoin-project/lotus#6405](https://github.com/filecoin-project/lotus/pull/6405))
@@ -917,11 +1169,11 @@ This is a **highly recommended** release of Lotus that have many bug fixes, impr
 - Transplant some useful commands to lotus-shed actor ([filecoin-project/lotus#5913](https://github.com/filecoin-project/lotus/pull/5913))
   - run `lotus-shed actor`
 - actor wrapper codegen ([filecoin-project/lotus#6108](https://github.com/filecoin-project/lotus/pull/6108))
-- Add a shed util to count miners by post type ([filecoin-project/lotus#6169](https://github.com/filecoin-project/lotus/pull/6169))  
+- Add a shed util to count miners by post type ([filecoin-project/lotus#6169](https://github.com/filecoin-project/lotus/pull/6169))
 - shed: command to list duplicate messages in tipsets (steb) ([filecoin-project/lotus#5847](https://github.com/filecoin-project/lotus/pull/5847))
 - feat: allow checkpointing to forks ([filecoin-project/lotus#6107](https://github.com/filecoin-project/lotus/pull/6107))
-- Add a CLI tool for miner proving deadline ([filecoin-project/lotus#6132](https://github.com/filecoin-project/lotus/pull/6132))   
-  - run `lotus state miner-proving-deadline` 
+- Add a CLI tool for miner proving deadline ([filecoin-project/lotus#6132](https://github.com/filecoin-project/lotus/pull/6132))
+  - run `lotus state miner-proving-deadline`
 
 
 ## Bug Fixes
@@ -930,7 +1182,7 @@ This is a **highly recommended** release of Lotus that have many bug fixes, impr
 - Make query-ask CLI more graceful ([filecoin-project/lotus#6590](https://github.com/filecoin-project/lotus/pull/6590))
 - scale up sector expiration to avoid sector expire in batch-pre-commit waitting ([filecoin-project/lotus#6566](https://github.com/filecoin-project/lotus/pull/6566))
 -  Fix an error in msigLockCancel ([filecoin-project/lotus#6582](https://github.com/filecoin-project/lotus/pull/6582)
-- fix circleci being out of sync. ([filecoin-project/lotus#6573](https://github.com/filecoin-project/lotus/pull/6573))  
+- fix circleci being out of sync. ([filecoin-project/lotus#6573](https://github.com/filecoin-project/lotus/pull/6573))
 - Fix helptext for ask price([filecoin-project/lotus#6560](https://github.com/filecoin-project/lotus/pull/6560))
 - fix commit finalize failed ([filecoin-project/lotus#6521](https://github.com/filecoin-project/lotus/pull/6521))
 - Fix soup ([filecoin-project/lotus#6501](https://github.com/filecoin-project/lotus/pull/6501))
@@ -955,7 +1207,7 @@ This is a **highly recommended** release of Lotus that have many bug fixes, impr
 - Speed up StateListMessages in some cases ([filecoin-project/lotus#6007](https://github.com/filecoin-project/lotus/pull/6007))
 - fix(splitstore): fix a panic on revert-only head changes ([filecoin-project/lotus#6133](https://github.com/filecoin-project/lotus/pull/6133))
 - drand: fix beacon cache ([filecoin-project/lotus#6164](https://github.com/filecoin-project/lotus/pull/6164))
- 
+
 ## Improvements
 - gateway: Add support for Version method ([filecoin-project/lotus#6618](https://github.com/filecoin-project/lotus/pull/6618))
 - revamped integration test kit (aka. Operation Sparks Joy) ([filecoin-project/lotus#6329](https://github.com/filecoin-project/lotus/pull/6329))
@@ -990,7 +1242,7 @@ This is a **highly recommended** release of Lotus that have many bug fixes, impr
 - Remove log line when tracing is not configured ([filecoin-project/lotus#6334](https://github.com/filecoin-project/lotus/pull/6334))
 - separate tracing environment variables ([filecoin-project/lotus#6323](https://github.com/filecoin-project/lotus/pull/6323))
 - feat: log dispute rate ([filecoin-project/lotus#6322](https://github.com/filecoin-project/lotus/pull/6322))
-- Move verifreg shed utils to CLI ([filecoin-project/lotus#6135](https://github.com/filecoin-project/lotus/pull/6135))  
+- Move verifreg shed utils to CLI ([filecoin-project/lotus#6135](https://github.com/filecoin-project/lotus/pull/6135))
 - consider storiface.PathStorage when calculating storage requirements ([filecoin-project/lotus#6233](https://github.com/filecoin-project/lotus/pull/6233))
 - `storage` module: add go docs and minor code quality refactors ([filecoin-project/lotus#6259](https://github.com/filecoin-project/lotus/pull/6259))
 - Increase data transfer timeouts ([filecoin-project/lotus#6300](https://github.com/filecoin-project/lotus/pull/6300))
@@ -1007,7 +1259,7 @@ This is a **highly recommended** release of Lotus that have many bug fixes, impr
 - Testground checks on push ([filecoin-project/lotus#5887](https://github.com/filecoin-project/lotus/pull/5887))
 - Use EmptyTSK where appropriate ([filecoin-project/lotus#6134](https://github.com/filecoin-project/lotus/pull/6134))
 - upgrade `lotus-soup` testplans and reduce deals concurrency to a single miner ([filecoin-project/lotus#6122](https://github.com/filecoin-project/lotus/pull/6122)
-  
+
 ## Dependency Updates
 - downgrade libp2p/go-libp2p-yamux to v0.5.1. ([filecoin-project/lotus#6605](https://github.com/filecoin-project/lotus/pull/6605))
 - Update libp2p to 0.14.2 ([filecoin-project/lotus#6404](https://github.com/filecoin-project/lotus/pull/6404))
@@ -1015,7 +1267,7 @@ This is a **highly recommended** release of Lotus that have many bug fixes, impr
 - Use new actor tags ([filecoin-project/lotus#6291](https://github.com/filecoin-project/lotus/pull/6291))
 - chore: update go-libp2p ([filecoin-project/lotus#6231](https://github.com/filecoin-project/lotus/pull/6231))
 - Update ffi to proofs v7 ([filecoin-project/lotus#6150](https://github.com/filecoin-project/lotus/pull/6150))
-  
+
 ## Others
 - Initial draft: basic build instructions on Readme ([filecoin-project/lotus#6498](https://github.com/filecoin-project/lotus/pull/6498))
 - Remove rc changelog, compile the new changelog for final release only ([filecoin-project/lotus#6444](https://github.com/filecoin-project/lotus/pull/6444))
@@ -1033,8 +1285,8 @@ This is a **highly recommended** release of Lotus that have many bug fixes, impr
 - Introduce a release issue template ([filecoin-project/lotus#5826](https://github.com/filecoin-project/lotus/pull/5826))
 - This is a 1:1 forward-port of PR#6183 from 1.9.x to master ([filecoin-project/lotus#6196](https://github.com/filecoin-project/lotus/pull/6196))
 - Update cli gen ([filecoin-project/lotus#6155](https://github.com/filecoin-project/lotus/pull/6155))
-- Generate CLI docs ([filecoin-project/lotus#6145](https://github.com/filecoin-project/lotus/pull/6145))  
-  
+- Generate CLI docs ([filecoin-project/lotus#6145](https://github.com/filecoin-project/lotus/pull/6145))
+
 ## Contributors
 
 | Contributor | Commits | Lines ± | Files Changed |
@@ -1046,7 +1298,7 @@ This is a **highly recommended** release of Lotus that have many bug fixes, impr
 | @Stebalien | 106 | +7653/-2718 | 273 |
 | dirkmc | 11 | +2580/-1371 | 77 |
 | @dirkmc | 39 | +1865/-1194 | 79 |
-| @Kubuxu | 19 | +1973/-485 | 81 |
+|  | 19 | +1973/-485 | 81 |
 | @vyzo | 4 | +1748/-330 | 50 |
 | @aarshkshah1992 | 5 | +1462/-213 | 27 |
 | @coryschwartz | 35 | +568/-206 | 59 |
@@ -1083,10 +1335,10 @@ This is an optional but **highly recommended** release of Lotus for lotus miners
 ## New Features
 - commit batch: AggregateAboveBaseFee config #6650
   - `AggregateAboveBaseFee` is added to miner sealing configuration for setting the network base fee to start aggregating proofs. When the network base fee is lower than this value, the prove commits will be submitted individually via `ProveCommitSector`. According to the [Batch Incentive Alignment](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0013.md#batch-incentive-alignment) introduced in FIP-0013, we recommend miners to set this value to 0.15 nanoFIL(which is the default value) to avoid unexpected aggregation fee in burn and enjoy the most benefits of aggregation!
-    
+
 ## Bug Fixes
 - storage: Fix FinalizeSector with sectors in storage paths #6652
-- Fix tiny error in check-client-datacap #6664  
+- Fix tiny error in check-client-datacap #6664
 - Fix: precommit_batch method used the wrong cfg.PreCommitBatchWait #6658
 - to optimize the batchwait #6636
 - fix getTicket: sector precommitted but expired case #6635
@@ -1115,10 +1367,10 @@ This is an optional but **highly recommended** release of Lotus for lotus miners
 ## New Features
 - commit batch: AggregateAboveBaseFee config #6650
   - `AggregateAboveBaseFee` is added to miner sealing configuration for setting the network base fee to start aggregating proofs. When the network base fee is lower than this value, the prove commits will be submitted individually via `ProveCommitSector`. According to the [Batch Incentive Alignment](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0013.md#batch-incentive-alignment) introduced in FIP-0013, we recommend miners to set this value to 0.15 nanoFIL(which is the default value) to avoid unexpected aggregation fee in burn and enjoy the most benefits of aggregation!
-    
+
 ## Bug Fixes
 - storage: Fix FinalizeSector with sectors in storage paths #6652
-- Fix tiny error in check-client-datacap #6664  
+- Fix tiny error in check-client-datacap #6664
 - Fix: precommit_batch method used the wrong cfg.PreCommitBatchWait #6658
 - to optimize the batchwait #6636
 - fix getTicket: sector precommitted but expired case #6635
@@ -1161,10 +1413,10 @@ FIPs [0008](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0008.m
 **Check out the documentation [here](https://docs.filecoin.io/mine/lotus/miner-configuration/#precommitsectorsbatch) for details on the new Lotus miner sealing config options, [here](https://docs.filecoin.io/mine/lotus/miner-configuration/#fees-section) for fee config options, and explanations of the new features.**
 
 Note:
-  - We recommend to keep `PreCommitSectorsBatch` as 1.
-  - We recommend miners to set `PreCommitBatchWait` lower than 30 hours.
-  - We recommend miners to set a longer `CommitBatchSlack` and  `PreCommitBatchSlack` to prevent message failures
-    due to expirations.
+- We recommend to keep `PreCommitSectorsBatch` as 1.
+- We recommend miners to set `PreCommitBatchWait` lower than 30 hours.
+- We recommend miners to set a longer `CommitBatchSlack` and  `PreCommitBatchSlack` to prevent message failures
+  due to expirations.
 
 ### Projected state tree growth
 
@@ -1175,9 +1427,9 @@ Given these assumptions:
 - We'd expect a network storage growth rate of around 530PiB per day. 😳 🎉 🥳 😅
 - We'd expect network bandwidth dedicated to `SubmitWindowedPoSt` to grow by about 0.02% per day.
 - We'd expect the [state-tree](https://spec.filecoin.io/#section-systems.filecoin_vm.state_tree) (and therefore [snapshot](https://docs.filecoin.io/get-started/lotus/chain/#lightweight-snapshot)) size to grow by 1.16GiB per day.
-   - Nearly all of the state-tree growth is expected to come from new sector metadata.
+  - Nearly all of the state-tree growth is expected to come from new sector metadata.
 - We'd expect the daily lotus datastore growth rate to increase by about 10-15% (from current ~21GiB/day).
-   - Most "growth" of the lotus datastore is due to "churn", historical data that's no longer referenced by the latest state-tree.
+  - Most "growth" of the lotus datastore is due to "churn", historical data that's no longer referenced by the latest state-tree.
 
 ### Future improvements
 
@@ -2866,4 +3118,4 @@ We are grateful for every contribution!
 
 We are very excited to release **lotus** 0.1.0. This is our testnet release. To install lotus and join the testnet, please visit [lotu.sh](lotu.sh). Please file bug reports as [issues](https://github.com/filecoin-project/lotus/issues).
 
-A huge thank you to all contributors for this testnet release!
+A huge thank you to all contributors for this testnet release!

Dockerfile.lotus

@@ -28,6 +28,14 @@ WORKDIR /opt/filecoin
 RUN make clean deps
 
 
+FROM builder-local AS builder-test
+MAINTAINER Lotus Development Team
+
+WORKDIR /opt/filecoin
+
+RUN make debug
+
+
 FROM builder-local AS builder
 MAINTAINER Lotus Development Team
 
@@ -207,3 +215,40 @@ EXPOSE 1234
 EXPOSE 2345
 EXPOSE 3456
 EXPOSE 1777
+
+###
+from base as lotus-test
+
+ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
+ENV LOTUS_MINER_PATH /var/lib/lotus-miner
+ENV LOTUS_PATH /var/lib/lotus
+ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
+ENV WALLET_PATH /var/lib/lotus-wallet
+
+COPY --from=builder-test /opt/filecoin/lotus      /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-miner /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-worker /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-seed /usr/local/bin/
+
+RUN mkdir /var/tmp/filecoin-proof-parameters
+RUN mkdir /var/lib/lotus
+RUN mkdir /var/lib/lotus-miner
+RUN mkdir /var/lib/lotus-worker
+RUN mkdir /var/lib/lotus-wallet
+RUN chown fc: /var/tmp/filecoin-proof-parameters
+RUN chown fc: /var/lib/lotus
+RUN chown fc: /var/lib/lotus-miner
+RUN chown fc: /var/lib/lotus-worker
+RUN chown fc: /var/lib/lotus-wallet
+
+
+VOLUME /var/tmp/filecoin-proof-parameters
+VOLUME /var/lib/lotus
+VOLUME /var/lib/lotus-miner
+VOLUME /var/lib/lotus-worker
+VOLUME /var/lib/lotus-wallet
+
+EXPOSE 1234
+EXPOSE 2345
+EXPOSE 3456
+EXPOSE 1777

Makefile

@@ -345,6 +345,8 @@ gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci
 	@echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO MAKE docsgen-cli"
 .PHONY: gen
 
+jen: gen
+
 snap: lotus lotus-miner lotus-worker
 	snapcraft
 	# snapcraft upload ./lotus_*.snap

api/api_full.go

@@ -689,7 +689,17 @@ type FullNode interface {
 	// MethodGroup: Paych
 	// The Paych methods are for interacting with and managing payment channels
 
-	PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*ChannelInfo, error)                     //perm:sign
+	// PaychGet gets or creates a payment channel between address pair
+	//  The specified amount will be reserved for use. If there aren't enough non-reserved funds
+	//    available, funds will be added through an on-chain message.
+	//  - When opts.OffChain is true, this call will not cause any messages to be sent to the chain (no automatic
+	//    channel creation/funds adding). If the operation can't be performed without sending a message an error will be
+	//    returned. Note that even when this option is specified, this call can be blocked by previous operations on the
+	//    channel waiting for on-chain operations.
+	PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt, opts PaychGetOpts) (*ChannelInfo, error) //perm:sign
+	// PaychFund gets or creates a payment channel between address pair.
+	// The specified amount will be added to the channel through on-chain send for future use
+	PaychFund(ctx context.Context, from, to address.Address, amt types.BigInt) (*ChannelInfo, error)                    //perm:sign
 	PaychGetWaitReady(context.Context, cid.Cid) (address.Address, error)                                                //perm:sign
 	PaychAvailableFunds(ctx context.Context, ch address.Address) (*ChannelAvailableFunds, error)                        //perm:sign
 	PaychAvailableFundsByFromTo(ctx context.Context, from, to address.Address) (*ChannelAvailableFunds, error)          //perm:sign
@@ -828,6 +838,10 @@ const (
 	PCHOutbound
 )
 
+type PaychGetOpts struct {
+	OffChain bool
+}
+
 type PaychStatus struct {
 	ControlAddr address.Address
 	Direction   PCHDir
@@ -845,16 +859,23 @@ type ChannelAvailableFunds struct {
 	From address.Address
 	// To is the to address of the channel
 	To address.Address
-	// ConfirmedAmt is the amount of funds that have been confirmed on-chain
-	// for the channel
+
+	// ConfirmedAmt is the total amount of funds that have been confirmed on-chain for the channel
 	ConfirmedAmt types.BigInt
 	// PendingAmt is the amount of funds that are pending confirmation on-chain
 	PendingAmt types.BigInt
+
+	// NonReservedAmt is part of ConfirmedAmt that is available for use (e.g. when the payment channel was pre-funded)
+	NonReservedAmt types.BigInt
+	// PendingAvailableAmt is the amount of funds that are pending confirmation on-chain that will become available once confirmed
+	PendingAvailableAmt types.BigInt
+
 	// PendingWaitSentinel can be used with PaychGetWaitReady to wait for
 	// confirmation of pending funds
 	PendingWaitSentinel *cid.Cid
 	// QueuedAmt is the amount that is queued up behind a pending request
 	QueuedAmt types.BigInt
+
 	// VoucherRedeemedAmt is the amount that is redeemed by vouchers on-chain
 	// and in the local datastore
 	VoucherReedeemedAmt types.BigInt
@@ -900,6 +921,7 @@ type QueryOffer struct {
 	Size                    uint64
 	MinPrice                types.BigInt
 	UnsealPrice             types.BigInt
+	PricePerByte            abi.TokenAmount
 	PaymentInterval         uint64
 	PaymentIntervalIncrease uint64
 	Miner                   address.Address

api/api_gateway.go

@@ -45,8 +45,9 @@ type Gateway interface {
 	GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
 	MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
 	MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
-	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
 	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
+	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
+	MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MsigVesting, error)
 	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
 	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
 	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)

api/api_net.go

@@ -51,6 +51,15 @@ type Net interface {
 	NetBlockRemove(ctx context.Context, acl NetBlockList) error //perm:admin
 	NetBlockList(ctx context.Context) (NetBlockList, error)     //perm:read
 
+	NetProtectAdd(ctx context.Context, acl []peer.ID) error    //perm:admin
+	NetProtectRemove(ctx context.Context, acl []peer.ID) error //perm:admin
+	NetProtectList(ctx context.Context) ([]peer.ID, error)     //perm:read
+
+	// ResourceManager API
+	NetStat(ctx context.Context, scope string) (NetStat, error)          //perm:read
+	NetLimit(ctx context.Context, scope string) (NetLimit, error)        //perm:read
+	NetSetLimit(ctx context.Context, scope string, limit NetLimit) error //perm:admin
+
 	// ID returns peerID of libp2p node backing this API
 	ID(context.Context) (peer.ID, error) //perm:read
 }

api/api_storage.go

@@ -154,6 +154,7 @@ type StorageMiner interface {
 	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                                          //perm:admin
 	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                               //perm:admin
 	StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error)                                                                                               //perm:admin
+	StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error)                                                                                                 //perm:admin
 
 	StorageLocal(ctx context.Context) (map[stores.ID]string, error)       //perm:admin
 	StorageStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) //perm:admin
@@ -169,6 +170,8 @@ type StorageMiner interface {
 	MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error)                                                                                                             //perm:read
 	MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)                                                                                                          //perm:write
 	MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)                                                                                                   //perm:write
+	// MarketDataTransferDiagnostics generates debugging information about current data transfers over graphsync
+	MarketDataTransferDiagnostics(ctx context.Context, p peer.ID) (*TransferDiagnostics, error) //perm:write
 	// MarketRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer
 	MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
 	// MarketCancelDataTransfer cancels a data transfer with the given transfer ID and other peer
@@ -219,6 +222,16 @@ type StorageMiner interface {
 	// DagstoreGC runs garbage collection on the DAG store.
 	DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin
 
+	// IndexerAnnounceDeal informs indexer nodes that a new deal was received,
+	// so they can download its index
+	IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin
+
+	// IndexerAnnounceAllDeals informs the indexer nodes aboutall active deals.
+	IndexerAnnounceAllDeals(ctx context.Context) error //perm:admin
+
+	// DagstoreLookupPieces returns information about shards that contain the given CID.
+	DagstoreLookupPieces(ctx context.Context, cid cid.Cid) ([]DagstoreShardInfo, error) //perm:admin
+
 	// RuntimeSubsystems returns the subsystems that are enabled
 	// in this instance.
 	RuntimeSubsystems(ctx context.Context) (MinerSubsystems, error) //perm:read

api/docgen/docgen.go

@@ -1,6 +1,7 @@
 package docgen
 
 import (
+	"encoding/json"
 	"fmt"
 	"go/ast"
 	"go/parser"
@@ -15,6 +16,7 @@ import (
 	"github.com/filecoin-project/go-bitfield"
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-graphsync"
 	"github.com/libp2p/go-libp2p-core/metrics"
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
@@ -120,6 +122,7 @@ func init() {
 	addExample(api.FullAPIVersion1)
 	addExample(api.PCHInbound)
 	addExample(time.Minute)
+	addExample(graphsync.NewRequestID())
 	addExample(datatransfer.TransferID(3))
 	addExample(datatransfer.Ongoing)
 	addExample(storeIDExample)
@@ -250,10 +253,18 @@ func init() {
 	addExample(map[abi.SectorNumber]string{
 		123: "can't acquire read lock",
 	})
+	addExample(json.RawMessage(`"json raw message"`))
 	addExample(map[api.SectorState]int{
 		api.SectorState(sealing.Proving): 120,
 	})
 	addExample([]abi.SectorNumber{123, 124})
+	addExample([]storiface.SectorLock{
+		{
+			Sector: abi.SectorID{Number: 123, Miner: 1000},
+			Write:  [storiface.FileTypes]uint{0, 0, 1},
+			Read:   [storiface.FileTypes]uint{2, 3, 0},
+		},
+	})
 
 	// worker specific
 	addExample(storiface.AcquireMove)
@@ -289,6 +300,34 @@ func init() {
 		Error: "<error>",
 	})
 	addExample(storiface.ResourceTable)
+	addExample(network.ScopeStat{
+		Memory:             123,
+		NumStreamsInbound:  1,
+		NumStreamsOutbound: 2,
+		NumConnsInbound:    3,
+		NumConnsOutbound:   4,
+		NumFD:              5,
+	})
+	addExample(map[string]network.ScopeStat{
+		"abc": {
+			Memory:             123,
+			NumStreamsInbound:  1,
+			NumStreamsOutbound: 2,
+			NumConnsInbound:    3,
+			NumConnsOutbound:   4,
+			NumFD:              5,
+		}})
+	addExample(api.NetLimit{
+		Memory:          123,
+		StreamsInbound:  1,
+		StreamsOutbound: 2,
+		Streams:         3,
+		ConnsInbound:    3,
+		ConnsOutbound:   4,
+		Conns:           4,
+		FD:              5,
+	})
+
 }
 
 func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) {
@@ -339,7 +378,7 @@ func ExampleValue(method string, t, parent reflect.Type) interface{} {
 	switch t.Kind() {
 	case reflect.Slice:
 		out := reflect.New(t).Elem()
-		reflect.Append(out, reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
+		out = reflect.Append(out, reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
 		return out.Interface()
 	case reflect.Chan:
 		return ExampleValue(method, t.Elem(), nil)

api/mocks/mock_full.go

@@ -1811,6 +1811,21 @@ func (mr *MockFullNodeMockRecorder) NetFindPeer(arg0, arg1 interface{}) *gomock.
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetFindPeer", reflect.TypeOf((*MockFullNode)(nil).NetFindPeer), arg0, arg1)
 }
 
+// NetLimit mocks base method.
+func (m *MockFullNode) NetLimit(arg0 context.Context, arg1 string) (api.NetLimit, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetLimit", arg0, arg1)
+	ret0, _ := ret[0].(api.NetLimit)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetLimit indicates an expected call of NetLimit.
+func (mr *MockFullNodeMockRecorder) NetLimit(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetLimit", reflect.TypeOf((*MockFullNode)(nil).NetLimit), arg0, arg1)
+}
+
 // NetPeerInfo mocks base method.
 func (m *MockFullNode) NetPeerInfo(arg0 context.Context, arg1 peer.ID) (*api.ExtendedPeerInfo, error) {
 	m.ctrl.T.Helper()
@@ -1841,6 +1856,49 @@ func (mr *MockFullNodeMockRecorder) NetPeers(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPeers", reflect.TypeOf((*MockFullNode)(nil).NetPeers), arg0)
 }
 
+// NetProtectAdd mocks base method.
+func (m *MockFullNode) NetProtectAdd(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectAdd", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectAdd indicates an expected call of NetProtectAdd.
+func (mr *MockFullNodeMockRecorder) NetProtectAdd(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectAdd", reflect.TypeOf((*MockFullNode)(nil).NetProtectAdd), arg0, arg1)
+}
+
+// NetProtectList mocks base method.
+func (m *MockFullNode) NetProtectList(arg0 context.Context) ([]peer.ID, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectList", arg0)
+	ret0, _ := ret[0].([]peer.ID)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetProtectList indicates an expected call of NetProtectList.
+func (mr *MockFullNodeMockRecorder) NetProtectList(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectList", reflect.TypeOf((*MockFullNode)(nil).NetProtectList), arg0)
+}
+
+// NetProtectRemove mocks base method.
+func (m *MockFullNode) NetProtectRemove(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectRemove", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectRemove indicates an expected call of NetProtectRemove.
+func (mr *MockFullNodeMockRecorder) NetProtectRemove(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectRemove", reflect.TypeOf((*MockFullNode)(nil).NetProtectRemove), arg0, arg1)
+}
+
 // NetPubsubScores mocks base method.
 func (m *MockFullNode) NetPubsubScores(arg0 context.Context) ([]api.PubsubScore, error) {
 	m.ctrl.T.Helper()
@@ -1856,6 +1914,35 @@ func (mr *MockFullNodeMockRecorder) NetPubsubScores(arg0 interface{}) *gomock.Ca
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPubsubScores", reflect.TypeOf((*MockFullNode)(nil).NetPubsubScores), arg0)
 }
 
+// NetSetLimit mocks base method.
+func (m *MockFullNode) NetSetLimit(arg0 context.Context, arg1 string, arg2 api.NetLimit) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetSetLimit", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetSetLimit indicates an expected call of NetSetLimit.
+func (mr *MockFullNodeMockRecorder) NetSetLimit(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetSetLimit", reflect.TypeOf((*MockFullNode)(nil).NetSetLimit), arg0, arg1, arg2)
+}
+
+// NetStat mocks base method.
+func (m *MockFullNode) NetStat(arg0 context.Context, arg1 string) (api.NetStat, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetStat", arg0, arg1)
+	ret0, _ := ret[0].(api.NetStat)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetStat indicates an expected call of NetStat.
+func (mr *MockFullNodeMockRecorder) NetStat(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetStat", reflect.TypeOf((*MockFullNode)(nil).NetStat), arg0, arg1)
+}
+
 // NodeStatus mocks base method.
 func (m *MockFullNode) NodeStatus(arg0 context.Context, arg1 bool) (api.NodeStatus, error) {
 	m.ctrl.T.Helper()
@@ -1931,19 +2018,34 @@ func (mr *MockFullNodeMockRecorder) PaychCollect(arg0, arg1 interface{}) *gomock
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychCollect", reflect.TypeOf((*MockFullNode)(nil).PaychCollect), arg0, arg1)
 }
 
-// PaychGet mocks base method.
-func (m *MockFullNode) PaychGet(arg0 context.Context, arg1, arg2 address.Address, arg3 big.Int) (*api.ChannelInfo, error) {
+// PaychFund mocks base method.
+func (m *MockFullNode) PaychFund(arg0 context.Context, arg1, arg2 address.Address, arg3 big.Int) (*api.ChannelInfo, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "PaychGet", arg0, arg1, arg2, arg3)
+	ret := m.ctrl.Call(m, "PaychFund", arg0, arg1, arg2, arg3)
+	ret0, _ := ret[0].(*api.ChannelInfo)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// PaychFund indicates an expected call of PaychFund.
+func (mr *MockFullNodeMockRecorder) PaychFund(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychFund", reflect.TypeOf((*MockFullNode)(nil).PaychFund), arg0, arg1, arg2, arg3)
+}
+
+// PaychGet mocks base method.
+func (m *MockFullNode) PaychGet(arg0 context.Context, arg1, arg2 address.Address, arg3 big.Int, arg4 api.PaychGetOpts) (*api.ChannelInfo, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "PaychGet", arg0, arg1, arg2, arg3, arg4)
 	ret0, _ := ret[0].(*api.ChannelInfo)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
 
 // PaychGet indicates an expected call of PaychGet.
-func (mr *MockFullNodeMockRecorder) PaychGet(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+func (mr *MockFullNodeMockRecorder) PaychGet(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychGet", reflect.TypeOf((*MockFullNode)(nil).PaychGet), arg0, arg1, arg2, arg3)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychGet", reflect.TypeOf((*MockFullNode)(nil).PaychGet), arg0, arg1, arg2, arg3, arg4)
 }
 
 // PaychGetWaitReady mocks base method.

api/proxy_gen.go

@@ -306,7 +306,9 @@ type FullNodeStruct struct {
 
 		PaychCollect func(p0 context.Context, p1 address.Address) (cid.Cid, error) `perm:"sign"`
 
-		PaychGet func(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt) (*ChannelInfo, error) `perm:"sign"`
+		PaychFund func(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt) (*ChannelInfo, error) `perm:"sign"`
+
+		PaychGet func(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt, p4 PaychGetOpts) (*ChannelInfo, error) `perm:"sign"`
 
 		PaychGetWaitReady func(p0 context.Context, p1 cid.Cid) (address.Address, error) `perm:"sign"`
 
@@ -516,6 +518,8 @@ type GatewayStruct struct {
 
 		MsigGetVested func(p0 context.Context, p1 address.Address, p2 types.TipSetKey, p3 types.TipSetKey) (types.BigInt, error) ``
 
+		MsigGetVestingSchedule func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MsigVesting, error) ``
+
 		StateAccountKey func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) ``
 
 		StateDealProviderCollateralBounds func(p0 context.Context, p1 abi.PaddedPieceSize, p2 bool, p3 types.TipSetKey) (DealCollateralBounds, error) ``
@@ -587,11 +591,23 @@ type NetStruct struct {
 
 		NetFindPeer func(p0 context.Context, p1 peer.ID) (peer.AddrInfo, error) `perm:"read"`
 
+		NetLimit func(p0 context.Context, p1 string) (NetLimit, error) `perm:"read"`
+
 		NetPeerInfo func(p0 context.Context, p1 peer.ID) (*ExtendedPeerInfo, error) `perm:"read"`
 
 		NetPeers func(p0 context.Context) ([]peer.AddrInfo, error) `perm:"read"`
 
+		NetProtectAdd func(p0 context.Context, p1 []peer.ID) error `perm:"admin"`
+
+		NetProtectList func(p0 context.Context) ([]peer.ID, error) `perm:"read"`
+
+		NetProtectRemove func(p0 context.Context, p1 []peer.ID) error `perm:"admin"`
+
 		NetPubsubScores func(p0 context.Context) ([]PubsubScore, error) `perm:"read"`
+
+		NetSetLimit func(p0 context.Context, p1 string, p2 NetLimit) error `perm:"admin"`
+
+		NetStat func(p0 context.Context, p1 string) (NetStat, error) `perm:"read"`
 	}
 }
 
@@ -633,6 +649,8 @@ type StorageMinerStruct struct {
 
 		DagstoreListShards func(p0 context.Context) ([]DagstoreShardInfo, error) `perm:"read"`
 
+		DagstoreLookupPieces func(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) `perm:"admin"`
+
 		DagstoreRecoverShard func(p0 context.Context, p1 string) error `perm:"write"`
 
 		DealsConsiderOfflineRetrievalDeals func(p0 context.Context) (bool, error) `perm:"admin"`
@@ -667,8 +685,14 @@ type StorageMinerStruct struct {
 
 		DealsSetPieceCidBlocklist func(p0 context.Context, p1 []cid.Cid) error `perm:"admin"`
 
+		IndexerAnnounceAllDeals func(p0 context.Context) error `perm:"admin"`
+
+		IndexerAnnounceDeal func(p0 context.Context, p1 cid.Cid) error `perm:"admin"`
+
 		MarketCancelDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"`
 
+		MarketDataTransferDiagnostics func(p0 context.Context, p1 peer.ID) (*TransferDiagnostics, error) `perm:"write"`
+
 		MarketDataTransferUpdates func(p0 context.Context) (<-chan DataTransferChannel, error) `perm:"write"`
 
 		MarketGetAsk func(p0 context.Context) (*storagemarket.SignedStorageAsk, error) `perm:"read"`
@@ -809,6 +833,8 @@ type StorageMinerStruct struct {
 
 		StorageFindSector func(p0 context.Context, p1 abi.SectorID, p2 storiface.SectorFileType, p3 abi.SectorSize, p4 bool) ([]stores.SectorStorageInfo, error) `perm:"admin"`
 
+		StorageGetLocks func(p0 context.Context) (storiface.SectorLocks, error) `perm:"admin"`
+
 		StorageInfo func(p0 context.Context, p1 stores.ID) (stores.StorageInfo, error) `perm:"admin"`
 
 		StorageList func(p0 context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"`
@@ -2175,14 +2201,25 @@ func (s *FullNodeStub) PaychCollect(p0 context.Context, p1 address.Address) (cid
 	return *new(cid.Cid), ErrNotSupported
 }
 
-func (s *FullNodeStruct) PaychGet(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt) (*ChannelInfo, error) {
+func (s *FullNodeStruct) PaychFund(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt) (*ChannelInfo, error) {
+	if s.Internal.PaychFund == nil {
+		return nil, ErrNotSupported
+	}
+	return s.Internal.PaychFund(p0, p1, p2, p3)
+}
+
+func (s *FullNodeStub) PaychFund(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt) (*ChannelInfo, error) {
+	return nil, ErrNotSupported
+}
+
+func (s *FullNodeStruct) PaychGet(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt, p4 PaychGetOpts) (*ChannelInfo, error) {
 	if s.Internal.PaychGet == nil {
 		return nil, ErrNotSupported
 	}
-	return s.Internal.PaychGet(p0, p1, p2, p3)
+	return s.Internal.PaychGet(p0, p1, p2, p3, p4)
 }
 
-func (s *FullNodeStub) PaychGet(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt) (*ChannelInfo, error) {
+func (s *FullNodeStub) PaychGet(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt, p4 PaychGetOpts) (*ChannelInfo, error) {
 	return nil, ErrNotSupported
 }
 
@@ -3275,6 +3312,17 @@ func (s *GatewayStub) MsigGetVested(p0 context.Context, p1 address.Address, p2 t
 	return *new(types.BigInt), ErrNotSupported
 }
 
+func (s *GatewayStruct) MsigGetVestingSchedule(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MsigVesting, error) {
+	if s.Internal.MsigGetVestingSchedule == nil {
+		return *new(MsigVesting), ErrNotSupported
+	}
+	return s.Internal.MsigGetVestingSchedule(p0, p1, p2)
+}
+
+func (s *GatewayStub) MsigGetVestingSchedule(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MsigVesting, error) {
+	return *new(MsigVesting), ErrNotSupported
+}
+
 func (s *GatewayStruct) StateAccountKey(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) {
 	if s.Internal.StateAccountKey == nil {
 		return *new(address.Address), ErrNotSupported
@@ -3627,6 +3675,17 @@ func (s *NetStub) NetFindPeer(p0 context.Context, p1 peer.ID) (peer.AddrInfo, er
 	return *new(peer.AddrInfo), ErrNotSupported
 }
 
+func (s *NetStruct) NetLimit(p0 context.Context, p1 string) (NetLimit, error) {
+	if s.Internal.NetLimit == nil {
+		return *new(NetLimit), ErrNotSupported
+	}
+	return s.Internal.NetLimit(p0, p1)
+}
+
+func (s *NetStub) NetLimit(p0 context.Context, p1 string) (NetLimit, error) {
+	return *new(NetLimit), ErrNotSupported
+}
+
 func (s *NetStruct) NetPeerInfo(p0 context.Context, p1 peer.ID) (*ExtendedPeerInfo, error) {
 	if s.Internal.NetPeerInfo == nil {
 		return nil, ErrNotSupported
@@ -3649,6 +3708,39 @@ func (s *NetStub) NetPeers(p0 context.Context) ([]peer.AddrInfo, error) {
 	return *new([]peer.AddrInfo), ErrNotSupported
 }
 
+func (s *NetStruct) NetProtectAdd(p0 context.Context, p1 []peer.ID) error {
+	if s.Internal.NetProtectAdd == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.NetProtectAdd(p0, p1)
+}
+
+func (s *NetStub) NetProtectAdd(p0 context.Context, p1 []peer.ID) error {
+	return ErrNotSupported
+}
+
+func (s *NetStruct) NetProtectList(p0 context.Context) ([]peer.ID, error) {
+	if s.Internal.NetProtectList == nil {
+		return *new([]peer.ID), ErrNotSupported
+	}
+	return s.Internal.NetProtectList(p0)
+}
+
+func (s *NetStub) NetProtectList(p0 context.Context) ([]peer.ID, error) {
+	return *new([]peer.ID), ErrNotSupported
+}
+
+func (s *NetStruct) NetProtectRemove(p0 context.Context, p1 []peer.ID) error {
+	if s.Internal.NetProtectRemove == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.NetProtectRemove(p0, p1)
+}
+
+func (s *NetStub) NetProtectRemove(p0 context.Context, p1 []peer.ID) error {
+	return ErrNotSupported
+}
+
 func (s *NetStruct) NetPubsubScores(p0 context.Context) ([]PubsubScore, error) {
 	if s.Internal.NetPubsubScores == nil {
 		return *new([]PubsubScore), ErrNotSupported
@@ -3660,6 +3752,28 @@ func (s *NetStub) NetPubsubScores(p0 context.Context) ([]PubsubScore, error) {
 	return *new([]PubsubScore), ErrNotSupported
 }
 
+func (s *NetStruct) NetSetLimit(p0 context.Context, p1 string, p2 NetLimit) error {
+	if s.Internal.NetSetLimit == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.NetSetLimit(p0, p1, p2)
+}
+
+func (s *NetStub) NetSetLimit(p0 context.Context, p1 string, p2 NetLimit) error {
+	return ErrNotSupported
+}
+
+func (s *NetStruct) NetStat(p0 context.Context, p1 string) (NetStat, error) {
+	if s.Internal.NetStat == nil {
+		return *new(NetStat), ErrNotSupported
+	}
+	return s.Internal.NetStat(p0, p1)
+}
+
+func (s *NetStub) NetStat(p0 context.Context, p1 string) (NetStat, error) {
+	return *new(NetStat), ErrNotSupported
+}
+
 func (s *SignableStruct) Sign(p0 context.Context, p1 SignFunc) error {
 	if s.Internal.Sign == nil {
 		return ErrNotSupported
@@ -3781,6 +3895,17 @@ func (s *StorageMinerStub) DagstoreListShards(p0 context.Context) ([]DagstoreSha
 	return *new([]DagstoreShardInfo), ErrNotSupported
 }
 
+func (s *StorageMinerStruct) DagstoreLookupPieces(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) {
+	if s.Internal.DagstoreLookupPieces == nil {
+		return *new([]DagstoreShardInfo), ErrNotSupported
+	}
+	return s.Internal.DagstoreLookupPieces(p0, p1)
+}
+
+func (s *StorageMinerStub) DagstoreLookupPieces(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) {
+	return *new([]DagstoreShardInfo), ErrNotSupported
+}
+
 func (s *StorageMinerStruct) DagstoreRecoverShard(p0 context.Context, p1 string) error {
 	if s.Internal.DagstoreRecoverShard == nil {
 		return ErrNotSupported
@@ -3968,6 +4093,28 @@ func (s *StorageMinerStub) DealsSetPieceCidBlocklist(p0 context.Context, p1 []ci
 	return ErrNotSupported
 }
 
+func (s *StorageMinerStruct) IndexerAnnounceAllDeals(p0 context.Context) error {
+	if s.Internal.IndexerAnnounceAllDeals == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.IndexerAnnounceAllDeals(p0)
+}
+
+func (s *StorageMinerStub) IndexerAnnounceAllDeals(p0 context.Context) error {
+	return ErrNotSupported
+}
+
+func (s *StorageMinerStruct) IndexerAnnounceDeal(p0 context.Context, p1 cid.Cid) error {
+	if s.Internal.IndexerAnnounceDeal == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.IndexerAnnounceDeal(p0, p1)
+}
+
+func (s *StorageMinerStub) IndexerAnnounceDeal(p0 context.Context, p1 cid.Cid) error {
+	return ErrNotSupported
+}
+
 func (s *StorageMinerStruct) MarketCancelDataTransfer(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error {
 	if s.Internal.MarketCancelDataTransfer == nil {
 		return ErrNotSupported
@@ -3979,6 +4126,17 @@ func (s *StorageMinerStub) MarketCancelDataTransfer(p0 context.Context, p1 datat
 	return ErrNotSupported
 }
 
+func (s *StorageMinerStruct) MarketDataTransferDiagnostics(p0 context.Context, p1 peer.ID) (*TransferDiagnostics, error) {
+	if s.Internal.MarketDataTransferDiagnostics == nil {
+		return nil, ErrNotSupported
+	}
+	return s.Internal.MarketDataTransferDiagnostics(p0, p1)
+}
+
+func (s *StorageMinerStub) MarketDataTransferDiagnostics(p0 context.Context, p1 peer.ID) (*TransferDiagnostics, error) {
+	return nil, ErrNotSupported
+}
+
 func (s *StorageMinerStruct) MarketDataTransferUpdates(p0 context.Context) (<-chan DataTransferChannel, error) {
 	if s.Internal.MarketDataTransferUpdates == nil {
 		return nil, ErrNotSupported
@@ -4749,6 +4907,17 @@ func (s *StorageMinerStub) StorageFindSector(p0 context.Context, p1 abi.SectorID
 	return *new([]stores.SectorStorageInfo), ErrNotSupported
 }
 
+func (s *StorageMinerStruct) StorageGetLocks(p0 context.Context) (storiface.SectorLocks, error) {
+	if s.Internal.StorageGetLocks == nil {
+		return *new(storiface.SectorLocks), ErrNotSupported
+	}
+	return s.Internal.StorageGetLocks(p0)
+}
+
+func (s *StorageMinerStub) StorageGetLocks(p0 context.Context) (storiface.SectorLocks, error) {
+	return *new(storiface.SectorLocks), ErrNotSupported
+}
+
 func (s *StorageMinerStruct) StorageInfo(p0 context.Context, p1 stores.ID) (stores.StorageInfo, error) {
 	if s.Internal.StorageInfo == nil {
 		return *new(stores.StorageInfo), ErrNotSupported

api/types.go

@@ -5,11 +5,14 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/libp2p/go-libp2p-core/network"
+
 	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-graphsync"
 
 	"github.com/libp2p/go-libp2p-core/peer"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
@@ -53,6 +56,30 @@ type MessageSendSpec struct {
 	MaxFee abi.TokenAmount
 }
 
+// GraphSyncDataTransfer provides diagnostics on a data transfer happening over graphsync
+type GraphSyncDataTransfer struct {
+	// GraphSync request id for this transfer
+	RequestID *graphsync.RequestID
+	// Graphsync state for this transfer
+	RequestState string
+	// If a channel ID is present, indicates whether this is the current graphsync request for this channel
+	// (could have changed in a restart)
+	IsCurrentChannelRequest bool
+	// Data transfer channel ID for this transfer
+	ChannelID *datatransfer.ChannelID
+	// Data transfer state for this transfer
+	ChannelState *DataTransferChannel
+	// Diagnostic information about this request -- and unexpected inconsistencies in
+	// request state
+	Diagnostics []string
+}
+
+// TransferDiagnostics give current information about transfers going over graphsync that may be helpful for debugging
+type TransferDiagnostics struct {
+	ReceivingTransfers []*GraphSyncDataTransfer
+	SendingTransfers   []*GraphSyncDataTransfer
+}
+
 type DataTransferChannel struct {
 	TransferID  datatransfer.TransferID
 	Status      datatransfer.Status
@@ -98,6 +125,28 @@ func NewDataTransferChannel(hostID peer.ID, channelState datatransfer.ChannelSta
 	return channel
 }
 
+type NetStat struct {
+	System    *network.ScopeStat           `json:",omitempty"`
+	Transient *network.ScopeStat           `json:",omitempty"`
+	Services  map[string]network.ScopeStat `json:",omitempty"`
+	Protocols map[string]network.ScopeStat `json:",omitempty"`
+	Peers     map[string]network.ScopeStat `json:",omitempty"`
+}
+
+type NetLimit struct {
+	Dynamic bool `json:",omitempty"`
+	// set if Dynamic is false
+	Memory int64 `json:",omitempty"`
+	// set if Dynamic is true
+	MemoryFraction float64 `json:",omitempty"`
+	MinMemory      int64   `json:",omitempty"`
+	MaxMemory      int64   `json:",omitempty"`
+
+	Streams, StreamsInbound, StreamsOutbound int
+	Conns, ConnsInbound, ConnsOutbound       int
+	FD                                       int
+}
+
 type NetBlockList struct {
 	Peers     []peer.ID
 	IPAddrs   []string

api/v0api/v0mocks/mock_full.go

@@ -1724,6 +1724,21 @@ func (mr *MockFullNodeMockRecorder) NetFindPeer(arg0, arg1 interface{}) *gomock.
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetFindPeer", reflect.TypeOf((*MockFullNode)(nil).NetFindPeer), arg0, arg1)
 }
 
+// NetLimit mocks base method.
+func (m *MockFullNode) NetLimit(arg0 context.Context, arg1 string) (api.NetLimit, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetLimit", arg0, arg1)
+	ret0, _ := ret[0].(api.NetLimit)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetLimit indicates an expected call of NetLimit.
+func (mr *MockFullNodeMockRecorder) NetLimit(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetLimit", reflect.TypeOf((*MockFullNode)(nil).NetLimit), arg0, arg1)
+}
+
 // NetPeerInfo mocks base method.
 func (m *MockFullNode) NetPeerInfo(arg0 context.Context, arg1 peer.ID) (*api.ExtendedPeerInfo, error) {
 	m.ctrl.T.Helper()
@@ -1754,6 +1769,49 @@ func (mr *MockFullNodeMockRecorder) NetPeers(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPeers", reflect.TypeOf((*MockFullNode)(nil).NetPeers), arg0)
 }
 
+// NetProtectAdd mocks base method.
+func (m *MockFullNode) NetProtectAdd(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectAdd", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectAdd indicates an expected call of NetProtectAdd.
+func (mr *MockFullNodeMockRecorder) NetProtectAdd(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectAdd", reflect.TypeOf((*MockFullNode)(nil).NetProtectAdd), arg0, arg1)
+}
+
+// NetProtectList mocks base method.
+func (m *MockFullNode) NetProtectList(arg0 context.Context) ([]peer.ID, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectList", arg0)
+	ret0, _ := ret[0].([]peer.ID)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetProtectList indicates an expected call of NetProtectList.
+func (mr *MockFullNodeMockRecorder) NetProtectList(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectList", reflect.TypeOf((*MockFullNode)(nil).NetProtectList), arg0)
+}
+
+// NetProtectRemove mocks base method.
+func (m *MockFullNode) NetProtectRemove(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectRemove", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectRemove indicates an expected call of NetProtectRemove.
+func (mr *MockFullNodeMockRecorder) NetProtectRemove(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectRemove", reflect.TypeOf((*MockFullNode)(nil).NetProtectRemove), arg0, arg1)
+}
+
 // NetPubsubScores mocks base method.
 func (m *MockFullNode) NetPubsubScores(arg0 context.Context) ([]api.PubsubScore, error) {
 	m.ctrl.T.Helper()
@@ -1769,6 +1827,35 @@ func (mr *MockFullNodeMockRecorder) NetPubsubScores(arg0 interface{}) *gomock.Ca
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPubsubScores", reflect.TypeOf((*MockFullNode)(nil).NetPubsubScores), arg0)
 }
 
+// NetSetLimit mocks base method.
+func (m *MockFullNode) NetSetLimit(arg0 context.Context, arg1 string, arg2 api.NetLimit) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetSetLimit", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetSetLimit indicates an expected call of NetSetLimit.
+func (mr *MockFullNodeMockRecorder) NetSetLimit(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetSetLimit", reflect.TypeOf((*MockFullNode)(nil).NetSetLimit), arg0, arg1, arg2)
+}
+
+// NetStat mocks base method.
+func (m *MockFullNode) NetStat(arg0 context.Context, arg1 string) (api.NetStat, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetStat", arg0, arg1)
+	ret0, _ := ret[0].(api.NetStat)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetStat indicates an expected call of NetStat.
+func (mr *MockFullNodeMockRecorder) NetStat(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetStat", reflect.TypeOf((*MockFullNode)(nil).NetStat), arg0, arg1)
+}
+
 // PaychAllocateLane mocks base method.
 func (m *MockFullNode) PaychAllocateLane(arg0 context.Context, arg1 address.Address) (uint64, error) {
 	m.ctrl.T.Helper()

api/v0api/v1_wrapper.go

@@ -337,4 +337,8 @@ func (w *WrapperV1Full) clientRetrieve(ctx context.Context, order RetrievalOrder
 	finish(w.ClientExport(ctx, eref, *ref))
 }
 
+func (w *WrapperV1Full) PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*api.ChannelInfo, error) {
+	return w.FullNode.PaychFund(ctx, from, to, amt)
+}
+
 var _ FullNode = &WrapperV1Full{}

api/version.go

@@ -57,8 +57,8 @@ var (
 	FullAPIVersion0 = newVer(1, 5, 0)
 	FullAPIVersion1 = newVer(2, 2, 0)
 
-	MinerAPIVersion0  = newVer(1, 4, 0)
-	WorkerAPIVersion0 = newVer(1, 5, 0)
+	MinerAPIVersion0  = newVer(1, 5, 0)
+	WorkerAPIVersion0 = newVer(1, 6, 0)
 )
 
 //nolint:varcheck,deadcode

blockstore/api.go

@@ -25,35 +25,35 @@ func NewAPIBlockstore(cio ChainIO) Blockstore {
 	return Adapt(bs) // return an adapted blockstore.
 }
 
-func (a *apiBlockstore) DeleteBlock(cid.Cid) error {
+func (a *apiBlockstore) DeleteBlock(context.Context, cid.Cid) error {
 	return xerrors.New("not supported")
 }
 
-func (a *apiBlockstore) Has(c cid.Cid) (bool, error) {
-	return a.api.ChainHasObj(context.TODO(), c)
+func (a *apiBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
+	return a.api.ChainHasObj(ctx, c)
 }
 
-func (a *apiBlockstore) Get(c cid.Cid) (blocks.Block, error) {
-	bb, err := a.api.ChainReadObj(context.TODO(), c)
+func (a *apiBlockstore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
+	bb, err := a.api.ChainReadObj(ctx, c)
 	if err != nil {
 		return nil, err
 	}
 	return blocks.NewBlockWithCid(bb, c)
 }
 
-func (a *apiBlockstore) GetSize(c cid.Cid) (int, error) {
-	bb, err := a.api.ChainReadObj(context.TODO(), c)
+func (a *apiBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
+	bb, err := a.api.ChainReadObj(ctx, c)
 	if err != nil {
 		return 0, err
 	}
 	return len(bb), nil
 }
 
-func (a *apiBlockstore) Put(blocks.Block) error {
+func (a *apiBlockstore) Put(context.Context, blocks.Block) error {
 	return xerrors.New("not supported")
 }
 
-func (a *apiBlockstore) PutMany([]blocks.Block) error {
+func (a *apiBlockstore) PutMany(context.Context, []blocks.Block) error {
 	return xerrors.New("not supported")
 }
 

blockstore/autobatch.go

@@ -64,7 +64,7 @@ func NewAutobatch(ctx context.Context, backingBs Blockstore, bufferCapacity int)
 	return bs
 }
 
-func (bs *AutobatchBlockstore) Put(blk block.Block) error {
+func (bs *AutobatchBlockstore) Put(ctx context.Context, blk block.Block) error {
 	bs.stateLock.Lock()
 	defer bs.stateLock.Unlock()
 
@@ -94,19 +94,19 @@ func (bs *AutobatchBlockstore) flushWorker(ctx context.Context) {
 		case <-bs.flushCh:
 			// TODO: check if we _should_ actually flush. We could get a spurious wakeup
 			// here.
-			putErr := bs.doFlush(false)
+			putErr := bs.doFlush(ctx, false)
 			for putErr != nil {
 				select {
 				case <-ctx.Done():
 					return
 				case <-time.After(bs.flushRetryDelay):
 					autolog.Errorf("FLUSH ERRORED: %w, retrying after %v", putErr, bs.flushRetryDelay)
-					putErr = bs.doFlush(true)
+					putErr = bs.doFlush(ctx, true)
 				}
 			}
 		case <-ctx.Done():
 			// Do one last flush.
-			_ = bs.doFlush(false)
+			_ = bs.doFlush(ctx, false)
 			return
 		}
 	}
@@ -114,13 +114,13 @@ func (bs *AutobatchBlockstore) flushWorker(ctx context.Context) {
 
 // caller must NOT hold stateLock
 // set retryOnly to true to only retry a failed flush and not flush anything new.
-func (bs *AutobatchBlockstore) doFlush(retryOnly bool) error {
+func (bs *AutobatchBlockstore) doFlush(ctx context.Context, retryOnly bool) error {
 	bs.doFlushLock.Lock()
 	defer bs.doFlushLock.Unlock()
 
 	// If we failed to flush last time, try flushing again.
 	if bs.flushErr != nil {
-		bs.flushErr = bs.backingBs.PutMany(bs.flushingBatch.blockList)
+		bs.flushErr = bs.backingBs.PutMany(ctx, bs.flushingBatch.blockList)
 	}
 
 	// If we failed, or we're _only_ retrying, bail.
@@ -137,7 +137,7 @@ func (bs *AutobatchBlockstore) doFlush(retryOnly bool) error {
 	bs.stateLock.Unlock()
 
 	// And try to flush it.
-	bs.flushErr = bs.backingBs.PutMany(bs.flushingBatch.blockList)
+	bs.flushErr = bs.backingBs.PutMany(ctx, bs.flushingBatch.blockList)
 
 	// If we succeeded, reset the batch. Otherwise, we'll try again next time.
 	if bs.flushErr == nil {
@@ -151,7 +151,7 @@ func (bs *AutobatchBlockstore) doFlush(retryOnly bool) error {
 
 // caller must NOT hold stateLock
 func (bs *AutobatchBlockstore) Flush(ctx context.Context) error {
-	return bs.doFlush(false)
+	return bs.doFlush(ctx, false)
 }
 
 func (bs *AutobatchBlockstore) Shutdown(ctx context.Context) error {
@@ -169,9 +169,9 @@ func (bs *AutobatchBlockstore) Shutdown(ctx context.Context) error {
 	return bs.flushErr
 }
 
-func (bs *AutobatchBlockstore) Get(c cid.Cid) (block.Block, error) {
+func (bs *AutobatchBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block, error) {
 	// may seem backward to check the backingBs first, but that is the likeliest case
-	blk, err := bs.backingBs.Get(c)
+	blk, err := bs.backingBs.Get(ctx, c)
 	if err == nil {
 		return blk, nil
 	}
@@ -192,10 +192,10 @@ func (bs *AutobatchBlockstore) Get(c cid.Cid) (block.Block, error) {
 		return v, nil
 	}
 
-	return bs.Get(c)
+	return bs.Get(ctx, c)
 }
 
-func (bs *AutobatchBlockstore) DeleteBlock(cid.Cid) error {
+func (bs *AutobatchBlockstore) DeleteBlock(context.Context, cid.Cid) error {
 	// if we wanted to support this, we would have to:
 	// - flush
 	// - delete from the backingBs (if present)
@@ -204,13 +204,13 @@ func (bs *AutobatchBlockstore) DeleteBlock(cid.Cid) error {
 	return xerrors.New("deletion is unsupported")
 }
 
-func (bs *AutobatchBlockstore) DeleteMany(cids []cid.Cid) error {
+func (bs *AutobatchBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
 	// see note in DeleteBlock()
 	return xerrors.New("deletion is unsupported")
 }
 
-func (bs *AutobatchBlockstore) Has(c cid.Cid) (bool, error) {
-	_, err := bs.Get(c)
+func (bs *AutobatchBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
+	_, err := bs.Get(ctx, c)
 	if err == nil {
 		return true, nil
 	}
@@ -221,8 +221,8 @@ func (bs *AutobatchBlockstore) Has(c cid.Cid) (bool, error) {
 	return false, err
 }
 
-func (bs *AutobatchBlockstore) GetSize(c cid.Cid) (int, error) {
-	blk, err := bs.Get(c)
+func (bs *AutobatchBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
+	blk, err := bs.Get(ctx, c)
 	if err != nil {
 		return 0, err
 	}
@@ -230,9 +230,9 @@ func (bs *AutobatchBlockstore) GetSize(c cid.Cid) (int, error) {
 	return len(blk.RawData()), nil
 }
 
-func (bs *AutobatchBlockstore) PutMany(blks []block.Block) error {
+func (bs *AutobatchBlockstore) PutMany(ctx context.Context, blks []block.Block) error {
 	for _, blk := range blks {
-		if err := bs.Put(blk); err != nil {
+		if err := bs.Put(ctx, blk); err != nil {
 			return err
 		}
 	}
@@ -252,8 +252,8 @@ func (bs *AutobatchBlockstore) HashOnRead(enabled bool) {
 	bs.backingBs.HashOnRead(enabled)
 }
 
-func (bs *AutobatchBlockstore) View(cid cid.Cid, callback func([]byte) error) error {
-	blk, err := bs.Get(cid)
+func (bs *AutobatchBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error {
+	blk, err := bs.Get(ctx, cid)
 	if err != nil {
 		return err
 	}

blockstore/autobatch_test.go

@@ -13,19 +13,19 @@ func TestAutobatchBlockstore(t *testing.T) {
 
 	ab := NewAutobatch(ctx, NewMemory(), len(b0.RawData())+len(b1.RawData())-1)
 
-	require.NoError(t, ab.Put(b0))
-	require.NoError(t, ab.Put(b1))
-	require.NoError(t, ab.Put(b2))
+	require.NoError(t, ab.Put(ctx, b0))
+	require.NoError(t, ab.Put(ctx, b1))
+	require.NoError(t, ab.Put(ctx, b2))
 
-	v0, err := ab.Get(b0.Cid())
+	v0, err := ab.Get(ctx, b0.Cid())
 	require.NoError(t, err)
 	require.Equal(t, b0.RawData(), v0.RawData())
 
-	v1, err := ab.Get(b1.Cid())
+	v1, err := ab.Get(ctx, b1.Cid())
 	require.NoError(t, err)
 	require.Equal(t, b1.RawData(), v1.RawData())
 
-	v2, err := ab.Get(b2.Cid())
+	v2, err := ab.Get(ctx, b2.Cid())
 	require.NoError(t, err)
 	require.Equal(t, b2.RawData(), v2.RawData())
 

blockstore/badger/blockstore.go

@@ -525,7 +525,7 @@ func (b *Blockstore) Size() (int64, error) {
 
 // View implements blockstore.Viewer, which leverages zero-copy read-only
 // access to values.
-func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
+func (b *Blockstore) View(ctx context.Context, cid cid.Cid, fn func([]byte) error) error {
 	if err := b.access(); err != nil {
 		return err
 	}
@@ -552,7 +552,7 @@ func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
 }
 
 // Has implements Blockstore.Has.
-func (b *Blockstore) Has(cid cid.Cid) (bool, error) {
+func (b *Blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 	if err := b.access(); err != nil {
 		return false, err
 	}
@@ -582,7 +582,7 @@ func (b *Blockstore) Has(cid cid.Cid) (bool, error) {
 }
 
 // Get implements Blockstore.Get.
-func (b *Blockstore) Get(cid cid.Cid) (blocks.Block, error) {
+func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
 	if !cid.Defined() {
 		return nil, blockstore.ErrNotFound
 	}
@@ -619,7 +619,7 @@ func (b *Blockstore) Get(cid cid.Cid) (blocks.Block, error) {
 }
 
 // GetSize implements Blockstore.GetSize.
-func (b *Blockstore) GetSize(cid cid.Cid) (int, error) {
+func (b *Blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
 	if err := b.access(); err != nil {
 		return 0, err
 	}
@@ -652,7 +652,7 @@ func (b *Blockstore) GetSize(cid cid.Cid) (int, error) {
 }
 
 // Put implements Blockstore.Put.
-func (b *Blockstore) Put(block blocks.Block) error {
+func (b *Blockstore) Put(ctx context.Context, block blocks.Block) error {
 	if err := b.access(); err != nil {
 		return err
 	}
@@ -691,7 +691,7 @@ func (b *Blockstore) Put(block blocks.Block) error {
 }
 
 // PutMany implements Blockstore.PutMany.
-func (b *Blockstore) PutMany(blocks []blocks.Block) error {
+func (b *Blockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
 	if err := b.access(); err != nil {
 		return err
 	}
@@ -755,7 +755,7 @@ func (b *Blockstore) PutMany(blocks []blocks.Block) error {
 }
 
 // DeleteBlock implements Blockstore.DeleteBlock.
-func (b *Blockstore) DeleteBlock(cid cid.Cid) error {
+func (b *Blockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
 	if err := b.access(); err != nil {
 		return err
 	}
@@ -774,7 +774,7 @@ func (b *Blockstore) DeleteBlock(cid cid.Cid) error {
 	})
 }
 
-func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
+func (b *Blockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
 	if err := b.access(); err != nil {
 		return err
 	}

blockstore/badger/blockstore_test.go

@@ -2,6 +2,7 @@ package badgerbs
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -98,6 +99,7 @@ func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB,
 }
 
 func testMove(t *testing.T, optsF func(string) Options) {
+	ctx := context.Background()
 	basePath, err := ioutil.TempDir("", "")
 	if err != nil {
 		t.Fatal(err)
@@ -122,7 +124,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
 	// add some blocks
 	for i := 0; i < 10; i++ {
 		blk := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
-		err := db.Put(blk)
+		err := db.Put(ctx, blk)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -132,7 +134,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
 	// delete some of them
 	for i := 5; i < 10; i++ {
 		c := have[i].Cid()
-		err := db.DeleteBlock(c)
+		err := db.DeleteBlock(ctx, c)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -145,7 +147,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
 	g.Go(func() error {
 		for i := 10; i < 1000; i++ {
 			blk := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
-			err := db.Put(blk)
+			err := db.Put(ctx, blk)
 			if err != nil {
 				return err
 			}
@@ -165,7 +167,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
 	// now check that we have all the blocks in have and none in the deleted lists
 	checkBlocks := func() {
 		for _, blk := range have {
-			has, err := db.Has(blk.Cid())
+			has, err := db.Has(ctx, blk.Cid())
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -174,7 +176,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
 				t.Fatal("missing block")
 			}
 
-			blk2, err := db.Get(blk.Cid())
+			blk2, err := db.Get(ctx, blk.Cid())
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -185,7 +187,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
 		}
 
 		for _, c := range deleted {
-			has, err := db.Has(c)
+			has, err := db.Has(ctx, c)
 			if err != nil {
 				t.Fatal(err)
 			}

blockstore/badger/blockstore_test_suite.go

@@ -44,28 +44,31 @@ func (s *Suite) RunTests(t *testing.T, prefix string) {
 }
 
 func (s *Suite) TestGetWhenKeyNotPresent(t *testing.T) {
+	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
 	}
 
 	c := cid.NewCidV0(u.Hash([]byte("stuff")))
-	bl, err := bs.Get(c)
+	bl, err := bs.Get(ctx, c)
 	require.Nil(t, bl)
 	require.Equal(t, blockstore.ErrNotFound, err)
 }
 
 func (s *Suite) TestGetWhenKeyIsNil(t *testing.T) {
+	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
 	}
 
-	_, err := bs.Get(cid.Undef)
+	_, err := bs.Get(ctx, cid.Undef)
 	require.Equal(t, blockstore.ErrNotFound, err)
 }
 
 func (s *Suite) TestPutThenGetBlock(t *testing.T) {
+	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
@@ -73,15 +76,16 @@ func (s *Suite) TestPutThenGetBlock(t *testing.T) {
 
 	orig := blocks.NewBlock([]byte("some data"))
 
-	err := bs.Put(orig)
+	err := bs.Put(ctx, orig)
 	require.NoError(t, err)
 
-	fetched, err := bs.Get(orig.Cid())
+	fetched, err := bs.Get(ctx, orig.Cid())
 	require.NoError(t, err)
 	require.Equal(t, orig.RawData(), fetched.RawData())
 }
 
 func (s *Suite) TestHas(t *testing.T) {
+	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
@@ -89,19 +93,20 @@ func (s *Suite) TestHas(t *testing.T) {
 
 	orig := blocks.NewBlock([]byte("some data"))
 
-	err := bs.Put(orig)
+	err := bs.Put(ctx, orig)
 	require.NoError(t, err)
 
-	ok, err := bs.Has(orig.Cid())
+	ok, err := bs.Has(ctx, orig.Cid())
 	require.NoError(t, err)
 	require.True(t, ok)
 
-	ok, err = bs.Has(blocks.NewBlock([]byte("another thing")).Cid())
+	ok, err = bs.Has(ctx, blocks.NewBlock([]byte("another thing")).Cid())
 	require.NoError(t, err)
 	require.False(t, ok)
 }
 
 func (s *Suite) TestCidv0v1(t *testing.T) {
+	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
@@ -109,15 +114,17 @@ func (s *Suite) TestCidv0v1(t *testing.T) {
 
 	orig := blocks.NewBlock([]byte("some data"))
 
-	err := bs.Put(orig)
+	err := bs.Put(ctx, orig)
 	require.NoError(t, err)
 
-	fetched, err := bs.Get(cid.NewCidV1(cid.DagProtobuf, orig.Cid().Hash()))
+	fetched, err := bs.Get(ctx, cid.NewCidV1(cid.DagProtobuf, orig.Cid().Hash()))
 	require.NoError(t, err)
 	require.Equal(t, orig.RawData(), fetched.RawData())
 }
 
 func (s *Suite) TestPutThenGetSizeBlock(t *testing.T) {
+	ctx := context.Background()
+
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
@@ -127,21 +134,21 @@ func (s *Suite) TestPutThenGetSizeBlock(t *testing.T) {
 	missingBlock := blocks.NewBlock([]byte("missingBlock"))
 	emptyBlock := blocks.NewBlock([]byte{})
 
-	err := bs.Put(block)
+	err := bs.Put(ctx, block)
 	require.NoError(t, err)
 
-	blockSize, err := bs.GetSize(block.Cid())
+	blockSize, err := bs.GetSize(ctx, block.Cid())
 	require.NoError(t, err)
 	require.Len(t, block.RawData(), blockSize)
 
-	err = bs.Put(emptyBlock)
+	err = bs.Put(ctx, emptyBlock)
 	require.NoError(t, err)
 
-	emptySize, err := bs.GetSize(emptyBlock.Cid())
+	emptySize, err := bs.GetSize(ctx, emptyBlock.Cid())
 	require.NoError(t, err)
 	require.Zero(t, emptySize)
 
-	missingSize, err := bs.GetSize(missingBlock.Cid())
+	missingSize, err := bs.GetSize(ctx, missingBlock.Cid())
 	require.Equal(t, blockstore.ErrNotFound, err)
 	require.Equal(t, -1, missingSize)
 }
@@ -203,6 +210,7 @@ func (s *Suite) TestDoubleClose(t *testing.T) {
 }
 
 func (s *Suite) TestReopenPutGet(t *testing.T) {
+	ctx := context.Background()
 	bs, path := s.NewBlockstore(t)
 	c, ok := bs.(io.Closer)
 	if !ok {
@@ -210,7 +218,7 @@ func (s *Suite) TestReopenPutGet(t *testing.T) {
 	}
 
 	orig := blocks.NewBlock([]byte("some data"))
-	err := bs.Put(orig)
+	err := bs.Put(ctx, orig)
 	require.NoError(t, err)
 
 	err = c.Close()
@@ -219,7 +227,7 @@ func (s *Suite) TestReopenPutGet(t *testing.T) {
 	bs, err = s.OpenBlockstore(t, path)
 	require.NoError(t, err)
 
-	fetched, err := bs.Get(orig.Cid())
+	fetched, err := bs.Get(ctx, orig.Cid())
 	require.NoError(t, err)
 	require.Equal(t, orig.RawData(), fetched.RawData())
 
@@ -228,6 +236,7 @@ func (s *Suite) TestReopenPutGet(t *testing.T) {
 }
 
 func (s *Suite) TestPutMany(t *testing.T) {
+	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
@@ -238,15 +247,15 @@ func (s *Suite) TestPutMany(t *testing.T) {
 		blocks.NewBlock([]byte("foo2")),
 		blocks.NewBlock([]byte("foo3")),
 	}
-	err := bs.PutMany(blks)
+	err := bs.PutMany(ctx, blks)
 	require.NoError(t, err)
 
 	for _, blk := range blks {
-		fetched, err := bs.Get(blk.Cid())
+		fetched, err := bs.Get(ctx, blk.Cid())
 		require.NoError(t, err)
 		require.Equal(t, blk.RawData(), fetched.RawData())
 
-		ok, err := bs.Has(blk.Cid())
+		ok, err := bs.Has(ctx, blk.Cid())
 		require.NoError(t, err)
 		require.True(t, ok)
 	}
@@ -259,6 +268,7 @@ func (s *Suite) TestPutMany(t *testing.T) {
 }
 
 func (s *Suite) TestDelete(t *testing.T) {
+	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
@@ -269,10 +279,10 @@ func (s *Suite) TestDelete(t *testing.T) {
 		blocks.NewBlock([]byte("foo2")),
 		blocks.NewBlock([]byte("foo3")),
 	}
-	err := bs.PutMany(blks)
+	err := bs.PutMany(ctx, blks)
 	require.NoError(t, err)
 
-	err = bs.DeleteBlock(blks[1].Cid())
+	err = bs.DeleteBlock(ctx, blks[1].Cid())
 	require.NoError(t, err)
 
 	ch, err := bs.AllKeysChan(context.Background())
@@ -285,17 +295,17 @@ func (s *Suite) TestDelete(t *testing.T) {
 		cid.NewCidV1(cid.Raw, blks[2].Cid().Hash()),
 	})
 
-	has, err := bs.Has(blks[1].Cid())
+	has, err := bs.Has(ctx, blks[1].Cid())
 	require.NoError(t, err)
 	require.False(t, has)
-
 }
 
 func insertBlocks(t *testing.T, bs blockstore.BasicBlockstore, count int) []cid.Cid {
+	ctx := context.Background()
 	keys := make([]cid.Cid, count)
 	for i := 0; i < count; i++ {
 		block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
-		err := bs.Put(block)
+		err := bs.Put(ctx, block)
 		require.NoError(t, err)
 		// NewBlock assigns a CIDv0; we convert it to CIDv1 because that's what
 		// the store returns.

blockstore/blockstore.go

@@ -1,6 +1,8 @@
 package blockstore
 
 import (
+	"context"
+
 	cid "github.com/ipfs/go-cid"
 	ds "github.com/ipfs/go-datastore"
 	logging "github.com/ipfs/go-log/v2"
@@ -27,7 +29,7 @@ type BasicBlockstore = blockstore.Blockstore
 type Viewer = blockstore.Viewer
 
 type BatchDeleter interface {
-	DeleteMany(cids []cid.Cid) error
+	DeleteMany(ctx context.Context, cids []cid.Cid) error
 }
 
 // BlockstoreIterator is a trait for efficient iteration
@@ -93,17 +95,17 @@ type adaptedBlockstore struct {
 
 var _ Blockstore = (*adaptedBlockstore)(nil)
 
-func (a *adaptedBlockstore) View(cid cid.Cid, callback func([]byte) error) error {
-	blk, err := a.Get(cid)
+func (a *adaptedBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error {
+	blk, err := a.Get(ctx, cid)
 	if err != nil {
 		return err
 	}
 	return callback(blk.RawData())
 }
 
-func (a *adaptedBlockstore) DeleteMany(cids []cid.Cid) error {
+func (a *adaptedBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
 	for _, cid := range cids {
-		err := a.DeleteBlock(cid)
+		err := a.DeleteBlock(ctx, cid)
 		if err != nil {
 			return err
 		}

blockstore/buffered.go

@@ -88,34 +88,34 @@ func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid,
 	return out, nil
 }
 
-func (bs *BufferedBlockstore) DeleteBlock(c cid.Cid) error {
-	if err := bs.read.DeleteBlock(c); err != nil {
+func (bs *BufferedBlockstore) DeleteBlock(ctx context.Context, c cid.Cid) error {
+	if err := bs.read.DeleteBlock(ctx, c); err != nil {
 		return err
 	}
 
-	return bs.write.DeleteBlock(c)
+	return bs.write.DeleteBlock(ctx, c)
 }
 
-func (bs *BufferedBlockstore) DeleteMany(cids []cid.Cid) error {
-	if err := bs.read.DeleteMany(cids); err != nil {
+func (bs *BufferedBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
+	if err := bs.read.DeleteMany(ctx, cids); err != nil {
 		return err
 	}
 
-	return bs.write.DeleteMany(cids)
+	return bs.write.DeleteMany(ctx, cids)
 }
 
-func (bs *BufferedBlockstore) View(c cid.Cid, callback func([]byte) error) error {
+func (bs *BufferedBlockstore) View(ctx context.Context, c cid.Cid, callback func([]byte) error) error {
 	// both stores are viewable.
-	if err := bs.write.View(c, callback); err == ErrNotFound {
+	if err := bs.write.View(ctx, c, callback); err == ErrNotFound {
 		// not found in write blockstore; fall through.
 	} else {
 		return err // propagate errors, or nil, i.e. found.
 	}
-	return bs.read.View(c, callback)
+	return bs.read.View(ctx, c, callback)
 }
 
-func (bs *BufferedBlockstore) Get(c cid.Cid) (block.Block, error) {
-	if out, err := bs.write.Get(c); err != nil {
+func (bs *BufferedBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block, error) {
+	if out, err := bs.write.Get(ctx, c); err != nil {
 		if err != ErrNotFound {
 			return nil, err
 		}
@@ -123,20 +123,20 @@ func (bs *BufferedBlockstore) Get(c cid.Cid) (block.Block, error) {
 		return out, nil
 	}
 
-	return bs.read.Get(c)
+	return bs.read.Get(ctx, c)
 }
 
-func (bs *BufferedBlockstore) GetSize(c cid.Cid) (int, error) {
-	s, err := bs.read.GetSize(c)
+func (bs *BufferedBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
+	s, err := bs.read.GetSize(ctx, c)
 	if err == ErrNotFound || s == 0 {
-		return bs.write.GetSize(c)
+		return bs.write.GetSize(ctx, c)
 	}
 
 	return s, err
 }
 
-func (bs *BufferedBlockstore) Put(blk block.Block) error {
-	has, err := bs.read.Has(blk.Cid()) // TODO: consider dropping this check
+func (bs *BufferedBlockstore) Put(ctx context.Context, blk block.Block) error {
+	has, err := bs.read.Has(ctx, blk.Cid()) // TODO: consider dropping this check
 	if err != nil {
 		return err
 	}
@@ -145,11 +145,11 @@ func (bs *BufferedBlockstore) Put(blk block.Block) error {
 		return nil
 	}
 
-	return bs.write.Put(blk)
+	return bs.write.Put(ctx, blk)
 }
 
-func (bs *BufferedBlockstore) Has(c cid.Cid) (bool, error) {
-	has, err := bs.write.Has(c)
+func (bs *BufferedBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
+	has, err := bs.write.Has(ctx, c)
 	if err != nil {
 		return false, err
 	}
@@ -157,7 +157,7 @@ func (bs *BufferedBlockstore) Has(c cid.Cid) (bool, error) {
 		return true, nil
 	}
 
-	return bs.read.Has(c)
+	return bs.read.Has(ctx, c)
 }
 
 func (bs *BufferedBlockstore) HashOnRead(hor bool) {
@@ -165,8 +165,8 @@ func (bs *BufferedBlockstore) HashOnRead(hor bool) {
 	bs.write.HashOnRead(hor)
 }
 
-func (bs *BufferedBlockstore) PutMany(blks []block.Block) error {
-	return bs.write.PutMany(blks)
+func (bs *BufferedBlockstore) PutMany(ctx context.Context, blks []block.Block) error {
+	return bs.write.PutMany(ctx, blks)
 }
 
 func (bs *BufferedBlockstore) Read() Blockstore {

blockstore/context.go

@@ -0,0 +1,21 @@
+package blockstore
+
+import (
+	"context"
+)
+
+type hotViewKey struct{}
+
+var hotView = hotViewKey{}
+
+// WithHotView constructs a new context with an option that provides a hint to the blockstore
+// (e.g. the splitstore) that the object (and its ipld references) should be kept hot.
+func WithHotView(ctx context.Context) context.Context {
+	return context.WithValue(ctx, hotView, struct{}{})
+}
+
+// IsHotView returns true if the hot view option is set in the context
+func IsHotView(ctx context.Context) bool {
+	v := ctx.Value(hotView)
+	return v != nil
+}

blockstore/discard.go

@@ -18,39 +18,39 @@ func NewDiscardStore(bs Blockstore) Blockstore {
 	return &discardstore{bs: bs}
 }
 
-func (b *discardstore) Has(cid cid.Cid) (bool, error) {
-	return b.bs.Has(cid)
+func (b *discardstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
+	return b.bs.Has(ctx, cid)
 }
 
 func (b *discardstore) HashOnRead(hor bool) {
 	b.bs.HashOnRead(hor)
 }
 
-func (b *discardstore) Get(cid cid.Cid) (blocks.Block, error) {
-	return b.bs.Get(cid)
+func (b *discardstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
+	return b.bs.Get(ctx, cid)
 }
 
-func (b *discardstore) GetSize(cid cid.Cid) (int, error) {
-	return b.bs.GetSize(cid)
+func (b *discardstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
+	return b.bs.GetSize(ctx, cid)
 }
 
-func (b *discardstore) View(cid cid.Cid, f func([]byte) error) error {
-	return b.bs.View(cid, f)
+func (b *discardstore) View(ctx context.Context, cid cid.Cid, f func([]byte) error) error {
+	return b.bs.View(ctx, cid, f)
 }
 
-func (b *discardstore) Put(blk blocks.Block) error {
+func (b *discardstore) Put(ctx context.Context, blk blocks.Block) error {
 	return nil
 }
 
-func (b *discardstore) PutMany(blks []blocks.Block) error {
+func (b *discardstore) PutMany(ctx context.Context, blks []blocks.Block) error {
 	return nil
 }
 
-func (b *discardstore) DeleteBlock(cid cid.Cid) error {
+func (b *discardstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
 	return nil
 }
 
-func (b *discardstore) DeleteMany(cids []cid.Cid) error {
+func (b *discardstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
 	return nil
 }
 

blockstore/fallback.go

@@ -71,14 +71,14 @@ func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
 	// chain bitswap puts blocks in temp blockstore which is cleaned up
 	// every few min (to drop any messages we fetched but don't want)
 	// in this case we want to keep this block around
-	if err := fbs.Put(b); err != nil {
+	if err := fbs.Put(ctx, b); err != nil {
 		return nil, xerrors.Errorf("persisting fallback-fetched block: %w", err)
 	}
 	return b, nil
 }
 
-func (fbs *FallbackStore) Get(c cid.Cid) (blocks.Block, error) {
-	b, err := fbs.Blockstore.Get(c)
+func (fbs *FallbackStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
+	b, err := fbs.Blockstore.Get(ctx, c)
 	switch err {
 	case nil:
 		return b, nil
@@ -89,8 +89,8 @@ func (fbs *FallbackStore) Get(c cid.Cid) (blocks.Block, error) {
 	}
 }
 
-func (fbs *FallbackStore) GetSize(c cid.Cid) (int, error) {
-	sz, err := fbs.Blockstore.GetSize(c)
+func (fbs *FallbackStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
+	sz, err := fbs.Blockstore.GetSize(ctx, c)
 	switch err {
 	case nil:
 		return sz, nil

blockstore/idstore.go

@@ -38,7 +38,7 @@ func decodeCid(cid cid.Cid) (inline bool, data []byte, err error) {
 	return false, nil, err
 }
 
-func (b *idstore) Has(cid cid.Cid) (bool, error) {
+func (b *idstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 	inline, _, err := decodeCid(cid)
 	if err != nil {
 		return false, xerrors.Errorf("error decoding Cid: %w", err)
@@ -48,10 +48,10 @@ func (b *idstore) Has(cid cid.Cid) (bool, error) {
 		return true, nil
 	}
 
-	return b.bs.Has(cid)
+	return b.bs.Has(ctx, cid)
 }
 
-func (b *idstore) Get(cid cid.Cid) (blocks.Block, error) {
+func (b *idstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
 	inline, data, err := decodeCid(cid)
 	if err != nil {
 		return nil, xerrors.Errorf("error decoding Cid: %w", err)
@@ -61,10 +61,10 @@ func (b *idstore) Get(cid cid.Cid) (blocks.Block, error) {
 		return blocks.NewBlockWithCid(data, cid)
 	}
 
-	return b.bs.Get(cid)
+	return b.bs.Get(ctx, cid)
 }
 
-func (b *idstore) GetSize(cid cid.Cid) (int, error) {
+func (b *idstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
 	inline, data, err := decodeCid(cid)
 	if err != nil {
 		return 0, xerrors.Errorf("error decoding Cid: %w", err)
@@ -74,10 +74,10 @@ func (b *idstore) GetSize(cid cid.Cid) (int, error) {
 		return len(data), err
 	}
 
-	return b.bs.GetSize(cid)
+	return b.bs.GetSize(ctx, cid)
 }
 
-func (b *idstore) View(cid cid.Cid, cb func([]byte) error) error {
+func (b *idstore) View(ctx context.Context, cid cid.Cid, cb func([]byte) error) error {
 	inline, data, err := decodeCid(cid)
 	if err != nil {
 		return xerrors.Errorf("error decoding Cid: %w", err)
@@ -87,10 +87,10 @@ func (b *idstore) View(cid cid.Cid, cb func([]byte) error) error {
 		return cb(data)
 	}
 
-	return b.bs.View(cid, cb)
+	return b.bs.View(ctx, cid, cb)
 }
 
-func (b *idstore) Put(blk blocks.Block) error {
+func (b *idstore) Put(ctx context.Context, blk blocks.Block) error {
 	inline, _, err := decodeCid(blk.Cid())
 	if err != nil {
 		return xerrors.Errorf("error decoding Cid: %w", err)
@@ -100,10 +100,10 @@ func (b *idstore) Put(blk blocks.Block) error {
 		return nil
 	}
 
-	return b.bs.Put(blk)
+	return b.bs.Put(ctx, blk)
 }
 
-func (b *idstore) PutMany(blks []blocks.Block) error {
+func (b *idstore) PutMany(ctx context.Context, blks []blocks.Block) error {
 	toPut := make([]blocks.Block, 0, len(blks))
 	for _, blk := range blks {
 		inline, _, err := decodeCid(blk.Cid())
@@ -118,13 +118,13 @@ func (b *idstore) PutMany(blks []blocks.Block) error {
 	}
 
 	if len(toPut) > 0 {
-		return b.bs.PutMany(toPut)
+		return b.bs.PutMany(ctx, toPut)
 	}
 
 	return nil
 }
 
-func (b *idstore) DeleteBlock(cid cid.Cid) error {
+func (b *idstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
 	inline, _, err := decodeCid(cid)
 	if err != nil {
 		return xerrors.Errorf("error decoding Cid: %w", err)
@@ -134,10 +134,10 @@ func (b *idstore) DeleteBlock(cid cid.Cid) error {
 		return nil
 	}
 
-	return b.bs.DeleteBlock(cid)
+	return b.bs.DeleteBlock(ctx, cid)
 }
 
-func (b *idstore) DeleteMany(cids []cid.Cid) error {
+func (b *idstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
 	toDelete := make([]cid.Cid, 0, len(cids))
 	for _, cid := range cids {
 		inline, _, err := decodeCid(cid)
@@ -152,7 +152,7 @@ func (b *idstore) DeleteMany(cids []cid.Cid) error {
 	}
 
 	if len(toDelete) > 0 {
-		return b.bs.DeleteMany(toDelete)
+		return b.bs.DeleteMany(ctx, toDelete)
 	}
 
 	return nil

blockstore/ipfs.go

@@ -79,12 +79,12 @@ func NewRemoteIPFSBlockstore(ctx context.Context, maddr multiaddr.Multiaddr, onl
 	return Adapt(bs), nil
 }
 
-func (i *IPFSBlockstore) DeleteBlock(cid cid.Cid) error {
+func (i *IPFSBlockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
 	return xerrors.Errorf("not supported")
 }
 
-func (i *IPFSBlockstore) Has(cid cid.Cid) (bool, error) {
-	_, err := i.offlineAPI.Block().Stat(i.ctx, path.IpldPath(cid))
+func (i *IPFSBlockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
+	_, err := i.offlineAPI.Block().Stat(ctx, path.IpldPath(cid))
 	if err != nil {
 		// The underlying client is running in Offline mode.
 		// Stat() will fail with an err if the block isn't in the
@@ -99,8 +99,8 @@ func (i *IPFSBlockstore) Has(cid cid.Cid) (bool, error) {
 	return true, nil
 }
 
-func (i *IPFSBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
-	rd, err := i.api.Block().Get(i.ctx, path.IpldPath(cid))
+func (i *IPFSBlockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
+	rd, err := i.api.Block().Get(ctx, path.IpldPath(cid))
 	if err != nil {
 		return nil, xerrors.Errorf("getting ipfs block: %w", err)
 	}
@@ -113,8 +113,8 @@ func (i *IPFSBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
 	return blocks.NewBlockWithCid(data, cid)
 }
 
-func (i *IPFSBlockstore) GetSize(cid cid.Cid) (int, error) {
-	st, err := i.api.Block().Stat(i.ctx, path.IpldPath(cid))
+func (i *IPFSBlockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
+	st, err := i.api.Block().Stat(ctx, path.IpldPath(cid))
 	if err != nil {
 		return 0, xerrors.Errorf("getting ipfs block: %w", err)
 	}
@@ -122,23 +122,23 @@ func (i *IPFSBlockstore) GetSize(cid cid.Cid) (int, error) {
 	return st.Size(), nil
 }
 
-func (i *IPFSBlockstore) Put(block blocks.Block) error {
+func (i *IPFSBlockstore) Put(ctx context.Context, block blocks.Block) error {
 	mhd, err := multihash.Decode(block.Cid().Hash())
 	if err != nil {
 		return err
 	}
 
-	_, err = i.api.Block().Put(i.ctx, bytes.NewReader(block.RawData()),
+	_, err = i.api.Block().Put(ctx, bytes.NewReader(block.RawData()),
 		options.Block.Hash(mhd.Code, mhd.Length),
 		options.Block.Format(cid.CodecToStr[block.Cid().Type()]))
 	return err
 }
 
-func (i *IPFSBlockstore) PutMany(blocks []blocks.Block) error {
+func (i *IPFSBlockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
 	// TODO: could be done in parallel
 
 	for _, block := range blocks {
-		if err := i.Put(block); err != nil {
+		if err := i.Put(ctx, block); err != nil {
 			return err
 		}
 	}

blockstore/mem.go

@@ -15,24 +15,24 @@ func NewMemory() MemBlockstore {
 // MemBlockstore is a terminal blockstore that keeps blocks in memory.
 type MemBlockstore map[cid.Cid]blocks.Block
 
-func (m MemBlockstore) DeleteBlock(k cid.Cid) error {
+func (m MemBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
 	delete(m, k)
 	return nil
 }
 
-func (m MemBlockstore) DeleteMany(ks []cid.Cid) error {
+func (m MemBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error {
 	for _, k := range ks {
 		delete(m, k)
 	}
 	return nil
 }
 
-func (m MemBlockstore) Has(k cid.Cid) (bool, error) {
+func (m MemBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error) {
 	_, ok := m[k]
 	return ok, nil
 }
 
-func (m MemBlockstore) View(k cid.Cid, callback func([]byte) error) error {
+func (m MemBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error {
 	b, ok := m[k]
 	if !ok {
 		return ErrNotFound
@@ -40,7 +40,7 @@ func (m MemBlockstore) View(k cid.Cid, callback func([]byte) error) error {
 	return callback(b.RawData())
 }
 
-func (m MemBlockstore) Get(k cid.Cid) (blocks.Block, error) {
+func (m MemBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) {
 	b, ok := m[k]
 	if !ok {
 		return nil, ErrNotFound
@@ -49,7 +49,7 @@ func (m MemBlockstore) Get(k cid.Cid) (blocks.Block, error) {
 }
 
 // GetSize returns the CIDs mapped BlockSize
-func (m MemBlockstore) GetSize(k cid.Cid) (int, error) {
+func (m MemBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
 	b, ok := m[k]
 	if !ok {
 		return 0, ErrNotFound
@@ -58,7 +58,7 @@ func (m MemBlockstore) GetSize(k cid.Cid) (int, error) {
 }
 
 // Put puts a given block to the underlying datastore
-func (m MemBlockstore) Put(b blocks.Block) error {
+func (m MemBlockstore) Put(ctx context.Context, b blocks.Block) error {
 	// Convert to a basic block for safety, but try to reuse the existing
 	// block if it's already a basic block.
 	k := b.Cid()
@@ -76,9 +76,9 @@ func (m MemBlockstore) Put(b blocks.Block) error {
 
 // PutMany puts a slice of blocks at the same time using batching
 // capabilities of the underlying datastore whenever possible.
-func (m MemBlockstore) PutMany(bs []blocks.Block) error {
+func (m MemBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
 	for _, b := range bs {
-		_ = m.Put(b) // can't fail
+		_ = m.Put(ctx, b) // can't fail
 	}
 	return nil
 }

blockstore/splitstore/README.md

@@ -49,10 +49,11 @@ These are options in the `[Chainstore.Splitstore]` section of the configuration:
   blockstore and discards writes; this is necessary to support syncing from a snapshot.
 - `MarkSetType` -- specifies the type of markset to use during compaction.
   The markset is the data structure used by compaction/gc to track live objects.
-  The default value is `"map"`, which will use an in-memory map; if you are limited
-  in memory (or indeed see compaction run out of memory), you can also specify
-  `"badger"` which will use an disk backed markset, using badger. This will use
-  much less memory, but will also make compaction slower.
+  The default value is "badger", which will use a disk backed markset using badger.
+  If you have a lot of memory (48G or more) you can also use "map", which will use
+  an in memory markset, speeding up compaction at the cost of higher memory usage.
+  Note: If you are using a VPS with a network volume, you need to provision at least
+  3000 IOPs with the badger markset.
 - `HotStoreMessageRetention` -- specifies how many finalities, beyond the 4
   finalities maintained by default, to maintain messages and message receipts in the
   hotstore. This is useful for assistive nodes that want to support syncing for other
@@ -105,6 +106,12 @@ Compaction works transactionally with the following algorithm:
   - We delete in small batches taking a lock; each batch is checked again for marks, from the concurrent transactional mark, so as to never delete anything live
 - We then end the transaction and compact/gc the hotstore.
 
+As of [#8008](https://github.com/filecoin-project/lotus/pull/8008) the compaction algorithm has been
+modified to eliminate sorting and maintain the cold object set on disk. This drastically reduces
+memory usage; in fact, when using badger as the markset compaction uses very little memory, and
+it should be now possible to run splitstore with 32GB of RAM or less without danger of running out of
+memory during compaction.
+
 ## Garbage Collection
 
 TBD -- see [#6577](https://github.com/filecoin-project/lotus/issues/6577)

blockstore/splitstore/checkpoint.go

@@ -0,0 +1,118 @@
+package splitstore
+
+import (
+	"bufio"
+	"io"
+	"os"
+
+	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
+	mh "github.com/multiformats/go-multihash"
+)
+
+type Checkpoint struct {
+	file *os.File
+	buf  *bufio.Writer
+}
+
+func NewCheckpoint(path string) (*Checkpoint, error) {
+	file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_SYNC, 0644)
+	if err != nil {
+		return nil, xerrors.Errorf("error creating checkpoint: %w", err)
+	}
+	buf := bufio.NewWriter(file)
+
+	return &Checkpoint{
+		file: file,
+		buf:  buf,
+	}, nil
+}
+
+func OpenCheckpoint(path string) (*Checkpoint, cid.Cid, error) {
+	filein, err := os.Open(path)
+	if err != nil {
+		return nil, cid.Undef, xerrors.Errorf("error opening checkpoint for reading: %w", err)
+	}
+	defer filein.Close() //nolint:errcheck
+
+	bufin := bufio.NewReader(filein)
+	start, err := readRawCid(bufin, nil)
+	if err != nil && err != io.EOF {
+		return nil, cid.Undef, xerrors.Errorf("error reading cid from checkpoint: %w", err)
+	}
+
+	fileout, err := os.OpenFile(path, os.O_WRONLY|os.O_SYNC, 0644)
+	if err != nil {
+		return nil, cid.Undef, xerrors.Errorf("error opening checkpoint for writing: %w", err)
+	}
+	bufout := bufio.NewWriter(fileout)
+
+	return &Checkpoint{
+		file: fileout,
+		buf:  bufout,
+	}, start, nil
+}
+
+func (cp *Checkpoint) Set(c cid.Cid) error {
+	if _, err := cp.file.Seek(0, io.SeekStart); err != nil {
+		return xerrors.Errorf("error seeking beginning of checkpoint: %w", err)
+	}
+
+	if err := writeRawCid(cp.buf, c, true); err != nil {
+		return xerrors.Errorf("error writing cid to checkpoint: %w", err)
+	}
+
+	return nil
+}
+
+func (cp *Checkpoint) Close() error {
+	if cp.file == nil {
+		return nil
+	}
+
+	err := cp.file.Close()
+	cp.file = nil
+	cp.buf = nil
+
+	return err
+}
+
+func readRawCid(buf *bufio.Reader, hbuf []byte) (cid.Cid, error) {
+	sz, err := buf.ReadByte()
+	if err != nil {
+		return cid.Undef, err // don't wrap EOF as it is not an error here
+	}
+
+	if hbuf == nil {
+		hbuf = make([]byte, int(sz))
+	} else {
+		hbuf = hbuf[:int(sz)]
+	}
+
+	if _, err := io.ReadFull(buf, hbuf); err != nil {
+		return cid.Undef, xerrors.Errorf("error reading hash: %w", err) // wrap EOF, it's corrupt
+	}
+
+	hash, err := mh.Cast(hbuf)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("error casting multihash: %w", err)
+	}
+
+	return cid.NewCidV1(cid.Raw, hash), nil
+}
+
+func writeRawCid(buf *bufio.Writer, c cid.Cid, flush bool) error {
+	hash := c.Hash()
+	if err := buf.WriteByte(byte(len(hash))); err != nil {
+		return err
+	}
+	if _, err := buf.Write(hash); err != nil {
+		return err
+	}
+	if flush {
+		return buf.Flush()
+	}
+
+	return nil
+}

blockstore/splitstore/checkpoint_test.go

@@ -0,0 +1,147 @@
+package splitstore
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+)
+
+func TestCheckpoint(t *testing.T) {
+	dir, err := ioutil.TempDir("", "checkpoint.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(dir)
+	})
+
+	path := filepath.Join(dir, "checkpoint")
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	cp, err := NewCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Set(k1); err != nil {
+		t.Fatal(err)
+	}
+	if err := cp.Set(k2); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err := OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !start.Equals(k2) {
+		t.Fatalf("expected start to be %s; got %s", k2, start)
+	}
+
+	if err := cp.Set(k3); err != nil {
+		t.Fatal(err)
+	}
+	if err := cp.Set(k4); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err = OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !start.Equals(k4) {
+		t.Fatalf("expected start to be %s; got %s", k4, start)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// also test correct operation with an empty checkpoint
+	cp, err = NewCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err = OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if start.Defined() {
+		t.Fatal("expected start to be undefined")
+	}
+
+	if err := cp.Set(k1); err != nil {
+		t.Fatal(err)
+	}
+	if err := cp.Set(k2); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err = OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !start.Equals(k2) {
+		t.Fatalf("expected start to be %s; got %s", k2, start)
+	}
+
+	if err := cp.Set(k3); err != nil {
+		t.Fatal(err)
+	}
+	if err := cp.Set(k4); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err = OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !start.Equals(k4) {
+		t.Fatalf("expected start to be %s; got %s", k4, start)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+}

blockstore/splitstore/coldset.go

@@ -0,0 +1,102 @@
+package splitstore
+
+import (
+	"bufio"
+	"io"
+	"os"
+
+	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
+)
+
+type ColdSetWriter struct {
+	file *os.File
+	buf  *bufio.Writer
+}
+
+type ColdSetReader struct {
+	file *os.File
+	buf  *bufio.Reader
+}
+
+func NewColdSetWriter(path string) (*ColdSetWriter, error) {
+	file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return nil, xerrors.Errorf("error creating coldset: %w", err)
+	}
+	buf := bufio.NewWriter(file)
+
+	return &ColdSetWriter{
+		file: file,
+		buf:  buf,
+	}, nil
+}
+
+func NewColdSetReader(path string) (*ColdSetReader, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, xerrors.Errorf("error opening coldset: %w", err)
+	}
+	buf := bufio.NewReader(file)
+
+	return &ColdSetReader{
+		file: file,
+		buf:  buf,
+	}, nil
+}
+
+func (s *ColdSetWriter) Write(c cid.Cid) error {
+	return writeRawCid(s.buf, c, false)
+}
+
+func (s *ColdSetWriter) Close() error {
+	if s.file == nil {
+		return nil
+	}
+
+	err1 := s.buf.Flush()
+	err2 := s.file.Close()
+	s.buf = nil
+	s.file = nil
+
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
+
+func (s *ColdSetReader) ForEach(f func(cid.Cid) error) error {
+	hbuf := make([]byte, 256)
+	for {
+		next, err := readRawCid(s.buf, hbuf)
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+
+			return xerrors.Errorf("error reading coldset: %w", err)
+		}
+
+		if err := f(next); err != nil {
+			return err
+		}
+	}
+}
+
+func (s *ColdSetReader) Reset() error {
+	_, err := s.file.Seek(0, io.SeekStart)
+	return err
+}
+
+func (s *ColdSetReader) Close() error {
+	if s.file == nil {
+		return nil
+	}
+
+	err := s.file.Close()
+	s.file = nil
+	s.buf = nil
+
+	return err
+}

blockstore/splitstore/coldset_test.go

@@ -0,0 +1,99 @@
+package splitstore
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+)
+
+func TestColdSet(t *testing.T) {
+	dir, err := ioutil.TempDir("", "coldset.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(dir)
+	})
+
+	path := filepath.Join(dir, "coldset")
+
+	makeCid := func(i int) cid.Cid {
+		h, err := multihash.Sum([]byte(fmt.Sprintf("cid.%d", i)), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	const count = 1000
+	cids := make([]cid.Cid, 0, count)
+	for i := 0; i < count; i++ {
+		cids = append(cids, makeCid(i))
+	}
+
+	cw, err := NewColdSetWriter(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, c := range cids {
+		if err := cw.Write(c); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if err := cw.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cr, err := NewColdSetReader(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	index := 0
+	err = cr.ForEach(func(c cid.Cid) error {
+		if index >= count {
+			t.Fatal("too many cids")
+		}
+
+		if !c.Equals(cids[index]) {
+			t.Fatalf("wrong cid %d; expected %s but got %s", index, cids[index], c)
+		}
+
+		index++
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cr.Reset(); err != nil {
+		t.Fatal(err)
+	}
+
+	index = 0
+	err = cr.ForEach(func(c cid.Cid) error {
+		if index >= count {
+			t.Fatal("too many cids")
+		}
+
+		if !c.Equals(cids[index]) {
+			t.Fatalf("wrong cid; expected %s but got %s", cids[index], c)
+		}
+
+		index++
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+}

blockstore/splitstore/markset.go

@@ -10,39 +10,36 @@ import (
 
 var errMarkSetClosed = errors.New("markset closed")
 
-// MarkSet is a utility to keep track of seen CID, and later query for them.
-//
-// * If the expected dataset is large, it can be backed by a datastore (e.g. bbolt).
-// * If a probabilistic result is acceptable, it can be backed by a bloom filter
+// MarkSet is an interface for tracking CIDs during chain and object walks
 type MarkSet interface {
+	ObjectVisitor
 	Mark(cid.Cid) error
+	MarkMany([]cid.Cid) error
 	Has(cid.Cid) (bool, error)
 	Close() error
-	SetConcurrent()
-}
 
-type MarkSetVisitor interface {
-	MarkSet
-	ObjectVisitor
+	// BeginCriticalSection ensures that the markset is persisted to disk for recovery in case
+	// of abnormal termination during the critical section span.
+	BeginCriticalSection() error
+	// EndCriticalSection ends the critical section span.
+	EndCriticalSection()
 }
 
 type MarkSetEnv interface {
-	// Create creates a new markset within the environment.
-	// name is a unique name for this markset, mapped to the filesystem in disk-backed environments
+	// New creates a new markset within the environment.
+	// name is a unique name for this markset, mapped to the filesystem for on-disk persistence.
 	// sizeHint is a hint about the expected size of the markset
-	Create(name string, sizeHint int64) (MarkSet, error)
-	// CreateVisitor is like Create, but returns a wider interface that supports atomic visits.
-	// It may not be supported by some markset types (e.g. bloom).
-	CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error)
-	// SupportsVisitor returns true if the marksets created by this environment support the visitor interface.
-	SupportsVisitor() bool
+	New(name string, sizeHint int64) (MarkSet, error)
+	// Recover recovers an existing markset persisted on-disk.
+	Recover(name string) (MarkSet, error)
+	// Close closes the markset
 	Close() error
 }
 
 func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) {
 	switch mtype {
 	case "map":
-		return NewMapMarkSetEnv()
+		return NewMapMarkSetEnv(path)
 	case "badger":
 		return NewBadgerMarkSetEnv(path)
 	default:

blockstore/splitstore/markset_badger.go

@@ -3,6 +3,7 @@ package splitstore
 import (
 	"os"
 	"path/filepath"
+	"runtime"
 	"sync"
 
 	"golang.org/x/xerrors"
@@ -28,13 +29,13 @@ type BadgerMarkSet struct {
 	writers int
 	seqno   int
 	version int
+	persist bool
 
 	db   *badger.DB
 	path string
 }
 
 var _ MarkSet = (*BadgerMarkSet)(nil)
-var _ MarkSetVisitor = (*BadgerMarkSet)(nil)
 
 var badgerMarkSetBatchSize = 16384
 
@@ -48,11 +49,10 @@ func NewBadgerMarkSetEnv(path string) (MarkSetEnv, error) {
 	return &BadgerMarkSetEnv{path: msPath}, nil
 }
 
-func (e *BadgerMarkSetEnv) create(name string, sizeHint int64) (*BadgerMarkSet, error) {
-	name += ".tmp"
+func (e *BadgerMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
 	path := filepath.Join(e.path, name)
 
-	db, err := openTransientBadgerDB(path)
+	db, err := openBadgerDB(path, false)
 	if err != nil {
 		return nil, xerrors.Errorf("error creating badger db: %w", err)
 	}
@@ -68,18 +68,72 @@ func (e *BadgerMarkSetEnv) create(name string, sizeHint int64) (*BadgerMarkSet,
 	return ms, nil
 }
 
-func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
-	return e.create(name, sizeHint)
-}
+func (e *BadgerMarkSetEnv) Recover(name string) (MarkSet, error) {
+	path := filepath.Join(e.path, name)
 
-func (e *BadgerMarkSetEnv) CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error) {
-	return e.create(name, sizeHint)
-}
+	if _, err := os.Stat(path); err != nil {
+		return nil, xerrors.Errorf("error stating badger db path: %w", err)
+	}
 
-func (e *BadgerMarkSetEnv) SupportsVisitor() bool { return true }
+	db, err := openBadgerDB(path, true)
+	if err != nil {
+		return nil, xerrors.Errorf("error creating badger db: %w", err)
+	}
+
+	ms := &BadgerMarkSet{
+		pend:    make(map[string]struct{}),
+		writing: make(map[int]map[string]struct{}),
+		db:      db,
+		path:    path,
+		persist: true,
+	}
+	ms.cond.L = &ms.mx
+
+	return ms, nil
+}
 
 func (e *BadgerMarkSetEnv) Close() error {
-	return os.RemoveAll(e.path)
+	return nil
+}
+
+func (s *BadgerMarkSet) BeginCriticalSection() error {
+	s.mx.Lock()
+
+	if s.persist {
+		s.mx.Unlock()
+		return nil
+	}
+
+	var write bool
+	var seqno int
+	if len(s.pend) > 0 {
+		write = true
+		seqno = s.nextBatch()
+	}
+
+	s.persist = true
+	s.mx.Unlock()
+
+	if write {
+		// all writes sync once perist is true
+		return s.write(seqno)
+	}
+
+	// wait for any pending writes and sync
+	s.mx.Lock()
+	for s.writers > 0 {
+		s.cond.Wait()
+	}
+	s.mx.Unlock()
+
+	return s.db.Sync()
+}
+
+func (s *BadgerMarkSet) EndCriticalSection() {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	s.persist = false
 }
 
 func (s *BadgerMarkSet) Mark(c cid.Cid) error {
@@ -99,6 +153,23 @@ func (s *BadgerMarkSet) Mark(c cid.Cid) error {
 	return nil
 }
 
+func (s *BadgerMarkSet) MarkMany(batch []cid.Cid) error {
+	s.mx.Lock()
+	if s.pend == nil {
+		s.mx.Unlock()
+		return errMarkSetClosed
+	}
+
+	write, seqno := s.putMany(batch)
+	s.mx.Unlock()
+
+	if write {
+		return s.write(seqno)
+	}
+
+	return nil
+}
+
 func (s *BadgerMarkSet) Has(c cid.Cid) (bool, error) {
 	s.mx.RLock()
 	defer s.mx.RUnlock()
@@ -204,16 +275,34 @@ func (s *BadgerMarkSet) tryDB(key []byte) (has bool, err error) {
 // writer holds the exclusive lock
 func (s *BadgerMarkSet) put(key string) (write bool, seqno int) {
 	s.pend[key] = struct{}{}
-	if len(s.pend) < badgerMarkSetBatchSize {
+	if !s.persist && len(s.pend) < badgerMarkSetBatchSize {
 		return false, 0
 	}
 
-	seqno = s.seqno
+	seqno = s.nextBatch()
+	return true, seqno
+}
+
+func (s *BadgerMarkSet) putMany(batch []cid.Cid) (write bool, seqno int) {
+	for _, c := range batch {
+		key := string(c.Hash())
+		s.pend[key] = struct{}{}
+	}
+
+	if !s.persist && len(s.pend) < badgerMarkSetBatchSize {
+		return false, 0
+	}
+
+	seqno = s.nextBatch()
+	return true, seqno
+}
+
+func (s *BadgerMarkSet) nextBatch() int {
+	seqno := s.seqno
 	s.seqno++
 	s.writing[seqno] = s.pend
 	s.pend = make(map[string]struct{})
-
-	return true, seqno
+	return seqno
 }
 
 func (s *BadgerMarkSet) write(seqno int) (err error) {
@@ -258,6 +347,14 @@ func (s *BadgerMarkSet) write(seqno int) (err error) {
 		return xerrors.Errorf("error flushing batch to badger markset: %w", err)
 	}
 
+	s.mx.RLock()
+	persist := s.persist
+	s.mx.RUnlock()
+
+	if persist {
+		return s.db.Sync()
+	}
+
 	return nil
 }
 
@@ -277,26 +374,29 @@ func (s *BadgerMarkSet) Close() error {
 	db := s.db
 	s.db = nil
 
-	return closeTransientBadgerDB(db, s.path)
+	return closeBadgerDB(db, s.path, s.persist)
 }
 
-func (s *BadgerMarkSet) SetConcurrent() {}
+func openBadgerDB(path string, recover bool) (*badger.DB, error) {
+	// if it is not a recovery, clean up first
+	if !recover {
+		err := os.RemoveAll(path)
+		if err != nil {
+			return nil, xerrors.Errorf("error clearing markset directory: %w", err)
+		}
 
-func openTransientBadgerDB(path string) (*badger.DB, error) {
-	// clean up first
-	err := os.RemoveAll(path)
-	if err != nil {
-		return nil, xerrors.Errorf("error clearing markset directory: %w", err)
-	}
-
-	err = os.MkdirAll(path, 0755) //nolint:gosec
-	if err != nil {
-		return nil, xerrors.Errorf("error creating markset directory: %w", err)
+		err = os.MkdirAll(path, 0755) //nolint:gosec
+		if err != nil {
+			return nil, xerrors.Errorf("error creating markset directory: %w", err)
+		}
 	}
 
 	opts := badger.DefaultOptions(path)
+	// we manually sync when we are in critical section
 	opts.SyncWrites = false
+	// no need to do that
 	opts.CompactL0OnClose = false
+	// we store hashes, not much to gain by compression
 	opts.Compression = options.None
 	// Note: We use FileIO for loading modes to avoid memory thrashing and interference
 	//       between the system blockstore and the markset.
@@ -305,6 +405,15 @@ func openTransientBadgerDB(path string) (*badger.DB, error) {
 	//       exceeded 1GB in size.
 	opts.TableLoadingMode = options.FileIO
 	opts.ValueLogLoadingMode = options.FileIO
+	// We increase the number of L0 tables before compaction to make it unlikely to
+	// be necessary.
+	opts.NumLevelZeroTables = 20      // default is 5
+	opts.NumLevelZeroTablesStall = 30 // default is 10
+	// increase the number of compactors from default 2 so that if we ever have to
+	// compact, it is fast
+	if runtime.NumCPU()/2 > opts.NumCompactors {
+		opts.NumCompactors = runtime.NumCPU() / 2
+	}
 	opts.Logger = &badgerLogger{
 		SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(),
 		skip2:         log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(),
@@ -313,12 +422,16 @@ func openTransientBadgerDB(path string) (*badger.DB, error) {
 	return badger.Open(opts)
 }
 
-func closeTransientBadgerDB(db *badger.DB, path string) error {
+func closeBadgerDB(db *badger.DB, path string, persist bool) error {
 	err := db.Close()
 	if err != nil {
 		return xerrors.Errorf("error closing badger markset: %w", err)
 	}
 
+	if persist {
+		return nil
+	}
+
 	err = os.RemoveAll(path)
 	if err != nil {
 		return xerrors.Errorf("error deleting badger markset: %w", err)

blockstore/splitstore/markset_map.go

@@ -1,12 +1,20 @@
 package splitstore
 
 import (
+	"bufio"
+	"io"
+	"os"
+	"path/filepath"
 	"sync"
 
+	"golang.org/x/xerrors"
+
 	cid "github.com/ipfs/go-cid"
 )
 
-type MapMarkSetEnv struct{}
+type MapMarkSetEnv struct {
+	path string
+}
 
 var _ MarkSetEnv = (*MapMarkSetEnv)(nil)
 
@@ -14,55 +22,194 @@ type MapMarkSet struct {
 	mx  sync.RWMutex
 	set map[string]struct{}
 
-	ts bool
+	persist bool
+	file    *os.File
+	buf     *bufio.Writer
+
+	path string
 }
 
 var _ MarkSet = (*MapMarkSet)(nil)
-var _ MarkSetVisitor = (*MapMarkSet)(nil)
 
-func NewMapMarkSetEnv() (*MapMarkSetEnv, error) {
-	return &MapMarkSetEnv{}, nil
+func NewMapMarkSetEnv(path string) (*MapMarkSetEnv, error) {
+	msPath := filepath.Join(path, "markset.map")
+	err := os.MkdirAll(msPath, 0755) //nolint:gosec
+	if err != nil {
+		return nil, xerrors.Errorf("error creating markset directory: %w", err)
+	}
+
+	return &MapMarkSetEnv{path: msPath}, nil
 }
 
-func (e *MapMarkSetEnv) create(name string, sizeHint int64) (*MapMarkSet, error) {
+func (e *MapMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
+	path := filepath.Join(e.path, name)
 	return &MapMarkSet{
-		set: make(map[string]struct{}, sizeHint),
+		set:  make(map[string]struct{}, sizeHint),
+		path: path,
 	}, nil
 }
 
-func (e *MapMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
-	return e.create(name, sizeHint)
-}
+func (e *MapMarkSetEnv) Recover(name string) (MarkSet, error) {
+	path := filepath.Join(e.path, name)
+	s := &MapMarkSet{
+		set:  make(map[string]struct{}),
+		path: path,
+	}
 
-func (e *MapMarkSetEnv) CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error) {
-	return e.create(name, sizeHint)
-}
+	in, err := os.Open(path)
+	if err != nil {
+		return nil, xerrors.Errorf("error opening markset file for read: %w", err)
+	}
+	defer in.Close() //nolint:errcheck
 
-func (e *MapMarkSetEnv) SupportsVisitor() bool { return true }
+	// wrap a buffered reader to make this faster
+	buf := bufio.NewReader(in)
+	for {
+		var sz byte
+		if sz, err = buf.ReadByte(); err != nil {
+			break
+		}
+
+		key := make([]byte, int(sz))
+		if _, err = io.ReadFull(buf, key); err != nil {
+			break
+		}
+
+		s.set[string(key)] = struct{}{}
+	}
+
+	if err != io.EOF {
+		return nil, xerrors.Errorf("error reading markset file: %w", err)
+	}
+
+	file, err := os.OpenFile(s.path, os.O_WRONLY|os.O_APPEND, 0)
+	if err != nil {
+		return nil, xerrors.Errorf("error opening markset file for write: %w", err)
+	}
+
+	s.persist = true
+	s.file = file
+	s.buf = bufio.NewWriter(file)
+
+	return s, nil
+}
 
 func (e *MapMarkSetEnv) Close() error {
 	return nil
 }
 
-func (s *MapMarkSet) Mark(cid cid.Cid) error {
-	if s.ts {
-		s.mx.Lock()
-		defer s.mx.Unlock()
-	}
+func (s *MapMarkSet) BeginCriticalSection() error {
+	s.mx.Lock()
+	defer s.mx.Unlock()
 
 	if s.set == nil {
 		return errMarkSetClosed
 	}
 
-	s.set[string(cid.Hash())] = struct{}{}
+	if s.persist {
+		return nil
+	}
+
+	file, err := os.OpenFile(s.path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return xerrors.Errorf("error opening markset file: %w", err)
+	}
+
+	// wrap a buffered writer to make this faster
+	s.buf = bufio.NewWriter(file)
+	for key := range s.set {
+		if err := s.writeKey([]byte(key), false); err != nil {
+			_ = file.Close()
+			s.buf = nil
+			return err
+		}
+	}
+	if err := s.buf.Flush(); err != nil {
+		_ = file.Close()
+		s.buf = nil
+		return xerrors.Errorf("error flushing markset file buffer: %w", err)
+	}
+
+	s.file = file
+	s.persist = true
+
+	return nil
+}
+
+func (s *MapMarkSet) EndCriticalSection() {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	if !s.persist {
+		return
+	}
+
+	_ = s.file.Close()
+	_ = os.Remove(s.path)
+	s.file = nil
+	s.buf = nil
+	s.persist = false
+}
+
+func (s *MapMarkSet) Mark(c cid.Cid) error {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	if s.set == nil {
+		return errMarkSetClosed
+	}
+
+	hash := c.Hash()
+	s.set[string(hash)] = struct{}{}
+
+	if s.persist {
+		if err := s.writeKey(hash, true); err != nil {
+			return err
+		}
+
+		if err := s.file.Sync(); err != nil {
+			return xerrors.Errorf("error syncing markset: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (s *MapMarkSet) MarkMany(batch []cid.Cid) error {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	if s.set == nil {
+		return errMarkSetClosed
+	}
+
+	for _, c := range batch {
+		hash := c.Hash()
+		s.set[string(hash)] = struct{}{}
+
+		if s.persist {
+			if err := s.writeKey(hash, false); err != nil {
+				return err
+			}
+		}
+	}
+
+	if s.persist {
+		if err := s.buf.Flush(); err != nil {
+			return xerrors.Errorf("error flushing markset buffer to disk: %w", err)
+		}
+
+		if err := s.file.Sync(); err != nil {
+			return xerrors.Errorf("error syncing markset: %w", err)
+		}
+	}
+
 	return nil
 }
 
 func (s *MapMarkSet) Has(cid cid.Cid) (bool, error) {
-	if s.ts {
-		s.mx.RLock()
-		defer s.mx.RUnlock()
-	}
+	s.mx.RLock()
+	defer s.mx.RUnlock()
 
 	if s.set == nil {
 		return false, errMarkSetClosed
@@ -73,33 +220,70 @@ func (s *MapMarkSet) Has(cid cid.Cid) (bool, error) {
 }
 
 func (s *MapMarkSet) Visit(c cid.Cid) (bool, error) {
-	if s.ts {
-		s.mx.Lock()
-		defer s.mx.Unlock()
-	}
+	s.mx.Lock()
+	defer s.mx.Unlock()
 
 	if s.set == nil {
 		return false, errMarkSetClosed
 	}
 
-	key := string(c.Hash())
+	hash := c.Hash()
+	key := string(hash)
 	if _, ok := s.set[key]; ok {
 		return false, nil
 	}
 
 	s.set[key] = struct{}{}
+
+	if s.persist {
+		if err := s.writeKey(hash, true); err != nil {
+			return false, err
+		}
+		if err := s.file.Sync(); err != nil {
+			return false, xerrors.Errorf("error syncing markset: %w", err)
+		}
+	}
+
 	return true, nil
 }
 
 func (s *MapMarkSet) Close() error {
-	if s.ts {
-		s.mx.Lock()
-		defer s.mx.Unlock()
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	if s.set == nil {
+		return nil
 	}
+
 	s.set = nil
+
+	if s.file != nil {
+		if err := s.file.Close(); err != nil {
+			log.Warnf("error closing markset file: %s", err)
+		}
+
+		if !s.persist {
+			if err := os.Remove(s.path); err != nil {
+				log.Warnf("error removing markset file: %s", err)
+			}
+		}
+	}
+
 	return nil
 }
 
-func (s *MapMarkSet) SetConcurrent() {
-	s.ts = true
+func (s *MapMarkSet) writeKey(k []byte, flush bool) error {
+	if err := s.buf.WriteByte(byte(len(k))); err != nil {
+		return xerrors.Errorf("error writing markset key length to disk: %w", err)
+	}
+	if _, err := s.buf.Write(k); err != nil {
+		return xerrors.Errorf("error writing markset key to disk: %w", err)
+	}
+	if flush {
+		if err := s.buf.Flush(); err != nil {
+			return xerrors.Errorf("error flushing markset buffer to disk: %w", err)
+		}
+	}
+
+	return nil
 }

blockstore/splitstore/markset_test.go

@@ -11,7 +11,10 @@ import (
 
 func TestMapMarkSet(t *testing.T) {
 	testMarkSet(t, "map")
+	testMarkSetRecovery(t, "map")
+	testMarkSetMarkMany(t, "map")
 	testMarkSetVisitor(t, "map")
+	testMarkSetVisitorRecovery(t, "map")
 }
 
 func TestBadgerMarkSet(t *testing.T) {
@@ -21,12 +24,13 @@ func TestBadgerMarkSet(t *testing.T) {
 		badgerMarkSetBatchSize = bs
 	})
 	testMarkSet(t, "badger")
+	testMarkSetRecovery(t, "badger")
+	testMarkSetMarkMany(t, "badger")
 	testMarkSetVisitor(t, "badger")
+	testMarkSetVisitorRecovery(t, "badger")
 }
 
 func testMarkSet(t *testing.T, lsType string) {
-	t.Helper()
-
 	path, err := ioutil.TempDir("", "markset.*")
 	if err != nil {
 		t.Fatal(err)
@@ -42,12 +46,12 @@ func testMarkSet(t *testing.T, lsType string) {
 	}
 	defer env.Close() //nolint:errcheck
 
-	hotSet, err := env.Create("hot", 0)
+	hotSet, err := env.New("hot", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	coldSet, err := env.Create("cold", 0)
+	coldSet, err := env.New("cold", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -62,6 +66,7 @@ func testMarkSet(t *testing.T, lsType string) {
 	}
 
 	mustHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
 		has, err := s.Has(cid)
 		if err != nil {
 			t.Fatal(err)
@@ -73,6 +78,7 @@ func testMarkSet(t *testing.T, lsType string) {
 	}
 
 	mustNotHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
 		has, err := s.Has(cid)
 		if err != nil {
 			t.Fatal(err)
@@ -114,12 +120,12 @@ func testMarkSet(t *testing.T, lsType string) {
 		t.Fatal(err)
 	}
 
-	hotSet, err = env.Create("hot", 0)
+	hotSet, err = env.New("hot", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	coldSet, err = env.Create("cold", 0)
+	coldSet, err = env.New("cold", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -150,8 +156,6 @@ func testMarkSet(t *testing.T, lsType string) {
 }
 
 func testMarkSetVisitor(t *testing.T, lsType string) {
-	t.Helper()
-
 	path, err := ioutil.TempDir("", "markset.*")
 	if err != nil {
 		t.Fatal(err)
@@ -167,7 +171,7 @@ func testMarkSetVisitor(t *testing.T, lsType string) {
 	}
 	defer env.Close() //nolint:errcheck
 
-	visitor, err := env.CreateVisitor("test", 0)
+	visitor, err := env.New("test", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -219,3 +223,322 @@ func testMarkSetVisitor(t *testing.T, lsType string) {
 	mustNotVisit(visitor, k3)
 	mustNotVisit(visitor, k4)
 }
+
+func testMarkSetVisitorRecovery(t *testing.T, lsType string) {
+	path, err := ioutil.TempDir("", "markset.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(path)
+	})
+
+	env, err := OpenMarkSetEnv(path, lsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer env.Close() //nolint:errcheck
+
+	visitor, err := env.New("test", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer visitor.Close() //nolint:errcheck
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	mustVisit := func(v ObjectVisitor, cid cid.Cid) {
+		visit, err := v.Visit(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !visit {
+			t.Fatal("object should be visited")
+		}
+	}
+
+	mustNotVisit := func(v ObjectVisitor, cid cid.Cid) {
+		visit, err := v.Visit(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if visit {
+			t.Fatal("unexpected visit")
+		}
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	mustVisit(visitor, k1)
+	mustVisit(visitor, k2)
+
+	if err := visitor.BeginCriticalSection(); err != nil {
+		t.Fatal(err)
+	}
+
+	mustVisit(visitor, k3)
+	mustVisit(visitor, k4)
+
+	mustNotVisit(visitor, k1)
+	mustNotVisit(visitor, k2)
+	mustNotVisit(visitor, k3)
+	mustNotVisit(visitor, k4)
+
+	if err := visitor.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	visitor, err = env.Recover("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mustNotVisit(visitor, k1)
+	mustNotVisit(visitor, k2)
+	mustNotVisit(visitor, k3)
+	mustNotVisit(visitor, k4)
+
+	visitor.EndCriticalSection()
+
+	if err := visitor.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = env.Recover("test")
+	if err == nil {
+		t.Fatal("expected recovery to fail")
+	}
+}
+
+func testMarkSetRecovery(t *testing.T, lsType string) {
+	path, err := ioutil.TempDir("", "markset.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(path)
+	})
+
+	env, err := OpenMarkSetEnv(path, lsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer env.Close() //nolint:errcheck
+
+	markSet, err := env.New("test", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	mustHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !has {
+			t.Fatal("mark not found")
+		}
+	}
+
+	mustNotHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("unexpected mark")
+		}
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	if err := markSet.Mark(k1); err != nil {
+		t.Fatal(err)
+	}
+	if err := markSet.Mark(k2); err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustNotHave(markSet, k3)
+	mustNotHave(markSet, k4)
+
+	if err := markSet.BeginCriticalSection(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := markSet.Mark(k3); err != nil {
+		t.Fatal(err)
+	}
+	if err := markSet.Mark(k4); err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustHave(markSet, k3)
+	mustHave(markSet, k4)
+
+	if err := markSet.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	markSet, err = env.Recover("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustHave(markSet, k3)
+	mustHave(markSet, k4)
+
+	markSet.EndCriticalSection()
+
+	if err := markSet.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = env.Recover("test")
+	if err == nil {
+		t.Fatal("expected recovery to fail")
+	}
+}
+
+func testMarkSetMarkMany(t *testing.T, lsType string) {
+	path, err := ioutil.TempDir("", "markset.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(path)
+	})
+
+	env, err := OpenMarkSetEnv(path, lsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer env.Close() //nolint:errcheck
+
+	markSet, err := env.New("test", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	mustHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !has {
+			t.Fatal("mark not found")
+		}
+	}
+
+	mustNotHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("unexpected mark")
+		}
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	if err := markSet.MarkMany([]cid.Cid{k1, k2}); err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustNotHave(markSet, k3)
+	mustNotHave(markSet, k4)
+
+	if err := markSet.BeginCriticalSection(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := markSet.MarkMany([]cid.Cid{k3, k4}); err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustHave(markSet, k3)
+	mustHave(markSet, k4)
+
+	if err := markSet.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	markSet, err = env.Recover("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustHave(markSet, k3)
+	mustHave(markSet, k4)
+
+	markSet.EndCriticalSection()
+
+	if err := markSet.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = env.Recover("test")
+	if err == nil {
+		t.Fatal("expected recovery to fail")
+	}
+}

blockstore/splitstore/splitstore.go

@@ -129,8 +129,6 @@ type SplitStore struct {
 
 	headChangeMx sync.Mutex
 
-	coldPurgeSize int
-
 	chain ChainAccessor
 	ds    dstore.Datastore
 	cold  bstore.Blockstore
@@ -158,6 +156,17 @@ type SplitStore struct {
 	txnRefsMx       sync.Mutex
 	txnRefs         map[cid.Cid]struct{}
 	txnMissing      map[cid.Cid]struct{}
+	txnMarkSet      MarkSet
+	txnSyncMx       sync.Mutex
+	txnSyncCond     sync.Cond
+	txnSync         bool
+
+	// background cold object reification
+	reifyWorkers    sync.WaitGroup
+	reifyMx         sync.Mutex
+	reifyCond       sync.Cond
+	reifyPend       map[cid.Cid]struct{}
+	reifyInProgress map[cid.Cid]struct{}
 
 	// registered protectors
 	protectors []func(func(cid.Cid) error) error
@@ -186,10 +195,6 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 		return nil, err
 	}
 
-	if !markSetEnv.SupportsVisitor() {
-		return nil, xerrors.Errorf("markset type does not support atomic visitors")
-	}
-
 	// and now we can make a SplitStore
 	ss := &SplitStore{
 		cfg:        cfg,
@@ -198,13 +203,16 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 		cold:       cold,
 		hot:        hots,
 		markSetEnv: markSetEnv,
-
-		coldPurgeSize: defaultColdPurgeSize,
 	}
 
 	ss.txnViewsCond.L = &ss.txnViewsMx
+	ss.txnSyncCond.L = &ss.txnSyncMx
 	ss.ctx, ss.cancel = context.WithCancel(context.Background())
 
+	ss.reifyCond.L = &ss.reifyMx
+	ss.reifyPend = make(map[cid.Cid]struct{})
+	ss.reifyInProgress = make(map[cid.Cid]struct{})
+
 	if enableDebugLog {
 		ss.debug, err = openDebugLog(path)
 		if err != nil {
@@ -212,21 +220,29 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 		}
 	}
 
+	if ss.checkpointExists() {
+		log.Info("found compaction checkpoint; resuming compaction")
+		if err := ss.completeCompaction(); err != nil {
+			markSetEnv.Close() //nolint:errcheck
+			return nil, xerrors.Errorf("error resuming compaction: %w", err)
+		}
+	}
+
 	return ss, nil
 }
 
 // Blockstore interface
-func (s *SplitStore) DeleteBlock(_ cid.Cid) error {
+func (s *SplitStore) DeleteBlock(_ context.Context, _ cid.Cid) error {
 	// afaict we don't seem to be using this method, so it's not implemented
 	return errors.New("DeleteBlock not implemented on SplitStore; don't do this Luke!") //nolint
 }
 
-func (s *SplitStore) DeleteMany(_ []cid.Cid) error {
+func (s *SplitStore) DeleteMany(_ context.Context, _ []cid.Cid) error {
 	// afaict we don't seem to be using this method, so it's not implemented
 	return errors.New("DeleteMany not implemented on SplitStore; don't do this Luke!") //nolint
 }
 
-func (s *SplitStore) Has(cid cid.Cid) (bool, error) {
+func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 	if isIdentiyCid(cid) {
 		return true, nil
 	}
@@ -234,7 +250,21 @@ func (s *SplitStore) Has(cid cid.Cid) (bool, error) {
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()
 
-	has, err := s.hot.Has(cid)
+	// critical section
+	if s.txnMarkSet != nil {
+		has, err := s.txnMarkSet.Has(cid)
+		if err != nil {
+			return false, err
+		}
+
+		if has {
+			return s.has(cid)
+		}
+
+		return s.cold.Has(ctx, cid)
+	}
+
+	has, err := s.hot.Has(ctx, cid)
 
 	if err != nil {
 		return has, err
@@ -245,10 +275,16 @@ func (s *SplitStore) Has(cid cid.Cid) (bool, error) {
 		return true, nil
 	}
 
-	return s.cold.Has(cid)
+	has, err = s.cold.Has(ctx, cid)
+	if has && bstore.IsHotView(ctx) {
+		s.reifyColdObject(cid)
+	}
+
+	return has, err
+
 }
 
-func (s *SplitStore) Get(cid cid.Cid) (blocks.Block, error) {
+func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
 	if isIdentiyCid(cid) {
 		data, err := decodeIdentityCid(cid)
 		if err != nil {
@@ -261,7 +297,21 @@ func (s *SplitStore) Get(cid cid.Cid) (blocks.Block, error) {
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()
 
-	blk, err := s.hot.Get(cid)
+	// critical section
+	if s.txnMarkSet != nil {
+		has, err := s.txnMarkSet.Has(cid)
+		if err != nil {
+			return nil, err
+		}
+
+		if has {
+			return s.get(cid)
+		}
+
+		return s.cold.Get(ctx, cid)
+	}
+
+	blk, err := s.hot.Get(ctx, cid)
 
 	switch err {
 	case nil:
@@ -273,10 +323,13 @@ func (s *SplitStore) Get(cid cid.Cid) (blocks.Block, error) {
 			s.debug.LogReadMiss(cid)
 		}
 
-		blk, err = s.cold.Get(cid)
+		blk, err = s.cold.Get(ctx, cid)
 		if err == nil {
-			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
+			if bstore.IsHotView(ctx) {
+				s.reifyColdObject(cid)
+			}
 
+			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return blk, err
 
@@ -285,7 +338,7 @@ func (s *SplitStore) Get(cid cid.Cid) (blocks.Block, error) {
 	}
 }
 
-func (s *SplitStore) GetSize(cid cid.Cid) (int, error) {
+func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
 	if isIdentiyCid(cid) {
 		data, err := decodeIdentityCid(cid)
 		if err != nil {
@@ -298,7 +351,21 @@ func (s *SplitStore) GetSize(cid cid.Cid) (int, error) {
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()
 
-	size, err := s.hot.GetSize(cid)
+	// critical section
+	if s.txnMarkSet != nil {
+		has, err := s.txnMarkSet.Has(cid)
+		if err != nil {
+			return 0, err
+		}
+
+		if has {
+			return s.getSize(cid)
+		}
+
+		return s.cold.GetSize(ctx, cid)
+	}
+
+	size, err := s.hot.GetSize(ctx, cid)
 
 	switch err {
 	case nil:
@@ -310,8 +377,12 @@ func (s *SplitStore) GetSize(cid cid.Cid) (int, error) {
 			s.debug.LogReadMiss(cid)
 		}
 
-		size, err = s.cold.GetSize(cid)
+		size, err = s.cold.GetSize(ctx, cid)
 		if err == nil {
+			if bstore.IsHotView(ctx) {
+				s.reifyColdObject(cid)
+			}
+
 			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return size, err
@@ -321,7 +392,7 @@ func (s *SplitStore) GetSize(cid cid.Cid) (int, error) {
 	}
 }
 
-func (s *SplitStore) Put(blk blocks.Block) error {
+func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error {
 	if isIdentiyCid(blk.Cid()) {
 		return nil
 	}
@@ -329,18 +400,24 @@ func (s *SplitStore) Put(blk blocks.Block) error {
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()
 
-	err := s.hot.Put(blk)
+	err := s.hot.Put(ctx, blk)
 	if err != nil {
 		return err
 	}
 
 	s.debug.LogWrite(blk)
 
+	// critical section
+	if s.txnMarkSet != nil {
+		s.markLiveRefs([]cid.Cid{blk.Cid()})
+		return nil
+	}
+
 	s.trackTxnRef(blk.Cid())
 	return nil
 }
 
-func (s *SplitStore) PutMany(blks []blocks.Block) error {
+func (s *SplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {
 	// filter identites
 	idcids := 0
 	for _, blk := range blks {
@@ -374,13 +451,19 @@ func (s *SplitStore) PutMany(blks []blocks.Block) error {
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()
 
-	err := s.hot.PutMany(blks)
+	err := s.hot.PutMany(ctx, blks)
 	if err != nil {
 		return err
 	}
 
 	s.debug.LogWriteMany(blks)
 
+	// critical section
+	if s.txnMarkSet != nil {
+		s.markLiveRefs(batch)
+		return nil
+	}
+
 	s.trackTxnRefMany(batch)
 	return nil
 }
@@ -430,7 +513,7 @@ func (s *SplitStore) HashOnRead(enabled bool) {
 	s.cold.HashOnRead(enabled)
 }
 
-func (s *SplitStore) View(cid cid.Cid, cb func([]byte) error) error {
+func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) error) error {
 	if isIdentiyCid(cid) {
 		data, err := decodeIdentityCid(cid)
 		if err != nil {
@@ -440,6 +523,23 @@ func (s *SplitStore) View(cid cid.Cid, cb func([]byte) error) error {
 		return cb(data)
 	}
 
+	// critical section
+	s.txnLk.RLock() // the lock is released in protectView if we are not in critical section
+	if s.txnMarkSet != nil {
+		has, err := s.txnMarkSet.Has(cid)
+		s.txnLk.RUnlock()
+
+		if err != nil {
+			return err
+		}
+
+		if has {
+			return s.view(cid, cb)
+		}
+
+		return s.cold.View(ctx, cid, cb)
+	}
+
 	// views are (optimistically) protected two-fold:
 	// - if there is an active transaction, then the reference is protected.
 	// - if there is no active transaction, active views are tracked in a
@@ -451,15 +551,19 @@ func (s *SplitStore) View(cid cid.Cid, cb func([]byte) error) error {
 	s.protectView(cid)
 	defer s.viewDone()
 
-	err := s.hot.View(cid, cb)
+	err := s.hot.View(ctx, cid, cb)
 	switch err {
 	case bstore.ErrNotFound:
 		if s.isWarm() {
 			s.debug.LogReadMiss(cid)
 		}
 
-		err = s.cold.View(cid, cb)
+		err = s.cold.View(ctx, cid, cb)
 		if err == nil {
+			if bstore.IsHotView(ctx) {
+				s.reifyColdObject(cid)
+			}
+
 			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return err
@@ -502,7 +606,7 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
 
 	// load base epoch from metadata ds
 	// if none, then use current epoch because it's a fresh start
-	bs, err := s.ds.Get(baseEpochKey)
+	bs, err := s.ds.Get(s.ctx, baseEpochKey)
 	switch err {
 	case nil:
 		s.baseEpoch = bytesToEpoch(bs)
@@ -523,7 +627,7 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
 	}
 
 	// load warmup epoch from metadata ds
-	bs, err = s.ds.Get(warmupEpochKey)
+	bs, err = s.ds.Get(s.ctx, warmupEpochKey)
 	switch err {
 	case nil:
 		s.warmupEpoch = bytesToEpoch(bs)
@@ -536,7 +640,7 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
 	}
 
 	// load markSetSize from metadata ds to provide a size hint for marksets
-	bs, err = s.ds.Get(markSetSizeKey)
+	bs, err = s.ds.Get(s.ctx, markSetSizeKey)
 	switch err {
 	case nil:
 		s.markSetSize = bytesToInt64(bs)
@@ -547,7 +651,7 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
 	}
 
 	// load compactionIndex from metadata ds to provide a hint as to when to perform moving gc
-	bs, err = s.ds.Get(compactionIndexKey)
+	bs, err = s.ds.Get(s.ctx, compactionIndexKey)
 	switch err {
 	case nil:
 		s.compactionIndex = bytesToInt64(bs)
@@ -569,6 +673,9 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
 		}
 	}
 
+	// spawn the reifier
+	go s.reifyOrchestrator()
+
 	// watch the chain
 	chain.SubscribeHeadChanges(s.HeadChange)
 
@@ -589,12 +696,19 @@ func (s *SplitStore) Close() error {
 	}
 
 	if atomic.LoadInt32(&s.compacting) == 1 {
+		s.txnSyncMx.Lock()
+		s.txnSync = true
+		s.txnSyncCond.Broadcast()
+		s.txnSyncMx.Unlock()
+
 		log.Warn("close with ongoing compaction in progress; waiting for it to finish...")
 		for atomic.LoadInt32(&s.compacting) == 1 {
 			time.Sleep(time.Second)
 		}
 	}
 
+	s.reifyCond.Broadcast()
+	s.reifyWorkers.Wait()
 	s.cancel()
 	return multierr.Combine(s.markSetEnv.Close(), s.debug.Close())
 }
@@ -609,5 +723,5 @@ func (s *SplitStore) checkClosing() error {
 
 func (s *SplitStore) setBaseEpoch(epoch abi.ChainEpoch) error {
 	s.baseEpoch = epoch
-	return s.ds.Put(baseEpochKey, epochToBytes(epoch))
+	return s.ds.Put(s.ctx, baseEpochKey, epochToBytes(epoch))
 }

blockstore/splitstore/splitstore_check.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -67,7 +68,10 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 	}
 	defer output.Close() //nolint:errcheck
 
+	var mx sync.Mutex
 	write := func(format string, args ...interface{}) {
+		mx.Lock()
+		defer mx.Unlock()
 		_, err := fmt.Fprintf(output, format+"\n", args...)
 		if err != nil {
 			log.Warnf("error writing check output: %s", err)
@@ -82,9 +86,10 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 	write("compaction index: %d", s.compactionIndex)
 	write("--")
 
-	var coldCnt, missingCnt int64
+	coldCnt := new(int64)
+	missingCnt := new(int64)
 
-	visitor, err := s.markSetEnv.CreateVisitor("check", 0)
+	visitor, err := s.markSetEnv.New("check", 0)
 	if err != nil {
 		return xerrors.Errorf("error creating visitor: %w", err)
 	}
@@ -96,7 +101,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 				return errStopWalk
 			}
 
-			has, err := s.hot.Has(c)
+			has, err := s.hot.Has(s.ctx, c)
 			if err != nil {
 				return xerrors.Errorf("error checking hotstore: %w", err)
 			}
@@ -105,16 +110,16 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 				return nil
 			}
 
-			has, err = s.cold.Has(c)
+			has, err = s.cold.Has(s.ctx, c)
 			if err != nil {
 				return xerrors.Errorf("error checking coldstore: %w", err)
 			}
 
 			if has {
-				coldCnt++
+				atomic.AddInt64(coldCnt, 1)
 				write("cold object reference: %s", c)
 			} else {
-				missingCnt++
+				atomic.AddInt64(missingCnt, 1)
 				write("missing object reference: %s", c)
 				return errStopWalk
 			}
@@ -128,9 +133,9 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 		return err
 	}
 
-	log.Infow("check done", "cold", coldCnt, "missing", missingCnt)
+	log.Infow("check done", "cold", *coldCnt, "missing", *missingCnt)
 	write("--")
-	write("cold: %d missing: %d", coldCnt, missingCnt)
+	write("cold: %d missing: %d", *coldCnt, *missingCnt)
 	write("DONE")
 
 	return nil

blockstore/splitstore/splitstore_expose.go

@@ -20,28 +20,28 @@ func (s *SplitStore) Expose() bstore.Blockstore {
 	return &exposedSplitStore{s: s}
 }
 
-func (es *exposedSplitStore) DeleteBlock(_ cid.Cid) error {
+func (es *exposedSplitStore) DeleteBlock(_ context.Context, _ cid.Cid) error {
 	return errors.New("DeleteBlock: operation not supported")
 }
 
-func (es *exposedSplitStore) DeleteMany(_ []cid.Cid) error {
+func (es *exposedSplitStore) DeleteMany(_ context.Context, _ []cid.Cid) error {
 	return errors.New("DeleteMany: operation not supported")
 }
 
-func (es *exposedSplitStore) Has(c cid.Cid) (bool, error) {
+func (es *exposedSplitStore) Has(ctx context.Context, c cid.Cid) (bool, error) {
 	if isIdentiyCid(c) {
 		return true, nil
 	}
 
-	has, err := es.s.hot.Has(c)
+	has, err := es.s.hot.Has(ctx, c)
 	if has || err != nil {
 		return has, err
 	}
 
-	return es.s.cold.Has(c)
+	return es.s.cold.Has(ctx, c)
 }
 
-func (es *exposedSplitStore) Get(c cid.Cid) (blocks.Block, error) {
+func (es *exposedSplitStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
 	if isIdentiyCid(c) {
 		data, err := decodeIdentityCid(c)
 		if err != nil {
@@ -51,16 +51,16 @@ func (es *exposedSplitStore) Get(c cid.Cid) (blocks.Block, error) {
 		return blocks.NewBlockWithCid(data, c)
 	}
 
-	blk, err := es.s.hot.Get(c)
+	blk, err := es.s.hot.Get(ctx, c)
 	switch err {
 	case bstore.ErrNotFound:
-		return es.s.cold.Get(c)
+		return es.s.cold.Get(ctx, c)
 	default:
 		return blk, err
 	}
 }
 
-func (es *exposedSplitStore) GetSize(c cid.Cid) (int, error) {
+func (es *exposedSplitStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
 	if isIdentiyCid(c) {
 		data, err := decodeIdentityCid(c)
 		if err != nil {
@@ -70,21 +70,21 @@ func (es *exposedSplitStore) GetSize(c cid.Cid) (int, error) {
 		return len(data), nil
 	}
 
-	size, err := es.s.hot.GetSize(c)
+	size, err := es.s.hot.GetSize(ctx, c)
 	switch err {
 	case bstore.ErrNotFound:
-		return es.s.cold.GetSize(c)
+		return es.s.cold.GetSize(ctx, c)
 	default:
 		return size, err
 	}
 }
 
-func (es *exposedSplitStore) Put(blk blocks.Block) error {
-	return es.s.Put(blk)
+func (es *exposedSplitStore) Put(ctx context.Context, blk blocks.Block) error {
+	return es.s.Put(ctx, blk)
 }
 
-func (es *exposedSplitStore) PutMany(blks []blocks.Block) error {
-	return es.s.PutMany(blks)
+func (es *exposedSplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {
+	return es.s.PutMany(ctx, blks)
 }
 
 func (es *exposedSplitStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
@@ -93,7 +93,7 @@ func (es *exposedSplitStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, e
 
 func (es *exposedSplitStore) HashOnRead(enabled bool) {}
 
-func (es *exposedSplitStore) View(c cid.Cid, f func([]byte) error) error {
+func (es *exposedSplitStore) View(ctx context.Context, c cid.Cid, f func([]byte) error) error {
 	if isIdentiyCid(c) {
 		data, err := decodeIdentityCid(c)
 		if err != nil {
@@ -103,10 +103,10 @@ func (es *exposedSplitStore) View(c cid.Cid, f func([]byte) error) error {
 		return f(data)
 	}
 
-	err := es.s.hot.View(c, f)
+	err := es.s.hot.View(ctx, c, f)
 	switch err {
 	case bstore.ErrNotFound:
-		return es.s.cold.View(c, f)
+		return es.s.cold.View(ctx, c, f)
 
 	default:
 		return err

blockstore/splitstore/splitstore_reify.go

@@ -0,0 +1,214 @@
+package splitstore
+
+import (
+	"errors"
+	"runtime"
+	"sync/atomic"
+
+	"golang.org/x/xerrors"
+
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
+)
+
+var (
+	errReifyLimit = errors.New("reification limit reached")
+	ReifyLimit    = 16384
+)
+
+func (s *SplitStore) reifyColdObject(c cid.Cid) {
+	if !s.isWarm() {
+		return
+	}
+
+	if isUnitaryObject(c) {
+		return
+	}
+
+	s.reifyMx.Lock()
+	defer s.reifyMx.Unlock()
+
+	_, ok := s.reifyInProgress[c]
+	if ok {
+		return
+	}
+
+	s.reifyPend[c] = struct{}{}
+	s.reifyCond.Broadcast()
+}
+
+func (s *SplitStore) reifyOrchestrator() {
+	workers := runtime.NumCPU() / 4
+	if workers < 2 {
+		workers = 2
+	}
+
+	workch := make(chan cid.Cid, workers)
+	defer close(workch)
+
+	for i := 0; i < workers; i++ {
+		s.reifyWorkers.Add(1)
+		go s.reifyWorker(workch)
+	}
+
+	for {
+		s.reifyMx.Lock()
+		for len(s.reifyPend) == 0 && atomic.LoadInt32(&s.closing) == 0 {
+			s.reifyCond.Wait()
+		}
+
+		if atomic.LoadInt32(&s.closing) != 0 {
+			s.reifyMx.Unlock()
+			return
+		}
+
+		reifyPend := s.reifyPend
+		s.reifyPend = make(map[cid.Cid]struct{})
+		s.reifyMx.Unlock()
+
+		for c := range reifyPend {
+			select {
+			case workch <- c:
+			case <-s.ctx.Done():
+				return
+			}
+		}
+	}
+}
+
+func (s *SplitStore) reifyWorker(workch chan cid.Cid) {
+	defer s.reifyWorkers.Done()
+	for c := range workch {
+		s.doReify(c)
+	}
+}
+
+func (s *SplitStore) doReify(c cid.Cid) {
+	var toreify, totrack, toforget []cid.Cid
+
+	defer func() {
+		s.reifyMx.Lock()
+		defer s.reifyMx.Unlock()
+
+		for _, c := range toreify {
+			delete(s.reifyInProgress, c)
+		}
+		for _, c := range totrack {
+			delete(s.reifyInProgress, c)
+		}
+		for _, c := range toforget {
+			delete(s.reifyInProgress, c)
+		}
+	}()
+
+	s.txnLk.RLock()
+	defer s.txnLk.RUnlock()
+
+	count := 0
+	err := s.walkObjectIncomplete(c, newTmpVisitor(),
+		func(c cid.Cid) error {
+			if isUnitaryObject(c) {
+				return errStopWalk
+			}
+
+			count++
+			if count > ReifyLimit {
+				return errReifyLimit
+			}
+
+			s.reifyMx.Lock()
+			_, inProgress := s.reifyInProgress[c]
+			if !inProgress {
+				s.reifyInProgress[c] = struct{}{}
+			}
+			s.reifyMx.Unlock()
+
+			if inProgress {
+				return errStopWalk
+			}
+
+			has, err := s.hot.Has(s.ctx, c)
+			if err != nil {
+				return xerrors.Errorf("error checking hotstore: %w", err)
+			}
+
+			if has {
+				if s.txnMarkSet != nil {
+					hasMark, err := s.txnMarkSet.Has(c)
+					if err != nil {
+						log.Warnf("error checking markset: %s", err)
+					} else if hasMark {
+						toforget = append(toforget, c)
+						return errStopWalk
+					}
+				} else {
+					totrack = append(totrack, c)
+					return errStopWalk
+				}
+			}
+
+			toreify = append(toreify, c)
+			return nil
+		},
+		func(missing cid.Cid) error {
+			log.Warnf("missing reference while reifying %s: %s", c, missing)
+			return errStopWalk
+		})
+
+	if err != nil {
+		if xerrors.Is(err, errReifyLimit) {
+			log.Debug("reification aborted; reify limit reached")
+			return
+		}
+
+		log.Warnf("error walking cold object for reification (cid: %s): %s", c, err)
+		return
+	}
+
+	log.Debugf("reifying %d objects rooted at %s", len(toreify), c)
+
+	// this should not get too big, maybe some 100s of objects.
+	batch := make([]blocks.Block, 0, len(toreify))
+	for _, c := range toreify {
+		blk, err := s.cold.Get(s.ctx, c)
+		if err != nil {
+			log.Warnf("error retrieving cold object for reification (cid: %s): %s", c, err)
+			continue
+		}
+
+		if err := s.checkClosing(); err != nil {
+			return
+		}
+
+		batch = append(batch, blk)
+	}
+
+	if len(batch) > 0 {
+		err = s.hot.PutMany(s.ctx, batch)
+		if err != nil {
+			log.Warnf("error reifying cold object (cid: %s): %s", c, err)
+			return
+		}
+	}
+
+	if s.txnMarkSet != nil {
+		if len(toreify) > 0 {
+			if err := s.txnMarkSet.MarkMany(toreify); err != nil {
+				log.Warnf("error marking reified objects: %s", err)
+			}
+		}
+		if len(totrack) > 0 {
+			if err := s.txnMarkSet.MarkMany(totrack); err != nil {
+				log.Warnf("error marking tracked objects: %s", err)
+			}
+		}
+	} else {
+		// if txnActive is false these are noops
+		if len(toreify) > 0 {
+			s.trackTxnRefMany(toreify)
+		}
+		if len(totrack) > 0 {
+			s.trackTxnRefMany(totrack)
+		}
+	}
+}

blockstore/splitstore/splitstore_test.go

@@ -4,6 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -20,16 +23,19 @@ import (
 	datastore "github.com/ipfs/go-datastore"
 	dssync "github.com/ipfs/go-datastore/sync"
 	logging "github.com/ipfs/go-log/v2"
+	mh "github.com/multiformats/go-multihash"
 )
 
 func init() {
 	CompactionThreshold = 5
 	CompactionBoundary = 2
 	WarmupBoundary = 0
+	SyncWaitTime = time.Millisecond
 	logging.SetLogLevel("splitstore", "DEBUG")
 }
 
 func testSplitStore(t *testing.T, cfg *Config) {
+	ctx := context.Background()
 	chain := &mockChain{t: t}
 
 	// the myriads of stores
@@ -39,7 +45,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
 
 	// this is necessary to avoid the garbage mock puts in the blocks
 	garbage := blocks.NewBlock([]byte{1, 2, 3})
-	err := cold.Put(garbage)
+	err := cold.Put(ctx, garbage)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -60,27 +66,36 @@ func testSplitStore(t *testing.T, cfg *Config) {
 		t.Fatal(err)
 	}
 
-	err = cold.Put(blk)
+	err = cold.Put(ctx, blk)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// create a garbage block that is protected with a rgistered protector
 	protected := blocks.NewBlock([]byte("protected!"))
-	err = hot.Put(protected)
+	err = hot.Put(ctx, protected)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// and another one that is not protected
 	unprotected := blocks.NewBlock([]byte("unprotected!"))
-	err = hot.Put(unprotected)
+	err = hot.Put(ctx, unprotected)
 	if err != nil {
 		t.Fatal(err)
 	}
 
+	path, err := ioutil.TempDir("", "splitstore.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(path)
+	})
+
 	// open the splitstore
-	ss, err := Open("", ds, hot, cold, cfg)
+	ss, err := Open(path, ds, hot, cold, cfg)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -109,11 +124,11 @@ func testSplitStore(t *testing.T, cfg *Config) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		err = ss.Put(stateRoot)
+		err = ss.Put(ctx, stateRoot)
 		if err != nil {
 			t.Fatal(err)
 		}
-		err = ss.Put(sblk)
+		err = ss.Put(ctx, sblk)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -124,6 +139,10 @@ func testSplitStore(t *testing.T, cfg *Config) {
 	}
 
 	waitForCompaction := func() {
+		ss.txnSyncMx.Lock()
+		ss.txnSync = true
+		ss.txnSyncCond.Broadcast()
+		ss.txnSyncMx.Unlock()
 		for atomic.LoadInt32(&ss.compacting) == 1 {
 			time.Sleep(100 * time.Millisecond)
 		}
@@ -176,7 +195,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
 	}
 
 	// ensure our protected block is still there
-	has, err := hot.Has(protected.Cid())
+	has, err := hot.Has(ctx, protected.Cid())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -186,7 +205,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
 	}
 
 	// ensure our unprotected block is in the coldstore now
-	has, err = hot.Has(unprotected.Cid())
+	has, err = hot.Has(ctx, unprotected.Cid())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -195,7 +214,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
 		t.Fatal("unprotected block is still in hotstore")
 	}
 
-	has, err = cold.Has(unprotected.Cid())
+	has, err = cold.Has(ctx, unprotected.Cid())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -222,6 +241,7 @@ func TestSplitStoreCompactionWithBadger(t *testing.T) {
 }
 
 func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
+	ctx := context.Background()
 	chain := &mockChain{t: t}
 
 	// the myriads of stores
@@ -231,7 +251,7 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
 
 	// this is necessary to avoid the garbage mock puts in the blocks
 	garbage := blocks.NewBlock([]byte{1, 2, 3})
-	err := cold.Put(garbage)
+	err := cold.Put(ctx, garbage)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -252,13 +272,22 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	err = cold.Put(blk)
+	err = cold.Put(ctx, blk)
 	if err != nil {
 		t.Fatal(err)
 	}
 
+	path, err := ioutil.TempDir("", "splitstore.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(path)
+	})
+
 	// open the splitstore
-	ss, err := Open("", ds, hot, cold, &Config{MarkSetType: "map"})
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -288,11 +317,11 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		err = ss.Put(stateRoot)
+		err = ss.Put(ctx, stateRoot)
 		if err != nil {
 			t.Fatal(err)
 		}
-		err = ss.Put(sblk)
+		err = ss.Put(ctx, sblk)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -303,6 +332,10 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
 	}
 
 	waitForCompaction := func() {
+		ss.txnSyncMx.Lock()
+		ss.txnSync = true
+		ss.txnSyncCond.Broadcast()
+		ss.txnSyncMx.Unlock()
 		for atomic.LoadInt32(&ss.compacting) == 1 {
 			time.Sleep(100 * time.Millisecond)
 		}
@@ -355,6 +388,235 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
 	}
 }
 
+func testSplitStoreReification(t *testing.T, f func(context.Context, blockstore.Blockstore, cid.Cid) error) {
+	ds := dssync.MutexWrap(datastore.NewMapDatastore())
+	hot := newMockStore()
+	cold := newMockStore()
+
+	mkRandomBlock := func() blocks.Block {
+		data := make([]byte, 128)
+		_, err := rand.Read(data)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return blocks.NewBlock(data)
+	}
+
+	block1 := mkRandomBlock()
+	block2 := mkRandomBlock()
+	block3 := mkRandomBlock()
+
+	hdr := mock.MkBlock(nil, 0, 0)
+	hdr.Messages = block1.Cid()
+	hdr.ParentMessageReceipts = block2.Cid()
+	hdr.ParentStateRoot = block3.Cid()
+	block4, err := hdr.ToStorageBlock()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	allBlocks := []blocks.Block{block1, block2, block3, block4}
+	for _, blk := range allBlocks {
+		err := cold.Put(context.Background(), blk)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	path, err := ioutil.TempDir("", "splitstore.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(path)
+	})
+
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close() //nolint
+
+	ss.warmupEpoch = 1
+	go ss.reifyOrchestrator()
+
+	waitForReification := func() {
+		for {
+			ss.reifyMx.Lock()
+			ready := len(ss.reifyPend) == 0 && len(ss.reifyInProgress) == 0
+			ss.reifyMx.Unlock()
+
+			if ready {
+				return
+			}
+
+			time.Sleep(time.Millisecond)
+		}
+	}
+
+	// first access using the standard view
+	err = f(context.Background(), ss, block4.Cid())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// nothing should be reified
+	waitForReification()
+	for _, blk := range allBlocks {
+		has, err := hot.Has(context.Background(), blk.Cid())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("block unexpectedly reified")
+		}
+	}
+
+	// now make the hot/reifying view and ensure access reifies
+	err = f(blockstore.WithHotView(context.Background()), ss, block4.Cid())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// everything should be reified
+	waitForReification()
+	for i, blk := range allBlocks {
+		has, err := hot.Has(context.Background(), blk.Cid())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !has {
+			t.Fatalf("block%d was not reified", i+1)
+		}
+	}
+}
+
+func testSplitStoreReificationLimit(t *testing.T, f func(context.Context, blockstore.Blockstore, cid.Cid) error) {
+	ds := dssync.MutexWrap(datastore.NewMapDatastore())
+	hot := newMockStore()
+	cold := newMockStore()
+
+	mkRandomBlock := func() blocks.Block {
+		data := make([]byte, 128)
+		_, err := rand.Read(data)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return blocks.NewBlock(data)
+	}
+
+	block1 := mkRandomBlock()
+	block2 := mkRandomBlock()
+	block3 := mkRandomBlock()
+
+	hdr := mock.MkBlock(nil, 0, 0)
+	hdr.Messages = block1.Cid()
+	hdr.ParentMessageReceipts = block2.Cid()
+	hdr.ParentStateRoot = block3.Cid()
+	block4, err := hdr.ToStorageBlock()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	allBlocks := []blocks.Block{block1, block2, block3, block4}
+	for _, blk := range allBlocks {
+		err := cold.Put(context.Background(), blk)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	path, err := ioutil.TempDir("", "splitstore.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(path)
+	})
+
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close() //nolint
+
+	ss.warmupEpoch = 1
+	go ss.reifyOrchestrator()
+
+	waitForReification := func() {
+		for {
+			ss.reifyMx.Lock()
+			ready := len(ss.reifyPend) == 0 && len(ss.reifyInProgress) == 0
+			ss.reifyMx.Unlock()
+
+			if ready {
+				return
+			}
+
+			time.Sleep(time.Millisecond)
+		}
+	}
+
+	// do a hot access -- nothing should be reified as the limit should be exceeded
+	oldReifyLimit := ReifyLimit
+	ReifyLimit = 2
+	t.Cleanup(func() {
+		ReifyLimit = oldReifyLimit
+	})
+
+	err = f(blockstore.WithHotView(context.Background()), ss, block4.Cid())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	waitForReification()
+
+	for _, blk := range allBlocks {
+		has, err := hot.Has(context.Background(), blk.Cid())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("block unexpectedly reified")
+		}
+	}
+
+}
+
+func TestSplitStoreReification(t *testing.T) {
+	t.Log("test reification with Has")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.Has(ctx, c)
+		return err
+	})
+	t.Log("test reification with Get")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.Get(ctx, c)
+		return err
+	})
+	t.Log("test reification with GetSize")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.GetSize(ctx, c)
+		return err
+	})
+	t.Log("test reification with View")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		return s.View(ctx, c, func(_ []byte) error { return nil })
+	})
+	t.Log("test reification limit")
+	testSplitStoreReificationLimit(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.Has(ctx, c)
+		return err
+	})
+}
+
 type mockChain struct {
 	t testing.TB
 
@@ -424,35 +686,43 @@ func (c *mockChain) SubscribeHeadChanges(change func(revert []*types.TipSet, app
 
 type mockStore struct {
 	mx  sync.Mutex
-	set map[cid.Cid]blocks.Block
+	set map[string]blocks.Block
 }
 
 func newMockStore() *mockStore {
-	return &mockStore{set: make(map[cid.Cid]blocks.Block)}
+	return &mockStore{set: make(map[string]blocks.Block)}
 }
 
-func (b *mockStore) Has(cid cid.Cid) (bool, error) {
+func (b *mockStore) keyOf(c cid.Cid) string {
+	return string(c.Hash())
+}
+
+func (b *mockStore) cidOf(k string) cid.Cid {
+	return cid.NewCidV1(cid.Raw, mh.Multihash([]byte(k)))
+}
+
+func (b *mockStore) Has(_ context.Context, cid cid.Cid) (bool, error) {
 	b.mx.Lock()
 	defer b.mx.Unlock()
-	_, ok := b.set[cid]
+	_, ok := b.set[b.keyOf(cid)]
 	return ok, nil
 }
 
 func (b *mockStore) HashOnRead(hor bool) {}
 
-func (b *mockStore) Get(cid cid.Cid) (blocks.Block, error) {
+func (b *mockStore) Get(_ context.Context, cid cid.Cid) (blocks.Block, error) {
 	b.mx.Lock()
 	defer b.mx.Unlock()
 
-	blk, ok := b.set[cid]
+	blk, ok := b.set[b.keyOf(cid)]
 	if !ok {
 		return nil, blockstore.ErrNotFound
 	}
 	return blk, nil
 }
 
-func (b *mockStore) GetSize(cid cid.Cid) (int, error) {
-	blk, err := b.Get(cid)
+func (b *mockStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
+	blk, err := b.Get(ctx, cid)
 	if err != nil {
 		return 0, err
 	}
@@ -460,46 +730,46 @@ func (b *mockStore) GetSize(cid cid.Cid) (int, error) {
 	return len(blk.RawData()), nil
 }
 
-func (b *mockStore) View(cid cid.Cid, f func([]byte) error) error {
-	blk, err := b.Get(cid)
+func (b *mockStore) View(ctx context.Context, cid cid.Cid, f func([]byte) error) error {
+	blk, err := b.Get(ctx, cid)
 	if err != nil {
 		return err
 	}
 	return f(blk.RawData())
 }
 
-func (b *mockStore) Put(blk blocks.Block) error {
+func (b *mockStore) Put(_ context.Context, blk blocks.Block) error {
 	b.mx.Lock()
 	defer b.mx.Unlock()
 
-	b.set[blk.Cid()] = blk
+	b.set[b.keyOf(blk.Cid())] = blk
 	return nil
 }
 
-func (b *mockStore) PutMany(blks []blocks.Block) error {
+func (b *mockStore) PutMany(_ context.Context, blks []blocks.Block) error {
 	b.mx.Lock()
 	defer b.mx.Unlock()
 
 	for _, blk := range blks {
-		b.set[blk.Cid()] = blk
+		b.set[b.keyOf(blk.Cid())] = blk
 	}
 	return nil
 }
 
-func (b *mockStore) DeleteBlock(cid cid.Cid) error {
+func (b *mockStore) DeleteBlock(_ context.Context, cid cid.Cid) error {
 	b.mx.Lock()
 	defer b.mx.Unlock()
 
-	delete(b.set, cid)
+	delete(b.set, b.keyOf(cid))
 	return nil
 }
 
-func (b *mockStore) DeleteMany(cids []cid.Cid) error {
+func (b *mockStore) DeleteMany(_ context.Context, cids []cid.Cid) error {
 	b.mx.Lock()
 	defer b.mx.Unlock()
 
 	for _, c := range cids {
-		delete(b.set, c)
+		delete(b.set, b.keyOf(c))
 	}
 	return nil
 }
@@ -513,7 +783,7 @@ func (b *mockStore) ForEachKey(f func(cid.Cid) error) error {
 	defer b.mx.Unlock()
 
 	for c := range b.set {
-		err := f(c)
+		err := f(b.cidOf(c))
 		if err != nil {
 			return err
 		}

blockstore/splitstore/splitstore_warmup.go

@@ -1,6 +1,7 @@
 package splitstore
 
 import (
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -55,12 +56,13 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 	if WarmupBoundary < epoch {
 		boundaryEpoch = epoch - WarmupBoundary
 	}
+	var mx sync.Mutex
 	batchHot := make([]blocks.Block, 0, batchSize)
-	count := int64(0)
-	xcount := int64(0)
-	missing := int64(0)
+	count := new(int64)
+	xcount := new(int64)
+	missing := new(int64)
 
-	visitor, err := s.markSetEnv.CreateVisitor("warmup", 0)
+	visitor, err := s.markSetEnv.New("warmup", 0)
 	if err != nil {
 		return xerrors.Errorf("error creating visitor: %w", err)
 	}
@@ -73,9 +75,9 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 				return errStopWalk
 			}
 
-			count++
+			atomic.AddInt64(count, 1)
 
-			has, err := s.hot.Has(c)
+			has, err := s.hot.Has(s.ctx, c)
 			if err != nil {
 				return err
 			}
@@ -84,25 +86,28 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 				return nil
 			}
 
-			blk, err := s.cold.Get(c)
+			blk, err := s.cold.Get(s.ctx, c)
 			if err != nil {
 				if err == bstore.ErrNotFound {
-					missing++
+					atomic.AddInt64(missing, 1)
 					return errStopWalk
 				}
 				return err
 			}
 
-			xcount++
+			atomic.AddInt64(xcount, 1)
 
+			mx.Lock()
 			batchHot = append(batchHot, blk)
 			if len(batchHot) == batchSize {
-				err = s.hot.PutMany(batchHot)
+				err = s.hot.PutMany(s.ctx, batchHot)
 				if err != nil {
+					mx.Unlock()
 					return err
 				}
 				batchHot = batchHot[:0]
 			}
+			mx.Unlock()
 
 			return nil
 		})
@@ -112,22 +117,22 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 	}
 
 	if len(batchHot) > 0 {
-		err = s.hot.PutMany(batchHot)
+		err = s.hot.PutMany(s.ctx, batchHot)
 		if err != nil {
 			return err
 		}
 	}
 
-	log.Infow("warmup stats", "visited", count, "warm", xcount, "missing", missing)
+	log.Infow("warmup stats", "visited", *count, "warm", *xcount, "missing", *missing)
 
-	s.markSetSize = count + count>>2 // overestimate a bit
-	err = s.ds.Put(markSetSizeKey, int64ToBytes(s.markSetSize))
+	s.markSetSize = *count + *count>>2 // overestimate a bit
+	err = s.ds.Put(s.ctx, markSetSizeKey, int64ToBytes(s.markSetSize))
 	if err != nil {
 		log.Warnf("error saving mark set size: %s", err)
 	}
 
 	// save the warmup epoch
-	err = s.ds.Put(warmupEpochKey, epochToBytes(epoch))
+	err = s.ds.Put(s.ctx, warmupEpochKey, epochToBytes(epoch))
 	if err != nil {
 		return xerrors.Errorf("error saving warm up epoch: %w", err)
 	}
@@ -136,7 +141,7 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 	s.mx.Unlock()
 
 	// also save the compactionIndex, as this is used as an indicator of warmup for upgraded nodes
-	err = s.ds.Put(compactionIndexKey, int64ToBytes(s.compactionIndex))
+	err = s.ds.Put(s.ctx, compactionIndexKey, int64ToBytes(s.compactionIndex))
 	if err != nil {
 		return xerrors.Errorf("error saving compaction index: %w", err)
 	}

blockstore/splitstore/visitor.go

@@ -1,6 +1,8 @@
 package splitstore
 
 import (
+	"sync"
+
 	cid "github.com/ipfs/go-cid"
 )
 
@@ -17,16 +19,42 @@ func (v *noopVisitor) Visit(_ cid.Cid) (bool, error) {
 	return true, nil
 }
 
-type cidSetVisitor struct {
+type tmpVisitor struct {
 	set *cid.Set
 }
 
-var _ ObjectVisitor = (*cidSetVisitor)(nil)
+var _ ObjectVisitor = (*tmpVisitor)(nil)
+
+func (v *tmpVisitor) Visit(c cid.Cid) (bool, error) {
+	if isUnitaryObject(c) {
+		return false, nil
+	}
 
-func (v *cidSetVisitor) Visit(c cid.Cid) (bool, error) {
 	return v.set.Visit(c), nil
 }
 
-func tmpVisitor() ObjectVisitor {
-	return &cidSetVisitor{set: cid.NewSet()}
+func newTmpVisitor() ObjectVisitor {
+	return &tmpVisitor{set: cid.NewSet()}
+}
+
+type concurrentVisitor struct {
+	mx  sync.Mutex
+	set *cid.Set
+}
+
+var _ ObjectVisitor = (*concurrentVisitor)(nil)
+
+func newConcurrentVisitor() *concurrentVisitor {
+	return &concurrentVisitor{set: cid.NewSet()}
+}
+
+func (v *concurrentVisitor) Visit(c cid.Cid) (bool, error) {
+	if isUnitaryObject(c) {
+		return false, nil
+	}
+
+	v.mx.Lock()
+	defer v.mx.Unlock()
+
+	return v.set.Visit(c), nil
 }

blockstore/sync.go

@@ -20,53 +20,53 @@ type SyncBlockstore struct {
 	bs MemBlockstore // specifically use a memStore to save indirection overhead.
 }
 
-func (m *SyncBlockstore) DeleteBlock(k cid.Cid) error {
+func (m *SyncBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	return m.bs.DeleteBlock(k)
+	return m.bs.DeleteBlock(ctx, k)
 }
 
-func (m *SyncBlockstore) DeleteMany(ks []cid.Cid) error {
+func (m *SyncBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	return m.bs.DeleteMany(ks)
+	return m.bs.DeleteMany(ctx, ks)
 }
 
-func (m *SyncBlockstore) Has(k cid.Cid) (bool, error) {
+func (m *SyncBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error) {
 	m.mu.RLock()
 	defer m.mu.RUnlock()
-	return m.bs.Has(k)
+	return m.bs.Has(ctx, k)
 }
 
-func (m *SyncBlockstore) View(k cid.Cid, callback func([]byte) error) error {
+func (m *SyncBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error {
 	m.mu.RLock()
 	defer m.mu.RUnlock()
 
-	return m.bs.View(k, callback)
+	return m.bs.View(ctx, k, callback)
 }
 
-func (m *SyncBlockstore) Get(k cid.Cid) (blocks.Block, error) {
+func (m *SyncBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) {
 	m.mu.RLock()
 	defer m.mu.RUnlock()
-	return m.bs.Get(k)
+	return m.bs.Get(ctx, k)
 }
 
-func (m *SyncBlockstore) GetSize(k cid.Cid) (int, error) {
+func (m *SyncBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
 	m.mu.RLock()
 	defer m.mu.RUnlock()
-	return m.bs.GetSize(k)
+	return m.bs.GetSize(ctx, k)
 }
 
-func (m *SyncBlockstore) Put(b blocks.Block) error {
+func (m *SyncBlockstore) Put(ctx context.Context, b blocks.Block) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	return m.bs.Put(b)
+	return m.bs.Put(ctx, b)
 }
 
-func (m *SyncBlockstore) PutMany(bs []blocks.Block) error {
+func (m *SyncBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	return m.bs.PutMany(bs)
+	return m.bs.PutMany(ctx, bs)
 }
 
 func (m *SyncBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {

blockstore/timed.go

@@ -92,28 +92,28 @@ func (t *TimedCacheBlockstore) rotate() {
 	t.mu.Unlock()
 }
 
-func (t *TimedCacheBlockstore) Put(b blocks.Block) error {
+func (t *TimedCacheBlockstore) Put(ctx context.Context, b blocks.Block) error {
 	// Don't check the inactive set here. We want to keep this block for at
 	// least one interval.
 	t.mu.Lock()
 	defer t.mu.Unlock()
-	return t.active.Put(b)
+	return t.active.Put(ctx, b)
 }
 
-func (t *TimedCacheBlockstore) PutMany(bs []blocks.Block) error {
+func (t *TimedCacheBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
-	return t.active.PutMany(bs)
+	return t.active.PutMany(ctx, bs)
 }
 
-func (t *TimedCacheBlockstore) View(k cid.Cid, callback func([]byte) error) error {
+func (t *TimedCacheBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error {
 	// The underlying blockstore is always a "mem" blockstore so there's no difference,
 	// from a performance perspective, between view & get. So we call Get to avoid
 	// calling an arbitrary callback while holding a lock.
 	t.mu.RLock()
-	block, err := t.active.Get(k)
+	block, err := t.active.Get(ctx, k)
 	if err == ErrNotFound {
-		block, err = t.inactive.Get(k)
+		block, err = t.inactive.Get(ctx, k)
 	}
 	t.mu.RUnlock()
 
@@ -123,51 +123,51 @@ func (t *TimedCacheBlockstore) View(k cid.Cid, callback func([]byte) error) erro
 	return callback(block.RawData())
 }
 
-func (t *TimedCacheBlockstore) Get(k cid.Cid) (blocks.Block, error) {
+func (t *TimedCacheBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
-	b, err := t.active.Get(k)
+	b, err := t.active.Get(ctx, k)
 	if err == ErrNotFound {
-		b, err = t.inactive.Get(k)
+		b, err = t.inactive.Get(ctx, k)
 	}
 	return b, err
 }
 
-func (t *TimedCacheBlockstore) GetSize(k cid.Cid) (int, error) {
+func (t *TimedCacheBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
-	size, err := t.active.GetSize(k)
+	size, err := t.active.GetSize(ctx, k)
 	if err == ErrNotFound {
-		size, err = t.inactive.GetSize(k)
+		size, err = t.inactive.GetSize(ctx, k)
 	}
 	return size, err
 }
 
-func (t *TimedCacheBlockstore) Has(k cid.Cid) (bool, error) {
+func (t *TimedCacheBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error) {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
-	if has, err := t.active.Has(k); err != nil {
+	if has, err := t.active.Has(ctx, k); err != nil {
 		return false, err
 	} else if has {
 		return true, nil
 	}
-	return t.inactive.Has(k)
+	return t.inactive.Has(ctx, k)
 }
 
 func (t *TimedCacheBlockstore) HashOnRead(_ bool) {
 	// no-op
 }
 
-func (t *TimedCacheBlockstore) DeleteBlock(k cid.Cid) error {
+func (t *TimedCacheBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
-	return multierr.Combine(t.active.DeleteBlock(k), t.inactive.DeleteBlock(k))
+	return multierr.Combine(t.active.DeleteBlock(ctx, k), t.inactive.DeleteBlock(ctx, k))
 }
 
-func (t *TimedCacheBlockstore) DeleteMany(ks []cid.Cid) error {
+func (t *TimedCacheBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
-	return multierr.Combine(t.active.DeleteMany(ks), t.inactive.DeleteMany(ks))
+	return multierr.Combine(t.active.DeleteMany(ctx, ks), t.inactive.DeleteMany(ctx, ks))
 }
 
 func (t *TimedCacheBlockstore) AllKeysChan(_ context.Context) (<-chan cid.Cid, error) {

blockstore/timed_test.go

@@ -19,6 +19,8 @@ func TestTimedCacheBlockstoreSimple(t *testing.T) {
 	tc.clock = mClock
 	tc.doneRotatingCh = make(chan struct{})
 
+	ctx := context.Background()
+
 	_ = tc.Start(context.Background())
 	mClock.Add(1) // IDK why it is needed but it makes it work
 
@@ -27,18 +29,18 @@ func TestTimedCacheBlockstoreSimple(t *testing.T) {
 	}()
 
 	b1 := blocks.NewBlock([]byte("foo"))
-	require.NoError(t, tc.Put(b1))
+	require.NoError(t, tc.Put(ctx, b1))
 
 	b2 := blocks.NewBlock([]byte("bar"))
-	require.NoError(t, tc.Put(b2))
+	require.NoError(t, tc.Put(ctx, b2))
 
 	b3 := blocks.NewBlock([]byte("baz"))
 
-	b1out, err := tc.Get(b1.Cid())
+	b1out, err := tc.Get(ctx, b1.Cid())
 	require.NoError(t, err)
 	require.Equal(t, b1.RawData(), b1out.RawData())
 
-	has, err := tc.Has(b1.Cid())
+	has, err := tc.Has(ctx, b1.Cid())
 	require.NoError(t, err)
 	require.True(t, has)
 
@@ -46,17 +48,17 @@ func TestTimedCacheBlockstoreSimple(t *testing.T) {
 	<-tc.doneRotatingCh
 
 	// We should still have everything.
-	has, err = tc.Has(b1.Cid())
+	has, err = tc.Has(ctx, b1.Cid())
 	require.NoError(t, err)
 	require.True(t, has)
 
-	has, err = tc.Has(b2.Cid())
+	has, err = tc.Has(ctx, b2.Cid())
 	require.NoError(t, err)
 	require.True(t, has)
 
 	// extend b2, add b3.
-	require.NoError(t, tc.Put(b2))
-	require.NoError(t, tc.Put(b3))
+	require.NoError(t, tc.Put(ctx, b2))
+	require.NoError(t, tc.Put(ctx, b3))
 
 	// all keys once.
 	allKeys, err := tc.AllKeysChan(context.Background())
@@ -71,15 +73,15 @@ func TestTimedCacheBlockstoreSimple(t *testing.T) {
 	<-tc.doneRotatingCh
 	// should still have b2, and b3, but not b1
 
-	has, err = tc.Has(b1.Cid())
+	has, err = tc.Has(ctx, b1.Cid())
 	require.NoError(t, err)
 	require.False(t, has)
 
-	has, err = tc.Has(b2.Cid())
+	has, err = tc.Has(ctx, b2.Cid())
 	require.NoError(t, err)
 	require.True(t, has)
 
-	has, err = tc.Has(b3.Cid())
+	has, err = tc.Has(ctx, b3.Cid())
 	require.NoError(t, err)
 	require.True(t, has)
 }

blockstore/union.go

@@ -19,72 +19,72 @@ func Union(stores ...Blockstore) Blockstore {
 	return unionBlockstore(stores)
 }
 
-func (m unionBlockstore) Has(cid cid.Cid) (has bool, err error) {
+func (m unionBlockstore) Has(ctx context.Context, cid cid.Cid) (has bool, err error) {
 	for _, bs := range m {
-		if has, err = bs.Has(cid); has || err != nil {
+		if has, err = bs.Has(ctx, cid); has || err != nil {
 			break
 		}
 	}
 	return has, err
 }
 
-func (m unionBlockstore) Get(cid cid.Cid) (blk blocks.Block, err error) {
+func (m unionBlockstore) Get(ctx context.Context, cid cid.Cid) (blk blocks.Block, err error) {
 	for _, bs := range m {
-		if blk, err = bs.Get(cid); err == nil || err != ErrNotFound {
+		if blk, err = bs.Get(ctx, cid); err == nil || err != ErrNotFound {
 			break
 		}
 	}
 	return blk, err
 }
 
-func (m unionBlockstore) View(cid cid.Cid, callback func([]byte) error) (err error) {
+func (m unionBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) (err error) {
 	for _, bs := range m {
-		if err = bs.View(cid, callback); err == nil || err != ErrNotFound {
+		if err = bs.View(ctx, cid, callback); err == nil || err != ErrNotFound {
 			break
 		}
 	}
 	return err
 }
 
-func (m unionBlockstore) GetSize(cid cid.Cid) (size int, err error) {
+func (m unionBlockstore) GetSize(ctx context.Context, cid cid.Cid) (size int, err error) {
 	for _, bs := range m {
-		if size, err = bs.GetSize(cid); err == nil || err != ErrNotFound {
+		if size, err = bs.GetSize(ctx, cid); err == nil || err != ErrNotFound {
 			break
 		}
 	}
 	return size, err
 }
 
-func (m unionBlockstore) Put(block blocks.Block) (err error) {
+func (m unionBlockstore) Put(ctx context.Context, block blocks.Block) (err error) {
 	for _, bs := range m {
-		if err = bs.Put(block); err != nil {
+		if err = bs.Put(ctx, block); err != nil {
 			break
 		}
 	}
 	return err
 }
 
-func (m unionBlockstore) PutMany(blks []blocks.Block) (err error) {
+func (m unionBlockstore) PutMany(ctx context.Context, blks []blocks.Block) (err error) {
 	for _, bs := range m {
-		if err = bs.PutMany(blks); err != nil {
+		if err = bs.PutMany(ctx, blks); err != nil {
 			break
 		}
 	}
 	return err
 }
 
-func (m unionBlockstore) DeleteBlock(cid cid.Cid) (err error) {
+func (m unionBlockstore) DeleteBlock(ctx context.Context, cid cid.Cid) (err error) {
 	for _, bs := range m {
-		if err = bs.DeleteBlock(cid); err != nil {
+		if err = bs.DeleteBlock(ctx, cid); err != nil {
 			break
 		}
 	}
 	return err
 }
 
-func (m unionBlockstore) DeleteMany(cids []cid.Cid) (err error) {
+func (m unionBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) (err error) {
 	for _, bs := range m {
-		if err = bs.DeleteMany(cids); err != nil {
+		if err = bs.DeleteMany(ctx, cids); err != nil {
 			break
 		}
 	}

blockstore/union_test.go

@@ -15,79 +15,81 @@ var (
 )
 
 func TestUnionBlockstore_Get(t *testing.T) {
+	ctx := context.Background()
 	m1 := NewMemory()
 	m2 := NewMemory()
 
-	_ = m1.Put(b1)
-	_ = m2.Put(b2)
+	_ = m1.Put(ctx, b1)
+	_ = m2.Put(ctx, b2)
 
 	u := Union(m1, m2)
 
-	v1, err := u.Get(b1.Cid())
+	v1, err := u.Get(ctx, b1.Cid())
 	require.NoError(t, err)
 	require.Equal(t, b1.RawData(), v1.RawData())
 
-	v2, err := u.Get(b2.Cid())
+	v2, err := u.Get(ctx, b2.Cid())
 	require.NoError(t, err)
 	require.Equal(t, b2.RawData(), v2.RawData())
 }
 
 func TestUnionBlockstore_Put_PutMany_Delete_AllKeysChan(t *testing.T) {
+	ctx := context.Background()
 	m1 := NewMemory()
 	m2 := NewMemory()
 
 	u := Union(m1, m2)
 
-	err := u.Put(b0)
+	err := u.Put(ctx, b0)
 	require.NoError(t, err)
 
 	var has bool
 
 	// write was broadcasted to all stores.
-	has, _ = m1.Has(b0.Cid())
+	has, _ = m1.Has(ctx, b0.Cid())
 	require.True(t, has)
 
-	has, _ = m2.Has(b0.Cid())
+	has, _ = m2.Has(ctx, b0.Cid())
 	require.True(t, has)
 
-	has, _ = u.Has(b0.Cid())
+	has, _ = u.Has(ctx, b0.Cid())
 	require.True(t, has)
 
 	// put many.
-	err = u.PutMany([]blocks.Block{b1, b2})
+	err = u.PutMany(ctx, []blocks.Block{b1, b2})
 	require.NoError(t, err)
 
 	// write was broadcasted to all stores.
-	has, _ = m1.Has(b1.Cid())
+	has, _ = m1.Has(ctx, b1.Cid())
 	require.True(t, has)
 
-	has, _ = m1.Has(b2.Cid())
+	has, _ = m1.Has(ctx, b2.Cid())
 	require.True(t, has)
 
-	has, _ = m2.Has(b1.Cid())
+	has, _ = m2.Has(ctx, b1.Cid())
 	require.True(t, has)
 
-	has, _ = m2.Has(b2.Cid())
+	has, _ = m2.Has(ctx, b2.Cid())
 	require.True(t, has)
 
 	// also in the union store.
-	has, _ = u.Has(b1.Cid())
+	has, _ = u.Has(ctx, b1.Cid())
 	require.True(t, has)
 
-	has, _ = u.Has(b2.Cid())
+	has, _ = u.Has(ctx, b2.Cid())
 	require.True(t, has)
 
 	// deleted from all stores.
-	err = u.DeleteBlock(b1.Cid())
+	err = u.DeleteBlock(ctx, b1.Cid())
 	require.NoError(t, err)
 
-	has, _ = u.Has(b1.Cid())
+	has, _ = u.Has(ctx, b1.Cid())
 	require.False(t, has)
 
-	has, _ = m1.Has(b1.Cid())
+	has, _ = m1.Has(ctx, b1.Cid())
 	require.False(t, has)
 
-	has, _ = m2.Has(b1.Cid())
+	has, _ = m2.Has(ctx, b1.Cid())
 	require.False(t, has)
 
 	// check that AllKeysChan returns b0 and b2, twice (once per backing store)

build/params_shared_funcs.go

@@ -13,6 +13,17 @@ import (
 
 func BlocksTopic(netName dtypes.NetworkName) string   { return "/fil/blocks/" + string(netName) }
 func MessagesTopic(netName dtypes.NetworkName) string { return "/fil/msgs/" + string(netName) }
+func IndexerIngestTopic(netName dtypes.NetworkName) string {
+
+	nn := string(netName)
+	// The network name testnetnet is here for historical reasons.
+	// Going forward we aim to use the name `mainnet` where possible.
+	if nn == "testnetnet" {
+		nn = "mainnet"
+	}
+
+	return "/indexer/ingest/" + nn
+}
 func DhtProtocolName(netName dtypes.NetworkName) protocol.ID {
 	return protocol.ID("/fil/kad/" + string(netName))
 }

build/version.go

@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }
 
 // BuildVersion is the local build version
-const BuildVersion = "1.14.2"
+const BuildVersion = "1.15.1"
 
 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

chain/actors/builtin/verifreg/actor.go.template

@@ -16,6 +16,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/types"
+	verifreg7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg"
 )
 
 func init() {
@@ -62,6 +63,11 @@ func GetActorCodeID(av actors.Version) (cid.Cid, error) {
 	return cid.Undef, xerrors.Errorf("unknown actor version %d", av)
 }
 
+type RemoveDataCapProposal = verifreg{{.latestVersion}}.RemoveDataCapProposal
+type RemoveDataCapRequest = verifreg{{.latestVersion}}.RemoveDataCapRequest
+type RemoveDataCapParams = verifreg{{.latestVersion}}.RemoveDataCapParams
+type RmDcProposalID = verifreg{{.latestVersion}}.RmDcProposalID
+const SignatureDomainSeparation_RemoveDataCap = verifreg{{.latestVersion}}.SignatureDomainSeparation_RemoveDataCap
 
 type State interface {
 	cbor.Marshaler
@@ -69,6 +75,7 @@ type State interface {
 	RootKey() (address.Address, error)
 	VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error)
 	VerifierDataCap(address.Address) (bool, abi.StoragePower, error)
+	RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error)
 	ForEachVerifier(func(addr address.Address, dcap abi.StoragePower) error) error
 	ForEachClient(func(addr address.Address, dcap abi.StoragePower) error) error
 	GetState() interface{}

chain/actors/builtin/verifreg/state.go.template

@@ -61,6 +61,10 @@ func (s *state{{.v}}) VerifierDataCap(addr address.Address) (bool, abi.StoragePo
 	return getDataCap(s.store, actors.Version{{.v}}, s.verifiers, addr)
 }
 
+func (s *state{{.v}}) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version{{.v}}, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state{{.v}}) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version{{.v}}, s.verifiers, cb)
 }
@@ -77,6 +81,11 @@ func (s *state{{.v}}) verifiers() (adt.Map, error) {
 	return adt{{.v}}.AsMap(s.store, s.Verifiers{{if (ge .v 3)}}, builtin{{.v}}.DefaultHamtBitwidth{{end}})
 }
 
+func (s *state{{.v}}) removeDataCapProposalIDs() (adt.Map, error) {
+    {{if le .v 6}}return nil, nil
+    {{else}}return adt{{.v}}.AsMap(s.store, s.RemoveDataCapProposalIDs, builtin{{.v}}.DefaultHamtBitwidth){{end}}
+}
+
 func (s *state{{.v}}) GetState() interface{} {
 	return &s.State
 }

chain/actors/builtin/verifreg/util.go

@@ -6,6 +6,7 @@ import (
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
+	"github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg"
 	"golang.org/x/xerrors"
 )
 
@@ -50,3 +51,28 @@ func forEachCap(store adt.Store, ver actors.Version, root rootFunc, cb func(addr
 		return cb(a, dcap)
 	})
 }
+
+func getRemoveDataCapProposalID(store adt.Store, ver actors.Version, root rootFunc, verifier address.Address, client address.Address) (bool, uint64, error) {
+	if verifier.Protocol() != address.ID {
+		return false, 0, xerrors.Errorf("can only look up ID addresses")
+	}
+	if client.Protocol() != address.ID {
+		return false, 0, xerrors.Errorf("can only look up ID addresses")
+	}
+	vh, err := root()
+	if err != nil {
+		return false, 0, xerrors.Errorf("loading verifreg: %w", err)
+	}
+	if vh == nil {
+		return false, 0, xerrors.Errorf("remove data cap proposal hamt not found. you are probably using an incompatible version of actors")
+	}
+
+	var id verifreg.RmDcProposalID
+	if found, err := vh.Get(abi.NewAddrPairKey(verifier, client), &id); err != nil {
+		return false, 0, xerrors.Errorf("looking up addr pair: %w", err)
+	} else if !found {
+		return false, 0, nil
+	}
+
+	return true, id.ProposalID, nil
+}

chain/actors/builtin/verifreg/v0.go

@@ -53,6 +53,10 @@ func (s *state0) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version0, s.verifiers, addr)
 }
 
+func (s *state0) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version0, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state0) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version0, s.verifiers, cb)
 }
@@ -69,6 +73,11 @@ func (s *state0) verifiers() (adt.Map, error) {
 	return adt0.AsMap(s.store, s.Verifiers)
 }
 
+func (s *state0) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state0) GetState() interface{} {
 	return &s.State
 }

chain/actors/builtin/verifreg/v2.go

@@ -53,6 +53,10 @@ func (s *state2) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version2, s.verifiers, addr)
 }
 
+func (s *state2) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version2, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state2) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version2, s.verifiers, cb)
 }
@@ -69,6 +73,11 @@ func (s *state2) verifiers() (adt.Map, error) {
 	return adt2.AsMap(s.store, s.Verifiers)
 }
 
+func (s *state2) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state2) GetState() interface{} {
 	return &s.State
 }

chain/actors/builtin/verifreg/v3.go

@@ -54,6 +54,10 @@ func (s *state3) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version3, s.verifiers, addr)
 }
 
+func (s *state3) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version3, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state3) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version3, s.verifiers, cb)
 }
@@ -70,6 +74,11 @@ func (s *state3) verifiers() (adt.Map, error) {
 	return adt3.AsMap(s.store, s.Verifiers, builtin3.DefaultHamtBitwidth)
 }
 
+func (s *state3) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state3) GetState() interface{} {
 	return &s.State
 }

chain/actors/builtin/verifreg/v4.go

@@ -54,6 +54,10 @@ func (s *state4) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version4, s.verifiers, addr)
 }
 
+func (s *state4) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version4, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state4) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version4, s.verifiers, cb)
 }
@@ -70,6 +74,11 @@ func (s *state4) verifiers() (adt.Map, error) {
 	return adt4.AsMap(s.store, s.Verifiers, builtin4.DefaultHamtBitwidth)
 }
 
+func (s *state4) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state4) GetState() interface{} {
 	return &s.State
 }

chain/actors/builtin/verifreg/v5.go

@@ -54,6 +54,10 @@ func (s *state5) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version5, s.verifiers, addr)
 }
 
+func (s *state5) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version5, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state5) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version5, s.verifiers, cb)
 }
@@ -70,6 +74,11 @@ func (s *state5) verifiers() (adt.Map, error) {
 	return adt5.AsMap(s.store, s.Verifiers, builtin5.DefaultHamtBitwidth)
 }
 
+func (s *state5) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state5) GetState() interface{} {
 	return &s.State
 }

chain/actors/builtin/verifreg/v6.go

@@ -54,6 +54,10 @@ func (s *state6) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version6, s.verifiers, addr)
 }
 
+func (s *state6) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version6, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state6) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version6, s.verifiers, cb)
 }
@@ -70,6 +74,11 @@ func (s *state6) verifiers() (adt.Map, error) {
 	return adt6.AsMap(s.store, s.Verifiers, builtin6.DefaultHamtBitwidth)
 }
 
+func (s *state6) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state6) GetState() interface{} {
 	return &s.State
 }

chain/actors/builtin/verifreg/v7.go

@@ -54,6 +54,10 @@ func (s *state7) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version7, s.verifiers, addr)
 }
 
+func (s *state7) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version7, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state7) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version7, s.verifiers, cb)
 }
@@ -70,6 +74,10 @@ func (s *state7) verifiers() (adt.Map, error) {
 	return adt7.AsMap(s.store, s.Verifiers, builtin7.DefaultHamtBitwidth)
 }
 
+func (s *state7) removeDataCapProposalIDs() (adt.Map, error) {
+	return adt7.AsMap(s.store, s.RemoveDataCapProposalIDs, builtin7.DefaultHamtBitwidth)
+}
+
 func (s *state7) GetState() interface{} {
 	return &s.State
 }

chain/actors/builtin/verifreg/verifreg.go

@@ -27,6 +27,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/types"
+	verifreg7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg"
 )
 
 func init() {
@@ -151,12 +152,20 @@ func GetActorCodeID(av actors.Version) (cid.Cid, error) {
 	return cid.Undef, xerrors.Errorf("unknown actor version %d", av)
 }
 
+type RemoveDataCapProposal = verifreg7.RemoveDataCapProposal
+type RemoveDataCapRequest = verifreg7.RemoveDataCapRequest
+type RemoveDataCapParams = verifreg7.RemoveDataCapParams
+type RmDcProposalID = verifreg7.RmDcProposalID
+
+const SignatureDomainSeparation_RemoveDataCap = verifreg7.SignatureDomainSeparation_RemoveDataCap
+
 type State interface {
 	cbor.Marshaler
 
 	RootKey() (address.Address, error)
 	VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error)
 	VerifierDataCap(address.Address) (bool, abi.StoragePower, error)
+	RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error)
 	ForEachVerifier(func(addr address.Address, dcap abi.StoragePower) error) error
 	ForEachClient(func(addr address.Address, dcap abi.StoragePower) error) error
 	GetState() interface{}

chain/beacon/drand/drand.go

@@ -142,7 +142,7 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re
 
 	go func() {
 		start := build.Clock.Now()
-		log.Infow("start fetching randomness", "round", round)
+		log.Debugw("start fetching randomness", "round", round)
 		resp, err := db.client.Get(ctx, round)
 
 		var br beacon.Response
@@ -152,7 +152,7 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re
 			br.Entry.Round = resp.Round()
 			br.Entry.Data = resp.Signature()
 		}
-		log.Infow("done fetching randomness", "round", round, "took", build.Clock.Since(start))
+		log.Debugw("done fetching randomness", "round", round, "took", build.Clock.Since(start))
 		out <- br
 		close(out)
 	}()

chain/checkpoint.go

@@ -13,7 +13,7 @@ func (syncer *Syncer) SyncCheckpoint(ctx context.Context, tsk types.TipSetKey) e
 		return xerrors.Errorf("called with empty tsk")
 	}
 
-	ts, err := syncer.ChainStore().LoadTipSet(tsk)
+	ts, err := syncer.ChainStore().LoadTipSet(ctx, tsk)
 	if err != nil {
 		tss, err := syncer.Exchange.GetBlocks(ctx, tsk, 1)
 		if err != nil {
@@ -28,7 +28,7 @@ func (syncer *Syncer) SyncCheckpoint(ctx context.Context, tsk types.TipSetKey) e
 		return xerrors.Errorf("failed to switch chain when syncing checkpoint: %w", err)
 	}
 
-	if err := syncer.ChainStore().SetCheckpoint(ts); err != nil {
+	if err := syncer.ChainStore().SetCheckpoint(ctx, ts); err != nil {
 		return xerrors.Errorf("failed to set the chain checkpoint: %w", err)
 	}
 
@@ -41,7 +41,7 @@ func (syncer *Syncer) switchChain(ctx context.Context, ts *types.TipSet) error {
 		return nil
 	}
 
-	if anc, err := syncer.store.IsAncestorOf(ts, hts); err == nil && anc {
+	if anc, err := syncer.store.IsAncestorOf(ctx, ts, hts); err == nil && anc {
 		return nil
 	}
 
@@ -50,7 +50,7 @@ func (syncer *Syncer) switchChain(ctx context.Context, ts *types.TipSet) error {
 		return xerrors.Errorf("failed to collect chain for checkpoint: %w", err)
 	}
 
-	if err := syncer.ChainStore().SetHead(ts); err != nil {
+	if err := syncer.ChainStore().SetHead(ctx, ts); err != nil {
 		return xerrors.Errorf("failed to set the chain head: %w", err)
 	}
 	return nil

chain/consensus/filcns/compute_state.go

@@ -2,6 +2,7 @@ package filcns
 
 import (
 	"context"
+	"os"
 	"sync/atomic"
 
 	"github.com/filecoin-project/lotus/chain/rand"
@@ -32,6 +33,7 @@ import (
 
 	/* inline-gen end */
 
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -92,29 +94,38 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, sm *stmgr.StateManager
 		partDone()
 	}()
 
-	makeVmWithBaseState := func(base cid.Cid) (*vm.VM, error) {
+	ctx = blockstore.WithHotView(ctx)
+	makeVmWithBaseStateAndEpoch := func(base cid.Cid, e abi.ChainEpoch) (vm.Interface, error) {
 		vmopt := &vm.VMOpts{
 			StateBase:      base,
-			Epoch:          epoch,
+			Epoch:          e,
 			Rand:           r,
 			Bstore:         sm.ChainStore().StateBlockstore(),
 			Actors:         NewActorRegistry(),
 			Syscalls:       sm.Syscalls,
 			CircSupplyCalc: sm.GetVMCirculatingSupply,
-			NtwkVersion:    sm.GetNtwkVersion,
+			NetworkVersion: sm.GetNetworkVersion(ctx, e),
 			BaseFee:        baseFee,
 			LookbackState:  stmgr.LookbackStateGetterForTipset(sm, ts),
 		}
 
+		if os.Getenv("LOTUS_USE_FVM_EXPERIMENTAL") == "1" {
+			// This is needed so that the FVM does not have to duplicate the genesis vesting schedule, one
+			// of the components of the circ supply calc.
+			// This field is NOT needed by the LegacyVM, and also NOT needed by the FVM from v15 onwards.
+			filVested, err := sm.GetFilVested(ctx, e)
+			if err != nil {
+				return nil, err
+			}
+
+			vmopt.FilVested = filVested
+			return vm.NewFVM(ctx, vmopt)
+		}
+
 		return sm.VMConstructor()(ctx, vmopt)
 	}
 
-	vmi, err := makeVmWithBaseState(pstate)
-	if err != nil {
-		return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err)
-	}
-
-	runCron := func(epoch abi.ChainEpoch) error {
+	runCron := func(vmCron vm.Interface, epoch abi.ChainEpoch) error {
 		cronMsg := &types.Message{
 			To:         cron.Address,
 			From:       builtin.SystemActorAddr,
@@ -126,59 +137,58 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, sm *stmgr.StateManager
 			Method:     cron.Methods.EpochTick,
 			Params:     nil,
 		}
-		ret, err := vmi.ApplyImplicitMessage(ctx, cronMsg)
+		ret, err := vmCron.ApplyImplicitMessage(ctx, cronMsg)
 		if err != nil {
-			return err
+			return xerrors.Errorf("running cron: %w", err)
 		}
+
 		if em != nil {
 			if err := em.MessageApplied(ctx, ts, cronMsg.Cid(), cronMsg, ret, true); err != nil {
 				return xerrors.Errorf("callback failed on cron message: %w", err)
 			}
 		}
 		if ret.ExitCode != 0 {
-			return xerrors.Errorf("CheckProofSubmissions exit was non-zero: %d", ret.ExitCode)
+			return xerrors.Errorf("cron exit was non-zero: %d", ret.ExitCode)
 		}
 
 		return nil
 	}
 
 	for i := parentEpoch; i < epoch; i++ {
+		var err error
 		if i > parentEpoch {
-			// run cron for null rounds if any
-			if err := runCron(i); err != nil {
-				return cid.Undef, cid.Undef, err
+			vmCron, err := makeVmWithBaseStateAndEpoch(pstate, i)
+			if err != nil {
+				return cid.Undef, cid.Undef, xerrors.Errorf("making cron vm: %w", err)
 			}
 
-			pstate, err = vmi.Flush(ctx)
+			// run cron for null rounds if any
+			if err = runCron(vmCron, i); err != nil {
+				return cid.Undef, cid.Undef, xerrors.Errorf("running cron: %w", err)
+			}
+
+			pstate, err = vmCron.Flush(ctx)
 			if err != nil {
-				return cid.Undef, cid.Undef, xerrors.Errorf("flushing vm: %w", err)
+				return cid.Undef, cid.Undef, xerrors.Errorf("flushing cron vm: %w", err)
 			}
 		}
 
 		// handle state forks
 		// XXX: The state tree
-		newState, err := sm.HandleStateForks(ctx, pstate, i, em, ts)
+		pstate, err = sm.HandleStateForks(ctx, pstate, i, em, ts)
 		if err != nil {
 			return cid.Undef, cid.Undef, xerrors.Errorf("error handling state forks: %w", err)
 		}
-
-		if pstate != newState {
-			vmi, err = makeVmWithBaseState(newState)
-			if err != nil {
-				return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err)
-			}
-		}
-
-		if err = vmi.SetBlockHeight(ctx, i+1); err != nil {
-			return cid.Undef, cid.Undef, xerrors.Errorf("error advancing vm an epoch: %w", err)
-		}
-
-		pstate = newState
 	}
 
 	partDone()
 	partDone = metrics.Timer(ctx, metrics.VMApplyMessages)
 
+	vmi, err := makeVmWithBaseStateAndEpoch(pstate, epoch)
+	if err != nil {
+		return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err)
+	}
+
 	var receipts []cbg.CBORMarshaler
 	processedMsgs := make(map[cid.Cid]struct{})
 	for _, b := range bms {
@@ -246,7 +256,7 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, sm *stmgr.StateManager
 	partDone()
 	partDone = metrics.Timer(ctx, metrics.VMApplyCron)
 
-	if err := runCron(epoch); err != nil {
+	if err := runCron(vmi, epoch); err != nil {
 		return cid.Cid{}, cid.Cid{}, err
 	}
 
@@ -294,7 +304,7 @@ func (t *TipSetExecutor) ExecuteTipSet(ctx context.Context, sm *stmgr.StateManag
 	var parentEpoch abi.ChainEpoch
 	pstate := blks[0].ParentStateRoot
 	if blks[0].Height > 0 {
-		parent, err := sm.ChainStore().GetBlock(blks[0].Parents[0])
+		parent, err := sm.ChainStore().GetBlock(ctx, blks[0].Parents[0])
 		if err != nil {
 			return cid.Undef, cid.Undef, xerrors.Errorf("getting parent block: %w", err)
 		}
@@ -302,9 +312,9 @@ func (t *TipSetExecutor) ExecuteTipSet(ctx context.Context, sm *stmgr.StateManag
 		parentEpoch = parent.Height
 	}
 
-	r := rand.NewStateRand(sm.ChainStore(), ts.Cids(), sm.Beacon())
+	r := rand.NewStateRand(sm.ChainStore(), ts.Cids(), sm.Beacon(), sm.GetNetworkVersion)
 
-	blkmsgs, err := sm.ChainStore().BlockMsgsForTipset(ts)
+	blkmsgs, err := sm.ChainStore().BlockMsgsForTipset(ctx, ts)
 	if err != nil {
 		return cid.Undef, cid.Undef, xerrors.Errorf("getting block messages for tipset: %w", err)
 	}

chain/consensus/filcns/filecoin.go

@@ -90,19 +90,19 @@ func (filec *FilecoinEC) ValidateBlock(ctx context.Context, b *types.FullBlock)
 
 	h := b.Header
 
-	baseTs, err := filec.store.LoadTipSet(types.NewTipSetKey(h.Parents...))
+	baseTs, err := filec.store.LoadTipSet(ctx, types.NewTipSetKey(h.Parents...))
 	if err != nil {
 		return xerrors.Errorf("load parent tipset failed (%s): %w", h.Parents, err)
 	}
 
-	winPoStNv := filec.sm.GetNtwkVersion(ctx, baseTs.Height())
+	winPoStNv := filec.sm.GetNetworkVersion(ctx, baseTs.Height())
 
 	lbts, lbst, err := stmgr.GetLookbackTipSetForRound(ctx, filec.sm, baseTs, h.Height)
 	if err != nil {
 		return xerrors.Errorf("failed to get lookback tipset for block: %w", err)
 	}
 
-	prevBeacon, err := filec.store.GetLatestBeaconEntry(baseTs)
+	prevBeacon, err := filec.store.GetLatestBeaconEntry(ctx, baseTs)
 	if err != nil {
 		return xerrors.Errorf("failed to get latest beacon entry: %w", err)
 	}
@@ -171,7 +171,7 @@ func (filec *FilecoinEC) ValidateBlock(ctx context.Context, b *types.FullBlock)
 		}
 
 		if stateroot != h.ParentStateRoot {
-			msgs, err := filec.store.MessagesForTipset(baseTs)
+			msgs, err := filec.store.MessagesForTipset(ctx, baseTs)
 			if err != nil {
 				log.Error("failed to load messages for tipset during tipset state mismatch error: ", err)
 			} else {
@@ -182,7 +182,7 @@ func (filec *FilecoinEC) ValidateBlock(ctx context.Context, b *types.FullBlock)
 				}
 			}
 
-			return xerrors.Errorf("parent state root did not match computed state (%s != %s)", stateroot, h.ParentStateRoot)
+			return xerrors.Errorf("parent state root did not match computed state (%s != %s)", h.ParentStateRoot, stateroot)
 		}
 
 		if precp != h.ParentMessageReceipts {
@@ -458,7 +458,7 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
 
 	stateroot, _, err := filec.sm.TipSetState(ctx, baseTs)
 	if err != nil {
-		return err
+		return xerrors.Errorf("failed to compute tipsettate for %s: %w", baseTs.Key(), err)
 	}
 
 	st, err := state.LoadStateTree(filec.store.ActorStore(ctx), stateroot)
@@ -466,8 +466,8 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
 		return xerrors.Errorf("failed to load base state tree: %w", err)
 	}
 
-	nv := filec.sm.GetNtwkVersion(ctx, b.Header.Height)
-	pl := vm.PricelistByEpoch(baseTs.Height())
+	nv := filec.sm.GetNetworkVersion(ctx, b.Header.Height)
+	pl := vm.PricelistByEpoch(b.Header.Height)
 	var sumGasLimit int64
 	checkMsg := func(msg types.ChainMsg) error {
 		m := msg.VMMessage()
@@ -475,7 +475,7 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
 		// Phase 1: syntactic validation, as defined in the spec
 		minGas := pl.OnChainMessage(msg.ChainLength())
 		if err := m.ValidForBlockInclusion(minGas.Total(), nv); err != nil {
-			return err
+			return xerrors.Errorf("msg %s invalid for block inclusion: %w", m.Cid(), err)
 		}
 
 		// ValidForBlockInclusion checks if any single message does not exceed BlockGasLimit
@@ -488,10 +488,10 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
 		// Phase 2: (Partial) semantic validation:
 		// the sender exists and is an account actor, and the nonces make sense
 		var sender address.Address
-		if filec.sm.GetNtwkVersion(ctx, b.Header.Height) >= network.Version13 {
+		if filec.sm.GetNetworkVersion(ctx, b.Header.Height) >= network.Version13 {
 			sender, err = st.LookupID(m.From)
 			if err != nil {
-				return err
+				return xerrors.Errorf("failed to lookup sender %s: %w", m.From, err)
 			}
 		} else {
 			sender = m.From
@@ -528,7 +528,7 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
 			return xerrors.Errorf("block had invalid bls message at index %d: %w", i, err)
 		}
 
-		c, err := store.PutMessage(tmpbs, m)
+		c, err := store.PutMessage(ctx, tmpbs, m)
 		if err != nil {
 			return xerrors.Errorf("failed to store message %s: %w", m.Cid(), err)
 		}
@@ -541,7 +541,7 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
 
 	smArr := blockadt.MakeEmptyArray(tmpstore)
 	for i, m := range b.SecpkMessages {
-		if filec.sm.GetNtwkVersion(ctx, b.Header.Height) >= network.Version14 {
+		if filec.sm.GetNetworkVersion(ctx, b.Header.Height) >= network.Version14 {
 			if m.Signature.Type != crypto.SigTypeSecp256k1 {
 				return xerrors.Errorf("block had invalid secpk message at index %d: %w", i, err)
 			}
@@ -562,7 +562,7 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
 			return xerrors.Errorf("secpk message %s has invalid signature: %w", m.Cid(), err)
 		}
 
-		c, err := store.PutMessage(tmpbs, m)
+		c, err := store.PutMessage(ctx, tmpbs, m)
 		if err != nil {
 			return xerrors.Errorf("failed to store message %s: %w", m.Cid(), err)
 		}
@@ -574,12 +574,13 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
 
 	bmroot, err := bmArr.Root()
 	if err != nil {
-		return err
+		return xerrors.Errorf("failed to root bls msgs: %w", err)
+
 	}
 
 	smroot, err := smArr.Root()
 	if err != nil {
-		return err
+		return xerrors.Errorf("failed to root secp msgs: %w", err)
 	}
 
 	mrcid, err := tmpstore.Put(ctx, &types.MsgMeta{
@@ -587,7 +588,7 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
 		SecpkMessages: smroot,
 	})
 	if err != nil {
-		return err
+		return xerrors.Errorf("failed to put msg meta: %w", err)
 	}
 
 	if b.Header.Messages != mrcid {
@@ -595,7 +596,12 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
 	}
 
 	// Finally, flush.
-	return vm.Copy(ctx, tmpbs, filec.store.ChainBlockstore(), mrcid)
+	err = vm.Copy(ctx, tmpbs, filec.store.ChainBlockstore(), mrcid)
+	if err != nil {
+		return xerrors.Errorf("failed to flush:%w", err)
+	}
+
+	return nil
 }
 
 func (filec *FilecoinEC) IsEpochBeyondCurrMax(epoch abi.ChainEpoch) bool {

chain/consensus/filcns/mine.go

@@ -15,7 +15,7 @@ import (
 )
 
 func (filec *FilecoinEC) CreateBlock(ctx context.Context, w api.Wallet, bt *api.BlockTemplate) (*types.FullBlock, error) {
-	pts, err := filec.sm.ChainStore().LoadTipSet(bt.Parents)
+	pts, err := filec.sm.ChainStore().LoadTipSet(ctx, bt.Parents)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to load parent tipset: %w", err)
 	}
@@ -59,14 +59,14 @@ func (filec *FilecoinEC) CreateBlock(ctx context.Context, w api.Wallet, bt *api.
 			blsSigs = append(blsSigs, msg.Signature)
 			blsMessages = append(blsMessages, &msg.Message)
 
-			c, err := filec.sm.ChainStore().PutMessage(&msg.Message)
+			c, err := filec.sm.ChainStore().PutMessage(ctx, &msg.Message)
 			if err != nil {
 				return nil, err
 			}
 
 			blsMsgCids = append(blsMsgCids, c)
 		} else if msg.Signature.Type == crypto.SigTypeSecp256k1 {
-			c, err := filec.sm.ChainStore().PutMessage(msg)
+			c, err := filec.sm.ChainStore().PutMessage(ctx, msg)
 			if err != nil {
 				return nil, err
 			}

chain/consensus/filcns/upgrades.go

@@ -639,7 +639,7 @@ func splitGenesisMultisig0(ctx context.Context, em stmgr.ExecMonitor, addr addre
 
 // TODO: After the Liftoff epoch, refactor this to use resetMultisigVesting
 func resetGenesisMsigs0(ctx context.Context, sm *stmgr.StateManager, store adt0.Store, tree *state.StateTree, startEpoch abi.ChainEpoch) error {
-	gb, err := sm.ChainStore().GetGenesis()
+	gb, err := sm.ChainStore().GetGenesis(ctx)
 	if err != nil {
 		return xerrors.Errorf("getting genesis block: %w", err)
 	}

chain/events/events_test.go

@@ -87,7 +87,7 @@ func (fcs *fakeCS) ChainGetPath(ctx context.Context, from, to types.TipSetKey) (
 	}
 
 	// copied from the chainstore
-	revert, apply, err := store.ReorgOps(func(tsk types.TipSetKey) (*types.TipSet, error) {
+	revert, apply, err := store.ReorgOps(ctx, func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) {
 		return fcs.ChainGetTipSet(ctx, tsk)
 	}, fromTs, toTs)
 	if err != nil {

chain/events/state/mock/api.go

@@ -27,11 +27,11 @@ func NewMockAPI(bs blockstore.Blockstore) *MockAPI {
 }
 
 func (m *MockAPI) ChainHasObj(ctx context.Context, c cid.Cid) (bool, error) {
-	return m.bs.Has(c)
+	return m.bs.Has(ctx, c)
 }
 
 func (m *MockAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) {
-	blk, err := m.bs.Get(c)
+	blk, err := m.bs.Get(ctx, c)
 	if err != nil {
 		return nil, xerrors.Errorf("blockstore get: %w", err)
 	}

chain/exchange/server.go

@@ -136,7 +136,7 @@ func (s *server) serviceRequest(ctx context.Context, req *validatedRequest) (*Re
 	_, span := trace.StartSpan(ctx, "chainxchg.ServiceRequest")
 	defer span.End()
 
-	chain, err := collectChainSegment(s.cs, req)
+	chain, err := collectChainSegment(ctx, s.cs, req)
 	if err != nil {
 		log.Warn("block sync request: collectChainSegment failed: ", err)
 		return &Response{
@@ -156,13 +156,13 @@ func (s *server) serviceRequest(ctx context.Context, req *validatedRequest) (*Re
 	}, nil
 }
 
-func collectChainSegment(cs *store.ChainStore, req *validatedRequest) ([]*BSTipSet, error) {
+func collectChainSegment(ctx context.Context, cs *store.ChainStore, req *validatedRequest) ([]*BSTipSet, error) {
 	var bstips []*BSTipSet
 
 	cur := req.head
 	for {
 		var bst BSTipSet
-		ts, err := cs.LoadTipSet(cur)
+		ts, err := cs.LoadTipSet(ctx, cur)
 		if err != nil {
 			return nil, xerrors.Errorf("failed loading tipset %s: %w", cur, err)
 		}
@@ -172,7 +172,7 @@ func collectChainSegment(cs *store.ChainStore, req *validatedRequest) ([]*BSTipS
 		}
 
 		if req.options.IncludeMessages {
-			bmsgs, bmincl, smsgs, smincl, err := gatherMessages(cs, ts)
+			bmsgs, bmincl, smsgs, smincl, err := gatherMessages(ctx, cs, ts)
 			if err != nil {
 				return nil, xerrors.Errorf("gather messages failed: %w", err)
 			}
@@ -197,14 +197,14 @@ func collectChainSegment(cs *store.ChainStore, req *validatedRequest) ([]*BSTipS
 	}
 }
 
-func gatherMessages(cs *store.ChainStore, ts *types.TipSet) ([]*types.Message, [][]uint64, []*types.SignedMessage, [][]uint64, error) {
+func gatherMessages(ctx context.Context, cs *store.ChainStore, ts *types.TipSet) ([]*types.Message, [][]uint64, []*types.SignedMessage, [][]uint64, error) {
 	blsmsgmap := make(map[cid.Cid]uint64)
 	secpkmsgmap := make(map[cid.Cid]uint64)
 	var secpkincl, blsincl [][]uint64
 
 	var blscids, secpkcids []cid.Cid
 	for _, block := range ts.Blocks() {
-		bc, sc, err := cs.ReadMsgMetaCids(block.Messages)
+		bc, sc, err := cs.ReadMsgMetaCids(ctx, block.Messages)
 		if err != nil {
 			return nil, nil, nil, nil, err
 		}
@@ -237,12 +237,12 @@ func gatherMessages(cs *store.ChainStore, ts *types.TipSet) ([]*types.Message, [
 		secpkincl = append(secpkincl, smi)
 	}
 
-	blsmsgs, err := cs.LoadMessagesFromCids(blscids)
+	blsmsgs, err := cs.LoadMessagesFromCids(ctx, blscids)
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}
 
-	secpkmsgs, err := cs.LoadSignedMessagesFromCids(secpkcids)
+	secpkmsgs, err := cs.LoadSignedMessagesFromCids(ctx, secpkcids)
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}

chain/gen/gen.go

@@ -241,7 +241,7 @@ func NewGeneratorWithSectorsAndUpgradeSchedule(numSectors int, us stmgr.UpgradeS
 	genfb := &types.FullBlock{Header: genb.Genesis}
 	gents := store.NewFullTipSet([]*types.FullBlock{genfb})
 
-	if err := cs.SetGenesis(genb.Genesis); err != nil {
+	if err := cs.SetGenesis(context.TODO(), genb.Genesis); err != nil {
 		return nil, xerrors.Errorf("set genesis failed: %w", err)
 	}
 
@@ -471,7 +471,7 @@ func (cg *ChainGen) NextTipSetFromMinersWithMessagesAndNulls(base *types.TipSet,
 					return nil, xerrors.Errorf("making a block for next tipset failed: %w", err)
 				}
 
-				if err := cg.cs.PersistBlockHeaders(fblk.Header); err != nil {
+				if err := cg.cs.PersistBlockHeaders(context.TODO(), fblk.Header); err != nil {
 					return nil, xerrors.Errorf("chainstore AddBlock: %w", err)
 				}
 

chain/gen/genesis/genesis.go

@@ -491,14 +491,13 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, sys vm.Sysca
 		Actors:         filcns.NewActorRegistry(),
 		Syscalls:       mkFakedSigSyscalls(sys),
 		CircSupplyCalc: csc,
-		NtwkVersion: func(_ context.Context, _ abi.ChainEpoch) network.Version {
-			return nv
-		},
-		BaseFee: types.NewInt(0),
+		FilVested:      big.Zero(),
+		NetworkVersion: nv,
+		BaseFee:        big.Zero(),
 	}
-	vm, err := vm.NewVM(ctx, &vmopt)
+	vm, err := vm.NewLegacyVM(ctx, &vmopt)
 	if err != nil {
-		return cid.Undef, xerrors.Errorf("failed to create NewVM: %w", err)
+		return cid.Undef, xerrors.Errorf("failed to create NewLegacyVM: %w", err)
 	}
 
 	for mi, m := range template.Miners {
@@ -595,7 +594,7 @@ func MakeGenesisBlock(ctx context.Context, j journal.Journal, bs bstore.Blocksto
 	if err != nil {
 		return nil, xerrors.Errorf("serializing msgmeta failed: %w", err)
 	}
-	if err := bs.Put(mmb); err != nil {
+	if err := bs.Put(ctx, mmb); err != nil {
 		return nil, xerrors.Errorf("putting msgmeta block to blockstore: %w", err)
 	}
 
@@ -625,7 +624,7 @@ func MakeGenesisBlock(ctx context.Context, j journal.Journal, bs bstore.Blocksto
 		return nil, xerrors.Errorf("filecoinGenesisCid != gblk.Cid")
 	}
 
-	if err := bs.Put(gblk); err != nil {
+	if err := bs.Put(ctx, gblk); err != nil {
 		return nil, xerrors.Errorf("failed writing filecoin genesis block to blockstore: %w", err)
 	}
 
@@ -656,7 +655,7 @@ func MakeGenesisBlock(ctx context.Context, j journal.Journal, bs bstore.Blocksto
 		return nil, xerrors.Errorf("serializing block header failed: %w", err)
 	}
 
-	if err := bs.Put(sb); err != nil {
+	if err := bs.Put(ctx, sb); err != nil {
 		return nil, xerrors.Errorf("putting header to blockstore: %w", err)
 	}
 

chain/gen/genesis/miners.go

@@ -94,15 +94,14 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.Syscal
 		Actors:         filcns.NewActorRegistry(),
 		Syscalls:       mkFakedSigSyscalls(sys),
 		CircSupplyCalc: csc,
-		NtwkVersion: func(_ context.Context, _ abi.ChainEpoch) network.Version {
-			return nv
-		},
-		BaseFee: types.NewInt(0),
+		NetworkVersion: nv,
+		BaseFee:        big.Zero(),
+		FilVested:      big.Zero(),
 	}
 
-	vm, err := vm.NewVM(ctx, vmopt)
+	vm, err := vm.NewLegacyVM(ctx, vmopt)
 	if err != nil {
-		return cid.Undef, xerrors.Errorf("failed to create NewVM: %w", err)
+		return cid.Undef, xerrors.Errorf("failed to create NewLegacyVM: %w", err)
 	}
 
 	if len(miners) == 0 {
@@ -510,37 +509,19 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.Syscal
 // TODO: copied from actors test harness, deduplicate or remove from here
 type fakeRand struct{}
 
-func (fr *fakeRand) GetChainRandomnessV2(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
+func (fr *fakeRand) GetChainRandomness(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
 	out := make([]byte, 32)
 	_, _ = rand.New(rand.NewSource(int64(randEpoch * 1000))).Read(out) //nolint
 	return out, nil
 }
 
-func (fr *fakeRand) GetChainRandomnessV1(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
-	out := make([]byte, 32)
-	_, _ = rand.New(rand.NewSource(int64(randEpoch * 1000))).Read(out) //nolint
-	return out, nil
-}
-
-func (fr *fakeRand) GetBeaconRandomnessV3(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
+func (fr *fakeRand) GetBeaconRandomness(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
 	out := make([]byte, 32)
 	_, _ = rand.New(rand.NewSource(int64(randEpoch))).Read(out) //nolint
 	return out, nil
 }
 
-func (fr *fakeRand) GetBeaconRandomnessV2(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
-	out := make([]byte, 32)
-	_, _ = rand.New(rand.NewSource(int64(randEpoch))).Read(out) //nolint
-	return out, nil
-}
-
-func (fr *fakeRand) GetBeaconRandomnessV1(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
-	out := make([]byte, 32)
-	_, _ = rand.New(rand.NewSource(int64(randEpoch))).Read(out) //nolint
-	return out, nil
-}
-
-func currentTotalPower(ctx context.Context, vm *vm.VM, maddr address.Address) (*power0.CurrentTotalPowerReturn, error) {
+func currentTotalPower(ctx context.Context, vm *vm.LegacyVM, maddr address.Address) (*power0.CurrentTotalPowerReturn, error) {
 	pwret, err := doExecValue(ctx, vm, power.Address, maddr, big.Zero(), builtin0.MethodsPower.CurrentTotalPower, nil)
 	if err != nil {
 		return nil, err
@@ -553,7 +534,7 @@ func currentTotalPower(ctx context.Context, vm *vm.VM, maddr address.Address) (*
 	return &pwr, nil
 }
 
-func dealWeight(ctx context.Context, vm *vm.VM, maddr address.Address, dealIDs []abi.DealID, sectorStart, sectorExpiry abi.ChainEpoch, av actors.Version) (abi.DealWeight, abi.DealWeight, error) {
+func dealWeight(ctx context.Context, vm *vm.LegacyVM, maddr address.Address, dealIDs []abi.DealID, sectorStart, sectorExpiry abi.ChainEpoch, av actors.Version) (abi.DealWeight, abi.DealWeight, error) {
 	// TODO: This hack should move to market actor wrapper
 	if av <= actors.Version2 {
 		params := &market0.VerifyDealsForActivationParams{
@@ -613,7 +594,7 @@ func dealWeight(ctx context.Context, vm *vm.VM, maddr address.Address, dealIDs [
 	return dealWeights.Sectors[0].DealWeight, dealWeights.Sectors[0].VerifiedDealWeight, nil
 }
 
-func currentEpochBlockReward(ctx context.Context, vm *vm.VM, maddr address.Address, av actors.Version) (abi.StoragePower, builtin.FilterEstimate, error) {
+func currentEpochBlockReward(ctx context.Context, vm *vm.LegacyVM, maddr address.Address, av actors.Version) (abi.StoragePower, builtin.FilterEstimate, error) {
 	rwret, err := doExecValue(ctx, vm, reward.Address, maddr, big.Zero(), reward.Methods.ThisEpochReward, nil)
 	if err != nil {
 		return big.Zero(), builtin.FilterEstimate{}, err
@@ -648,7 +629,7 @@ func currentEpochBlockReward(ctx context.Context, vm *vm.VM, maddr address.Addre
 	return epochReward.ThisEpochBaselinePower, builtin.FilterEstimate(epochReward.ThisEpochRewardSmoothed), nil
 }
 
-func circSupply(ctx context.Context, vmi *vm.VM, maddr address.Address) abi.TokenAmount {
+func circSupply(ctx context.Context, vmi *vm.LegacyVM, maddr address.Address) abi.TokenAmount {
 	unsafeVM := &vm.UnsafeVM{VM: vmi}
 	rt := unsafeVM.MakeRuntime(ctx, &types.Message{
 		GasLimit: 1_000_000_000,

chain/gen/genesis/util.go

@@ -21,7 +21,7 @@ func mustEnc(i cbg.CBORMarshaler) []byte {
 	return enc
 }
 
-func doExecValue(ctx context.Context, vm *vm.VM, to, from address.Address, value types.BigInt, method abi.MethodNum, params []byte) ([]byte, error) {
+func doExecValue(ctx context.Context, vm *vm.LegacyVM, to, from address.Address, value types.BigInt, method abi.MethodNum, params []byte) ([]byte, error) {
 	act, err := vm.StateTree().GetActor(from)
 	if err != nil {
 		return nil, xerrors.Errorf("doExec failed to get from actor (%s): %w", from, err)

chain/gen/slashfilter/slashfilter.go

@@ -1,6 +1,7 @@
 package slashfilter
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/filecoin-project/lotus/build"
@@ -27,7 +28,7 @@ func New(dstore ds.Batching) *SlashFilter {
 	}
 }
 
-func (f *SlashFilter) MinedBlock(bh *types.BlockHeader, parentEpoch abi.ChainEpoch) error {
+func (f *SlashFilter) MinedBlock(ctx context.Context, bh *types.BlockHeader, parentEpoch abi.ChainEpoch) error {
 	if build.IsNearUpgrade(bh.Height, build.UpgradeOrangeHeight) {
 		return nil
 	}
@@ -35,7 +36,7 @@ func (f *SlashFilter) MinedBlock(bh *types.BlockHeader, parentEpoch abi.ChainEpo
 	epochKey := ds.NewKey(fmt.Sprintf("/%s/%d", bh.Miner, bh.Height))
 	{
 		// double-fork mining (2 blocks at one epoch)
-		if err := checkFault(f.byEpoch, epochKey, bh, "double-fork mining faults"); err != nil {
+		if err := checkFault(ctx, f.byEpoch, epochKey, bh, "double-fork mining faults"); err != nil {
 			return err
 		}
 	}
@@ -43,7 +44,7 @@ func (f *SlashFilter) MinedBlock(bh *types.BlockHeader, parentEpoch abi.ChainEpo
 	parentsKey := ds.NewKey(fmt.Sprintf("/%s/%x", bh.Miner, types.NewTipSetKey(bh.Parents...).Bytes()))
 	{
 		// time-offset mining faults (2 blocks with the same parents)
-		if err := checkFault(f.byParents, parentsKey, bh, "time-offset mining faults"); err != nil {
+		if err := checkFault(ctx, f.byParents, parentsKey, bh, "time-offset mining faults"); err != nil {
 			return err
 		}
 	}
@@ -53,14 +54,14 @@ func (f *SlashFilter) MinedBlock(bh *types.BlockHeader, parentEpoch abi.ChainEpo
 
 		// First check if we have mined a block on the parent epoch
 		parentEpochKey := ds.NewKey(fmt.Sprintf("/%s/%d", bh.Miner, parentEpoch))
-		have, err := f.byEpoch.Has(parentEpochKey)
+		have, err := f.byEpoch.Has(ctx, parentEpochKey)
 		if err != nil {
 			return err
 		}
 
 		if have {
 			// If we had, make sure it's in our parent tipset
-			cidb, err := f.byEpoch.Get(parentEpochKey)
+			cidb, err := f.byEpoch.Get(ctx, parentEpochKey)
 			if err != nil {
 				return xerrors.Errorf("getting other block cid: %w", err)
 			}
@@ -83,25 +84,25 @@ func (f *SlashFilter) MinedBlock(bh *types.BlockHeader, parentEpoch abi.ChainEpo
 		}
 	}
 
-	if err := f.byParents.Put(parentsKey, bh.Cid().Bytes()); err != nil {
+	if err := f.byParents.Put(ctx, parentsKey, bh.Cid().Bytes()); err != nil {
 		return xerrors.Errorf("putting byEpoch entry: %w", err)
 	}
 
-	if err := f.byEpoch.Put(epochKey, bh.Cid().Bytes()); err != nil {
+	if err := f.byEpoch.Put(ctx, epochKey, bh.Cid().Bytes()); err != nil {
 		return xerrors.Errorf("putting byEpoch entry: %w", err)
 	}
 
 	return nil
 }
 
-func checkFault(t ds.Datastore, key ds.Key, bh *types.BlockHeader, faultType string) error {
-	fault, err := t.Has(key)
+func checkFault(ctx context.Context, t ds.Datastore, key ds.Key, bh *types.BlockHeader, faultType string) error {
+	fault, err := t.Has(ctx, key)
 	if err != nil {
 		return err
 	}
 
 	if fault {
-		cidb, err := t.Get(key)
+		cidb, err := t.Get(ctx, key)
 		if err != nil {
 			return xerrors.Errorf("getting other block cid: %w", err)
 		}

chain/market/fundmanager.go

@@ -89,7 +89,7 @@ func (fm *FundManager) Start() error {
 	// - in State() only load addresses with in-progress messages
 	// - load the others just-in-time from getFundedAddress
 	// - delete(fm.fundedAddrs, addr) when the queue has been processed
-	return fm.str.forEach(func(state *FundedAddressState) {
+	return fm.str.forEach(fm.ctx, func(state *FundedAddressState) {
 		fa := newFundedAddress(fm, state.Addr)
 		fa.state = state
 		fm.fundedAddrs[fa.state.Addr] = fa
@@ -322,7 +322,7 @@ func (a *fundedAddress) clearWaitState() {
 // Save state to datastore
 func (a *fundedAddress) saveState() {
 	// Not much we can do if saving to the datastore fails, just log
-	err := a.str.save(a.state)
+	err := a.str.save(a.ctx, a.state)
 	if err != nil {
 		log.Errorf("saving state to store for addr %s: %v", a.state.Addr, err)
 	}

chain/market/store.go

@@ -2,6 +2,7 @@ package market
 
 import (
 	"bytes"
+	"context"
 
 	cborrpc "github.com/filecoin-project/go-cbor-util"
 	"github.com/ipfs/go-datastore"
@@ -27,7 +28,7 @@ func newStore(ds dtypes.MetadataDS) *Store {
 }
 
 // save the state to the datastore
-func (ps *Store) save(state *FundedAddressState) error {
+func (ps *Store) save(ctx context.Context, state *FundedAddressState) error {
 	k := dskeyForAddr(state.Addr)
 
 	b, err := cborrpc.Dump(state)
@@ -35,14 +36,14 @@ func (ps *Store) save(state *FundedAddressState) error {
 		return err
 	}
 
-	return ps.ds.Put(k, b)
+	return ps.ds.Put(ctx, k, b)
 }
 
 // get the state for the given address
-func (ps *Store) get(addr address.Address) (*FundedAddressState, error) {
+func (ps *Store) get(ctx context.Context, addr address.Address) (*FundedAddressState, error) {
 	k := dskeyForAddr(addr)
 
-	data, err := ps.ds.Get(k)
+	data, err := ps.ds.Get(ctx, k)
 	if err != nil {
 		return nil, err
 	}
@@ -56,8 +57,8 @@ func (ps *Store) get(addr address.Address) (*FundedAddressState, error) {
 }
 
 // forEach calls iter with each address in the datastore
-func (ps *Store) forEach(iter func(*FundedAddressState)) error {
-	res, err := ps.ds.Query(dsq.Query{Prefix: dsKeyAddr})
+func (ps *Store) forEach(ctx context.Context, iter func(*FundedAddressState)) error {
+	res, err := ps.ds.Query(ctx, dsq.Query{Prefix: dsKeyAddr})
 	if err != nil {
 		return err
 	}

chain/messagepool/check.go

@@ -106,7 +106,7 @@ func (mp *MessagePool) checkMessages(ctx context.Context, msgs []*types.Message,
 	curTs := mp.curTs
 	mp.curTsLk.Unlock()
 
-	epoch := curTs.Height()
+	epoch := curTs.Height() + 1
 
 	var baseFee big.Int
 	if len(curTs.Blocks()) > 0 {

chain/messagepool/check_test.go

@@ -0,0 +1,224 @@
+//stm: #unit
+package messagepool
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/ipfs/go-datastore"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/consensus/filcns"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/types/mock"
+	"github.com/filecoin-project/lotus/chain/wallet"
+	_ "github.com/filecoin-project/lotus/lib/sigs/bls"
+	_ "github.com/filecoin-project/lotus/lib/sigs/secp"
+)
+
+func init() {
+	_ = logging.SetLogLevel("*", "INFO")
+}
+
+func getCheckMessageStatus(statusCode api.CheckStatusCode, msgStatuses []api.MessageCheckStatus) (*api.MessageCheckStatus, error) {
+	for i := 0; i < len(msgStatuses); i++ {
+		iMsgStatuses := msgStatuses[i]
+		if iMsgStatuses.CheckStatus.Code == statusCode {
+			return &iMsgStatuses, nil
+		}
+	}
+	return nil, fmt.Errorf("Could not find CheckStatusCode %s", statusCode)
+}
+
+func TestCheckMessages(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_CHECK_MESSAGES_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sender, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(sender, 1000e15)
+	target := mock.Address(1001)
+
+	var protos []*api.MessagePrototype
+	for i := 0; i < 5; i++ {
+		msg := &types.Message{
+			To:         target,
+			From:       sender,
+			Value:      types.NewInt(1),
+			Nonce:      uint64(i),
+			GasLimit:   50000000,
+			GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+			GasPremium: types.NewInt(1),
+			Params:     make([]byte, 2<<10),
+		}
+		proto := &api.MessagePrototype{
+			Message:    *msg,
+			ValidNonce: true,
+		}
+		protos = append(protos, proto)
+	}
+
+	messageStatuses, err := mp.CheckMessages(context.TODO(), protos)
+	assert.NoError(t, err)
+	for i := 0; i < len(messageStatuses); i++ {
+		iMsgStatuses := messageStatuses[i]
+		for j := 0; j < len(iMsgStatuses); j++ {
+			jStatus := iMsgStatuses[i]
+			assert.True(t, jStatus.OK)
+		}
+	}
+}
+
+func TestCheckPendingMessages(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_CHECK_PENDING_MESSAGES_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sender, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(sender, 1000e15)
+	target := mock.Address(1001)
+
+	// add a valid message to the pool
+	msg := &types.Message{
+		To:         target,
+		From:       sender,
+		Value:      types.NewInt(1),
+		Nonce:      0,
+		GasLimit:   50000000,
+		GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+		GasPremium: types.NewInt(1),
+		Params:     make([]byte, 2<<10),
+	}
+
+	sig, err := w.WalletSign(context.TODO(), sender, msg.Cid().Bytes(), api.MsgMeta{})
+	if err != nil {
+		panic(err)
+	}
+	sm := &types.SignedMessage{
+		Message:   *msg,
+		Signature: *sig,
+	}
+	mustAdd(t, mp, sm)
+
+	messageStatuses, err := mp.CheckPendingMessages(context.TODO(), sender)
+	assert.NoError(t, err)
+	for i := 0; i < len(messageStatuses); i++ {
+		iMsgStatuses := messageStatuses[i]
+		for j := 0; j < len(iMsgStatuses); j++ {
+			jStatus := iMsgStatuses[i]
+			assert.True(t, jStatus.OK)
+		}
+	}
+}
+
+func TestCheckReplaceMessages(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_CHECK_REPLACE_MESSAGES_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sender, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(sender, 1000e15)
+	target := mock.Address(1001)
+
+	// add a valid message to the pool
+	msg := &types.Message{
+		To:         target,
+		From:       sender,
+		Value:      types.NewInt(1),
+		Nonce:      0,
+		GasLimit:   50000000,
+		GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+		GasPremium: types.NewInt(1),
+		Params:     make([]byte, 2<<10),
+	}
+
+	sig, err := w.WalletSign(context.TODO(), sender, msg.Cid().Bytes(), api.MsgMeta{})
+	if err != nil {
+		panic(err)
+	}
+	sm := &types.SignedMessage{
+		Message:   *msg,
+		Signature: *sig,
+	}
+	mustAdd(t, mp, sm)
+
+	// create a new message with the same data, except that it is too big
+	var msgs []*types.Message
+	invalidmsg := &types.Message{
+		To:         target,
+		From:       sender,
+		Value:      types.NewInt(1),
+		Nonce:      0,
+		GasLimit:   50000000,
+		GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+		GasPremium: types.NewInt(1),
+		Params:     make([]byte, 128<<10),
+	}
+	msgs = append(msgs, invalidmsg)
+
+	{
+		messageStatuses, err := mp.CheckReplaceMessages(context.TODO(), msgs)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for i := 0; i < len(messageStatuses); i++ {
+			iMsgStatuses := messageStatuses[i]
+
+			status, err := getCheckMessageStatus(api.CheckStatusMessageSize, iMsgStatuses)
+			if err != nil {
+				t.Fatal(err)
+			}
+			// the replacement message should cause a status error
+			assert.False(t, status.OK)
+		}
+	}
+
+}

chain/messagepool/config.go

@@ -1,6 +1,7 @@
 package messagepool
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"time"
@@ -20,8 +21,8 @@ var (
 	ConfigKey = datastore.NewKey("/mpool/config")
 )
 
-func loadConfig(ds dtypes.MetadataDS) (*types.MpoolConfig, error) {
-	haveCfg, err := ds.Has(ConfigKey)
+func loadConfig(ctx context.Context, ds dtypes.MetadataDS) (*types.MpoolConfig, error) {
+	haveCfg, err := ds.Has(ctx, ConfigKey)
 	if err != nil {
 		return nil, err
 	}
@@ -30,7 +31,7 @@ func loadConfig(ds dtypes.MetadataDS) (*types.MpoolConfig, error) {
 		return DefaultConfig(), nil
 	}
 
-	cfgBytes, err := ds.Get(ConfigKey)
+	cfgBytes, err := ds.Get(ctx, ConfigKey)
 	if err != nil {
 		return nil, err
 	}
@@ -39,12 +40,12 @@ func loadConfig(ds dtypes.MetadataDS) (*types.MpoolConfig, error) {
 	return cfg, err
 }
 
-func saveConfig(cfg *types.MpoolConfig, ds dtypes.MetadataDS) error {
+func saveConfig(ctx context.Context, cfg *types.MpoolConfig, ds dtypes.MetadataDS) error {
 	cfgBytes, err := json.Marshal(cfg)
 	if err != nil {
 		return err
 	}
-	return ds.Put(ConfigKey, cfgBytes)
+	return ds.Put(ctx, ConfigKey, cfgBytes)
 }
 
 func (mp *MessagePool) GetConfig() *types.MpoolConfig {
@@ -68,7 +69,7 @@ func validateConfg(cfg *types.MpoolConfig) error {
 	return nil
 }
 
-func (mp *MessagePool) SetConfig(cfg *types.MpoolConfig) error {
+func (mp *MessagePool) SetConfig(ctx context.Context, cfg *types.MpoolConfig) error {
 	if err := validateConfg(cfg); err != nil {
 		return err
 	}
@@ -76,7 +77,7 @@ func (mp *MessagePool) SetConfig(cfg *types.MpoolConfig) error {
 
 	mp.cfgLk.Lock()
 	mp.cfg = cfg
-	err := saveConfig(cfg, mp.ds)
+	err := saveConfig(ctx, cfg, mp.ds)
 	if err != nil {
 		log.Warnf("error persisting mpool config: %s", err)
 	}

chain/messagepool/messagepool.go

@@ -173,10 +173,17 @@ type MessagePool struct {
 
 	sigValCache *lru.TwoQueueCache
 
+	nonceCache *lru.Cache
+
 	evtTypes [3]journal.EventType
 	journal  journal.Journal
 }
 
+type nonceCacheKey struct {
+	tsk  types.TipSetKey
+	addr address.Address
+}
+
 type msgSet struct {
 	msgs          map[uint64]*types.SignedMessage
 	nextNonce     uint64
@@ -196,10 +203,10 @@ func ComputeMinRBF(curPrem abi.TokenAmount) abi.TokenAmount {
 	return types.BigAdd(minPrice, types.NewInt(1))
 }
 
-func CapGasFee(mff dtypes.DefaultMaxFeeFunc, msg *types.Message, sendSepc *api.MessageSendSpec) {
+func CapGasFee(mff dtypes.DefaultMaxFeeFunc, msg *types.Message, sendSpec *api.MessageSendSpec) {
 	var maxFee abi.TokenAmount
-	if sendSepc != nil {
-		maxFee = sendSepc.MaxFee
+	if sendSpec != nil {
+		maxFee = sendSpec.MaxFee
 	}
 	if maxFee.Int == nil || maxFee.Equals(big.Zero()) {
 		mf, err := mff()
@@ -358,11 +365,12 @@ func (ms *msgSet) toSlice() []*types.SignedMessage {
 	return set
 }
 
-func New(api Provider, ds dtypes.MetadataDS, us stmgr.UpgradeSchedule, netName dtypes.NetworkName, j journal.Journal) (*MessagePool, error) {
+func New(ctx context.Context, api Provider, ds dtypes.MetadataDS, us stmgr.UpgradeSchedule, netName dtypes.NetworkName, j journal.Journal) (*MessagePool, error) {
 	cache, _ := lru.New2Q(build.BlsSignatureCacheSize)
 	verifcache, _ := lru.New2Q(build.VerifSigCacheSize)
+	noncecache, _ := lru.New(256)
 
-	cfg, err := loadConfig(ds)
+	cfg, err := loadConfig(ctx, ds)
 	if err != nil {
 		return nil, xerrors.Errorf("error loading mpool config: %w", err)
 	}
@@ -386,6 +394,7 @@ func New(api Provider, ds dtypes.MetadataDS, us stmgr.UpgradeSchedule, netName d
 		pruneCooldown:  make(chan struct{}, 1),
 		blsSigCache:    cache,
 		sigValCache:    verifcache,
+		nonceCache:     noncecache,
 		changes:        lps.New(50),
 		localMsgs:      namespace.Wrap(ds, datastore.NewKey(localMsgsDs)),
 		api:            api,
@@ -601,7 +610,7 @@ func (mp *MessagePool) addLocal(ctx context.Context, m *types.SignedMessage) err
 		return xerrors.Errorf("error serializing message: %w", err)
 	}
 
-	if err := mp.localMsgs.Put(datastore.NewKey(string(m.Cid().Bytes())), msgb); err != nil {
+	if err := mp.localMsgs.Put(ctx, datastore.NewKey(string(m.Cid().Bytes())), msgb); err != nil {
 		return xerrors.Errorf("persisting local message: %w", err)
 	}
 
@@ -619,7 +628,7 @@ func (mp *MessagePool) addLocal(ctx context.Context, m *types.SignedMessage) err
 // For non local messages, if the message cannot be included in the next 20 blocks it returns
 // a (soft) validation error.
 func (mp *MessagePool) verifyMsgBeforeAdd(m *types.SignedMessage, curTs *types.TipSet, local bool) (bool, error) {
-	epoch := curTs.Height()
+	epoch := curTs.Height() + 1
 	minGas := vm.PricelistByEpoch(epoch).OnChainMessage(m.ChainLength())
 
 	if err := m.VMMessage().ValidForBlockInclusion(minGas.Total(), build.NewestNetworkVersion); err != nil {
@@ -909,12 +918,12 @@ func (mp *MessagePool) addLocked(ctx context.Context, m *types.SignedMessage, st
 		mp.blsSigCache.Add(m.Cid(), m.Signature)
 	}
 
-	if _, err := mp.api.PutMessage(m); err != nil {
+	if _, err := mp.api.PutMessage(ctx, m); err != nil {
 		log.Warnf("mpooladd cs.PutMessage failed: %s", err)
 		return err
 	}
 
-	if _, err := mp.api.PutMessage(&m.Message); err != nil {
+	if _, err := mp.api.PutMessage(ctx, &m.Message); err != nil {
 		log.Warnf("mpooladd cs.PutMessage failed: %s", err)
 		return err
 	}
@@ -1016,11 +1025,23 @@ func (mp *MessagePool) getStateNonce(ctx context.Context, addr address.Address,
 	done := metrics.Timer(ctx, metrics.MpoolGetNonceDuration)
 	defer done()
 
+	nk := nonceCacheKey{
+		tsk:  ts.Key(),
+		addr: addr,
+	}
+
+	n, ok := mp.nonceCache.Get(nk)
+	if ok {
+		return n.(uint64), nil
+	}
+
 	act, err := mp.api.GetActorAfter(addr, ts)
 	if err != nil {
 		return 0, err
 	}
 
+	mp.nonceCache.Add(nk, act.Nonce)
+
 	return act.Nonce, nil
 }
 
@@ -1207,7 +1228,7 @@ func (mp *MessagePool) HeadChange(ctx context.Context, revert []*types.TipSet, a
 	var merr error
 
 	for _, ts := range revert {
-		pts, err := mp.api.LoadTipSet(ts.Parents())
+		pts, err := mp.api.LoadTipSet(ctx, ts.Parents())
 		if err != nil {
 			log.Errorf("error loading reverted tipset parent: %s", err)
 			merr = multierror.Append(merr, err)
@@ -1216,7 +1237,7 @@ func (mp *MessagePool) HeadChange(ctx context.Context, revert []*types.TipSet, a
 
 		mp.curTs = pts
 
-		msgs, err := mp.MessagesForBlocks(ts.Blocks())
+		msgs, err := mp.MessagesForBlocks(ctx, ts.Blocks())
 		if err != nil {
 			log.Errorf("error retrieving messages for reverted block: %s", err)
 			merr = multierror.Append(merr, err)
@@ -1232,7 +1253,7 @@ func (mp *MessagePool) HeadChange(ctx context.Context, revert []*types.TipSet, a
 		mp.curTs = ts
 
 		for _, b := range ts.Blocks() {
-			bmsgs, smsgs, err := mp.api.MessagesForBlock(b)
+			bmsgs, smsgs, err := mp.api.MessagesForBlock(ctx, b)
 			if err != nil {
 				xerr := xerrors.Errorf("failed to get messages for apply block %s(height %d) (msgroot = %s): %w", b.Cid(), b.Height, b.Messages, err)
 				log.Errorf("error retrieving messages for block: %s", xerr)
@@ -1338,7 +1359,7 @@ func (mp *MessagePool) HeadChange(ctx context.Context, revert []*types.TipSet, a
 	return merr
 }
 
-func (mp *MessagePool) runHeadChange(from *types.TipSet, to *types.TipSet, rmsgs map[address.Address]map[uint64]*types.SignedMessage) error {
+func (mp *MessagePool) runHeadChange(ctx context.Context, from *types.TipSet, to *types.TipSet, rmsgs map[address.Address]map[uint64]*types.SignedMessage) error {
 	add := func(m *types.SignedMessage) {
 		s, ok := rmsgs[m.Message.From]
 		if !ok {
@@ -1360,7 +1381,7 @@ func (mp *MessagePool) runHeadChange(from *types.TipSet, to *types.TipSet, rmsgs
 
 	}
 
-	revert, apply, err := store.ReorgOps(mp.api.LoadTipSet, from, to)
+	revert, apply, err := store.ReorgOps(ctx, mp.api.LoadTipSet, from, to)
 	if err != nil {
 		return xerrors.Errorf("failed to compute reorg ops for mpool pending messages: %w", err)
 	}
@@ -1368,7 +1389,7 @@ func (mp *MessagePool) runHeadChange(from *types.TipSet, to *types.TipSet, rmsgs
 	var merr error
 
 	for _, ts := range revert {
-		msgs, err := mp.MessagesForBlocks(ts.Blocks())
+		msgs, err := mp.MessagesForBlocks(ctx, ts.Blocks())
 		if err != nil {
 			log.Errorf("error retrieving messages for reverted block: %s", err)
 			merr = multierror.Append(merr, err)
@@ -1382,7 +1403,7 @@ func (mp *MessagePool) runHeadChange(from *types.TipSet, to *types.TipSet, rmsgs
 
 	for _, ts := range apply {
 		for _, b := range ts.Blocks() {
-			bmsgs, smsgs, err := mp.api.MessagesForBlock(b)
+			bmsgs, smsgs, err := mp.api.MessagesForBlock(ctx, b)
 			if err != nil {
 				xerr := xerrors.Errorf("failed to get messages for apply block %s(height %d) (msgroot = %s): %w", b.Cid(), b.Height, b.Messages, err)
 				log.Errorf("error retrieving messages for block: %s", xerr)
@@ -1407,11 +1428,11 @@ type statBucket struct {
 	msgs map[uint64]*types.SignedMessage
 }
 
-func (mp *MessagePool) MessagesForBlocks(blks []*types.BlockHeader) ([]*types.SignedMessage, error) {
+func (mp *MessagePool) MessagesForBlocks(ctx context.Context, blks []*types.BlockHeader) ([]*types.SignedMessage, error) {
 	out := make([]*types.SignedMessage, 0)
 
 	for _, b := range blks {
-		bmsgs, smsgs, err := mp.api.MessagesForBlock(b)
+		bmsgs, smsgs, err := mp.api.MessagesForBlock(ctx, b)
 		if err != nil {
 			return nil, xerrors.Errorf("failed to get messages for apply block %s(height %d) (msgroot = %s): %w", b.Cid(), b.Height, b.Messages, err)
 		}
@@ -1477,7 +1498,7 @@ func (mp *MessagePool) Updates(ctx context.Context) (<-chan api.MpoolUpdate, err
 }
 
 func (mp *MessagePool) loadLocal(ctx context.Context) error {
-	res, err := mp.localMsgs.Query(query.Query{})
+	res, err := mp.localMsgs.Query(ctx, query.Query{})
 	if err != nil {
 		return xerrors.Errorf("query local messages: %w", err)
 	}
@@ -1525,7 +1546,7 @@ func (mp *MessagePool) Clear(ctx context.Context, local bool) {
 
 			if ok {
 				for _, m := range mset.msgs {
-					err := mp.localMsgs.Delete(datastore.NewKey(string(m.Cid().Bytes())))
+					err := mp.localMsgs.Delete(ctx, datastore.NewKey(string(m.Cid().Bytes())))
 					if err != nil {
 						log.Warnf("error deleting local message: %s", err)
 					}

chain/messagepool/messagepool_test.go

@@ -1,3 +1,4 @@
+//stm: #unit
 package messagepool
 
 import (
@@ -8,6 +9,7 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
 	logging "github.com/ipfs/go-log/v2"
@@ -103,7 +105,7 @@ func (tma *testMpoolAPI) SubscribeHeadChanges(cb func(rev, app []*types.TipSet)
 	return tma.tipsets[0]
 }
 
-func (tma *testMpoolAPI) PutMessage(m types.ChainMsg) (cid.Cid, error) {
+func (tma *testMpoolAPI) PutMessage(ctx context.Context, m types.ChainMsg) (cid.Cid, error) {
 	return cid.Undef, nil
 }
 
@@ -164,16 +166,16 @@ func (tma *testMpoolAPI) StateAccountKeyAtFinality(ctx context.Context, addr add
 	return addr, nil
 }
 
-func (tma *testMpoolAPI) MessagesForBlock(h *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) {
+func (tma *testMpoolAPI) MessagesForBlock(ctx context.Context, h *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) {
 	return nil, tma.bmsgs[h.Cid()], nil
 }
 
-func (tma *testMpoolAPI) MessagesForTipset(ts *types.TipSet) ([]types.ChainMsg, error) {
+func (tma *testMpoolAPI) MessagesForTipset(ctx context.Context, ts *types.TipSet) ([]types.ChainMsg, error) {
 	if len(ts.Blocks()) != 1 {
 		panic("cant deal with multiblock tipsets in this test")
 	}
 
-	bm, sm, err := tma.MessagesForBlock(ts.Blocks()[0])
+	bm, sm, err := tma.MessagesForBlock(ctx, ts.Blocks()[0])
 	if err != nil {
 		return nil, err
 	}
@@ -190,7 +192,7 @@ func (tma *testMpoolAPI) MessagesForTipset(ts *types.TipSet) ([]types.ChainMsg,
 	return out, nil
 }
 
-func (tma *testMpoolAPI) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error) {
+func (tma *testMpoolAPI) LoadTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) {
 	for _, ts := range tma.tipsets {
 		if types.CidArrsEqual(tsk.Cids(), ts.Cids()) {
 			return ts, nil
@@ -206,6 +208,7 @@ func (tma *testMpoolAPI) ChainComputeBaseFee(ctx context.Context, ts *types.TipS
 
 func assertNonce(t *testing.T, mp *MessagePool, addr address.Address, val uint64) {
 	t.Helper()
+	//stm: @CHAIN_MEMPOOL_GET_NONCE_001
 	n, err := mp.GetNonce(context.TODO(), addr, types.EmptyTSK)
 	if err != nil {
 		t.Fatal(err)
@@ -224,6 +227,8 @@ func mustAdd(t *testing.T, mp *MessagePool, msg *types.SignedMessage) {
 }
 
 func TestMessagePool(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_GET_NONCE_001
+
 	tma := newTestMpoolAPI()
 
 	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
@@ -233,7 +238,7 @@ func TestMessagePool(t *testing.T) {
 
 	ds := datastore.NewMapDatastore()
 
-	mp, err := New(tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -277,7 +282,7 @@ func TestCheckMessageBig(t *testing.T) {
 
 	ds := datastore.NewMapDatastore()
 
-	mp, err := New(tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
 	assert.NoError(t, err)
 
 	to := mock.Address(1001)
@@ -325,6 +330,7 @@ func TestCheckMessageBig(t *testing.T) {
 			Message:   *msg,
 			Signature: *sig,
 		}
+		//stm: @CHAIN_MEMPOOL_PUSH_001
 		err = mp.Add(context.TODO(), sm)
 		assert.ErrorIs(t, err, ErrMessageTooBig)
 	}
@@ -340,7 +346,7 @@ func TestMessagePoolMessagesInEachBlock(t *testing.T) {
 
 	ds := datastore.NewMapDatastore()
 
-	mp, err := New(tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -366,8 +372,10 @@ func TestMessagePoolMessagesInEachBlock(t *testing.T) {
 	tma.applyBlock(t, a)
 	tsa := mock.TipSet(a)
 
+	//stm: @CHAIN_MEMPOOL_PENDING_001
 	_, _ = mp.Pending(context.TODO())
 
+	//stm: @CHAIN_MEMPOOL_SELECT_001
 	selm, _ := mp.SelectMessages(context.Background(), tsa, 1)
 	if len(selm) == 0 {
 		t.Fatal("should have returned the rest of the messages")
@@ -389,7 +397,7 @@ func TestRevertMessages(t *testing.T) {
 
 	ds := datastore.NewMapDatastore()
 
-	mp, err := New(tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -428,6 +436,7 @@ func TestRevertMessages(t *testing.T) {
 
 	assertNonce(t, mp, sender, 4)
 
+	//stm: @CHAIN_MEMPOOL_PENDING_001
 	p, _ := mp.Pending(context.TODO())
 	fmt.Printf("%+v\n", p)
 	if len(p) != 3 {
@@ -452,7 +461,7 @@ func TestPruningSimple(t *testing.T) {
 
 	ds := datastore.NewMapDatastore()
 
-	mp, err := New(tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -486,6 +495,7 @@ func TestPruningSimple(t *testing.T) {
 
 	mp.Prune()
 
+	//stm: @CHAIN_MEMPOOL_PENDING_001
 	msgs, _ := mp.Pending(context.TODO())
 	if len(msgs) != 5 {
 		t.Fatal("expected only 5 messages in pool, got: ", len(msgs))
@@ -496,7 +506,7 @@ func TestLoadLocal(t *testing.T) {
 	tma := newTestMpoolAPI()
 	ds := datastore.NewMapDatastore()
 
-	mp, err := New(tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -528,6 +538,7 @@ func TestLoadLocal(t *testing.T) {
 	msgs := make(map[cid.Cid]struct{})
 	for i := 0; i < 10; i++ {
 		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		//stm: @CHAIN_MEMPOOL_PUSH_001
 		cid, err := mp.Push(context.TODO(), m)
 		if err != nil {
 			t.Fatal(err)
@@ -539,11 +550,12 @@ func TestLoadLocal(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	mp, err = New(tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	mp, err = New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
+	//stm: @CHAIN_MEMPOOL_PENDING_001
 	pmsgs, _ := mp.Pending(context.TODO())
 	if len(msgs) != len(pmsgs) {
 		t.Fatalf("expected %d messages, but got %d", len(msgs), len(pmsgs))
@@ -568,7 +580,7 @@ func TestClearAll(t *testing.T) {
 	tma := newTestMpoolAPI()
 	ds := datastore.NewMapDatastore()
 
-	mp, err := New(tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -599,6 +611,7 @@ func TestClearAll(t *testing.T) {
 	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}]
 	for i := 0; i < 10; i++ {
 		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		//stm: @CHAIN_MEMPOOL_PUSH_001
 		_, err := mp.Push(context.TODO(), m)
 		if err != nil {
 			t.Fatal(err)
@@ -610,8 +623,10 @@ func TestClearAll(t *testing.T) {
 		mustAdd(t, mp, m)
 	}
 
+	//stm: @CHAIN_MEMPOOL_CLEAR_001
 	mp.Clear(context.Background(), true)
 
+	//stm: @CHAIN_MEMPOOL_PENDING_001
 	pending, _ := mp.Pending(context.TODO())
 	if len(pending) > 0 {
 		t.Fatalf("cleared the mpool, but got %d pending messages", len(pending))
@@ -622,7 +637,7 @@ func TestClearNonLocal(t *testing.T) {
 	tma := newTestMpoolAPI()
 	ds := datastore.NewMapDatastore()
 
-	mp, err := New(tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -654,6 +669,7 @@ func TestClearNonLocal(t *testing.T) {
 	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}]
 	for i := 0; i < 10; i++ {
 		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		//stm: @CHAIN_MEMPOOL_PUSH_001
 		_, err := mp.Push(context.TODO(), m)
 		if err != nil {
 			t.Fatal(err)
@@ -665,8 +681,10 @@ func TestClearNonLocal(t *testing.T) {
 		mustAdd(t, mp, m)
 	}
 
+	//stm: @CHAIN_MEMPOOL_CLEAR_001
 	mp.Clear(context.Background(), false)
 
+	//stm: @CHAIN_MEMPOOL_PENDING_001
 	pending, _ := mp.Pending(context.TODO())
 	if len(pending) != 10 {
 		t.Fatalf("expected 10 pending messages, but got %d instead", len(pending))
@@ -683,7 +701,7 @@ func TestUpdates(t *testing.T) {
 	tma := newTestMpoolAPI()
 	ds := datastore.NewMapDatastore()
 
-	mp, err := New(tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -724,6 +742,7 @@ func TestUpdates(t *testing.T) {
 
 	for i := 0; i < 10; i++ {
 		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		//stm: @CHAIN_MEMPOOL_PUSH_001
 		_, err := mp.Push(context.TODO(), m)
 		if err != nil {
 			t.Fatal(err)
@@ -745,3 +764,302 @@ func TestUpdates(t *testing.T) {
 		t.Fatal("expected closed channel, but got an update instead")
 	}
 }
+
+func TestMessageBelowMinGasFee(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	// fee is just below minimum gas fee
+	fee := minimumBaseFee.Uint64() - 1
+	{
+		msg := &types.Message{
+			To:         to,
+			From:       from,
+			Value:      types.NewInt(1),
+			Nonce:      0,
+			GasLimit:   50000000,
+			GasFeeCap:  types.NewInt(fee),
+			GasPremium: types.NewInt(1),
+			Params:     make([]byte, 32<<10),
+		}
+
+		sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{})
+		if err != nil {
+			panic(err)
+		}
+		sm := &types.SignedMessage{
+			Message:   *msg,
+			Signature: *sig,
+		}
+		err = mp.Add(context.TODO(), sm)
+		assert.ErrorIs(t, err, ErrGasFeeCapTooLow)
+	}
+}
+
+func TestMessageValueTooHigh(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	totalFil := types.TotalFilecoinInt
+	extra := types.NewInt(1)
+
+	value := types.BigAdd(totalFil, extra)
+	{
+		msg := &types.Message{
+			To:         to,
+			From:       from,
+			Value:      value,
+			Nonce:      0,
+			GasLimit:   50000000,
+			GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+			GasPremium: types.NewInt(1),
+			Params:     make([]byte, 32<<10),
+		}
+
+		sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{})
+		if err != nil {
+			panic(err)
+		}
+		sm := &types.SignedMessage{
+			Message:   *msg,
+			Signature: *sig,
+		}
+
+		err = mp.Add(context.TODO(), sm)
+		assert.Error(t, err)
+	}
+}
+
+func TestMessageSignatureInvalid(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		msg := &types.Message{
+			To:         to,
+			From:       from,
+			Value:      types.NewInt(1),
+			Nonce:      0,
+			GasLimit:   50000000,
+			GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+			GasPremium: types.NewInt(1),
+			Params:     make([]byte, 32<<10),
+		}
+
+		badSig := &crypto.Signature{
+			Type: crypto.SigTypeSecp256k1,
+			Data: make([]byte, 0),
+		}
+		sm := &types.SignedMessage{
+			Message:   *msg,
+			Signature: *badSig,
+		}
+		err = mp.Add(context.TODO(), sm)
+		assert.Error(t, err)
+		// assert.Contains(t, err.Error(), "invalid signature length")
+		assert.Error(t, err)
+	}
+}
+
+func TestAddMessageTwice(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		// create a valid messages
+		sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		// try to add it twice
+		err = mp.Add(context.TODO(), sm)
+		// assert.Contains(t, err.Error(), "with nonce 0 already in mpool")
+		assert.Error(t, err)
+	}
+}
+
+func TestAddMessageTwiceNonceGap(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		// create message with invalid nonce (1)
+		sm := makeTestMessage(w, from, to, 1, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		// then try to add message again
+		err = mp.Add(context.TODO(), sm)
+		// assert.Contains(t, err.Error(), "unfulfilled nonce gap")
+		assert.Error(t, err)
+	}
+}
+
+func TestAddMessageTwiceCidDiff(t *testing.T) {
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		// Create message with different data, so CID is different
+		sm2 := makeTestMessage(w, from, to, 0, 50_000_001, minimumBaseFee.Uint64())
+
+		//stm: @CHAIN_MEMPOOL_PUSH_001
+		// then try to add message again
+		err = mp.Add(context.TODO(), sm2)
+		// assert.Contains(t, err.Error(), "replace by fee has too low GasPremium")
+		assert.Error(t, err)
+	}
+}
+
+func TestAddMessageTwiceCidDiffReplaced(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		// Create message with different data, so CID is different
+		sm2 := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64()*2)
+		mustAdd(t, mp, sm2)
+	}
+}
+
+func TestRemoveMessage(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		//stm: @CHAIN_MEMPOOL_REMOVE_001
+		// remove message for sender
+		mp.Remove(context.TODO(), from, sm.Message.Nonce, true)
+
+		//stm: @CHAIN_MEMPOOL_PENDING_FOR_001
+		// check messages in pool: should be none present
+		msgs := mp.pendingFor(context.TODO(), from)
+		assert.Len(t, msgs, 0)
+	}
+}

chain/messagepool/provider.go

@@ -23,13 +23,13 @@ var (
 
 type Provider interface {
 	SubscribeHeadChanges(func(rev, app []*types.TipSet) error) *types.TipSet
-	PutMessage(m types.ChainMsg) (cid.Cid, error)
+	PutMessage(ctx context.Context, m types.ChainMsg) (cid.Cid, error)
 	PubSubPublish(string, []byte) error
 	GetActorAfter(address.Address, *types.TipSet) (*types.Actor, error)
 	StateAccountKeyAtFinality(context.Context, address.Address, *types.TipSet) (address.Address, error)
-	MessagesForBlock(*types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error)
-	MessagesForTipset(*types.TipSet) ([]types.ChainMsg, error)
-	LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error)
+	MessagesForBlock(context.Context, *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error)
+	MessagesForTipset(context.Context, *types.TipSet) ([]types.ChainMsg, error)
+	LoadTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error)
 	ChainComputeBaseFee(ctx context.Context, ts *types.TipSet) (types.BigInt, error)
 	IsLite() bool
 }
@@ -66,8 +66,8 @@ func (mpp *mpoolProvider) SubscribeHeadChanges(cb func(rev, app []*types.TipSet)
 	return mpp.sm.ChainStore().GetHeaviestTipSet()
 }
 
-func (mpp *mpoolProvider) PutMessage(m types.ChainMsg) (cid.Cid, error) {
-	return mpp.sm.ChainStore().PutMessage(m)
+func (mpp *mpoolProvider) PutMessage(ctx context.Context, m types.ChainMsg) (cid.Cid, error) {
+	return mpp.sm.ChainStore().PutMessage(ctx, m)
 }
 
 func (mpp *mpoolProvider) PubSubPublish(k string, v []byte) error {
@@ -103,16 +103,16 @@ func (mpp *mpoolProvider) StateAccountKeyAtFinality(ctx context.Context, addr ad
 	return mpp.sm.ResolveToKeyAddressAtFinality(ctx, addr, ts)
 }
 
-func (mpp *mpoolProvider) MessagesForBlock(h *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) {
-	return mpp.sm.ChainStore().MessagesForBlock(h)
+func (mpp *mpoolProvider) MessagesForBlock(ctx context.Context, h *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) {
+	return mpp.sm.ChainStore().MessagesForBlock(ctx, h)
 }
 
-func (mpp *mpoolProvider) MessagesForTipset(ts *types.TipSet) ([]types.ChainMsg, error) {
-	return mpp.sm.ChainStore().MessagesForTipset(ts)
+func (mpp *mpoolProvider) MessagesForTipset(ctx context.Context, ts *types.TipSet) ([]types.ChainMsg, error) {
+	return mpp.sm.ChainStore().MessagesForTipset(ctx, ts)
 }
 
-func (mpp *mpoolProvider) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error) {
-	return mpp.sm.ChainStore().LoadTipSet(tsk)
+func (mpp *mpoolProvider) LoadTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) {
+	return mpp.sm.ChainStore().LoadTipSet(ctx, tsk)
 }
 
 func (mpp *mpoolProvider) ChainComputeBaseFee(ctx context.Context, ts *types.TipSet) (types.BigInt, error) {

chain/messagepool/pruning.go

@@ -49,7 +49,7 @@ func (mp *MessagePool) pruneMessages(ctx context.Context, ts *types.TipSet) erro
 	}
 	baseFeeLowerBound := getBaseFeeLowerBound(baseFee, baseFeeLowerBoundFactor)
 
-	pending, _ := mp.getPendingMessages(ts, ts)
+	pending, _ := mp.getPendingMessages(ctx, ts, ts)
 
 	// protected actors -- not pruned
 	protected := make(map[address.Address]struct{})

chain/messagepool/repub_test.go

@@ -1,3 +1,4 @@
+//stm: #unit
 package messagepool
 
 import (
@@ -16,6 +17,7 @@ import (
 )
 
 func TestRepubMessages(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001
 	oldRepublishBatchDelay := RepublishBatchDelay
 	RepublishBatchDelay = time.Microsecond
 	defer func() {
@@ -25,7 +27,7 @@ func TestRepubMessages(t *testing.T) {
 	tma := newTestMpoolAPI()
 	ds := datastore.NewMapDatastore()
 
-	mp, err := New(tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -57,6 +59,7 @@ func TestRepubMessages(t *testing.T) {
 
 	for i := 0; i < 10; i++ {
 		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		//stm: @CHAIN_MEMPOOL_PUSH_001
 		_, err := mp.Push(context.TODO(), m)
 		if err != nil {
 			t.Fatal(err)

chain/messagepool/selection.go

@@ -210,7 +210,7 @@ func (mp *MessagePool) selectMessagesOptimal(ctx context.Context, curTs, ts *typ
 
 	// 0. Load messages from the target tipset; if it is the same as the current tipset in
 	//    the mpool, then this is just the pending messages
-	pending, err := mp.getPendingMessages(curTs, ts)
+	pending, err := mp.getPendingMessages(ctx, curTs, ts)
 	if err != nil {
 		return nil, err
 	}
@@ -458,7 +458,7 @@ func (mp *MessagePool) selectMessagesGreedy(ctx context.Context, curTs, ts *type
 
 	// 0. Load messages for the target tipset; if it is the same as the current tipset in the mpool
 	//    then this is just the pending messages
-	pending, err := mp.getPendingMessages(curTs, ts)
+	pending, err := mp.getPendingMessages(ctx, curTs, ts)
 	if err != nil {
 		return nil, err
 	}
@@ -695,7 +695,7 @@ tailLoop:
 	return result
 }
 
-func (mp *MessagePool) getPendingMessages(curTs, ts *types.TipSet) (map[address.Address]map[uint64]*types.SignedMessage, error) {
+func (mp *MessagePool) getPendingMessages(ctx context.Context, curTs, ts *types.TipSet) (map[address.Address]map[uint64]*types.SignedMessage, error) {
 	start := time.Now()
 
 	result := make(map[address.Address]map[uint64]*types.SignedMessage)
@@ -731,7 +731,7 @@ func (mp *MessagePool) getPendingMessages(curTs, ts *types.TipSet) (map[address.
 		return result, nil
 	}
 
-	if err := mp.runHeadChange(curTs, ts, result); err != nil {
+	if err := mp.runHeadChange(ctx, curTs, ts, result); err != nil {
 		return nil, xerrors.Errorf("failed to process difference between mpool head and given head: %w", err)
 	}
 

chain/messagepool/selection_test.go

@@ -1,3 +1,4 @@
+//stm: #unit
 package messagepool
 
 import (
@@ -65,7 +66,7 @@ func makeTestMessage(w *wallet.LocalWallet, from, to address.Address, nonce uint
 func makeTestMpool() (*MessagePool, *testMpoolAPI) {
 	tma := newTestMpoolAPI()
 	ds := datastore.NewMapDatastore()
-	mp, err := New(tma, ds, filcns.DefaultUpgradeSchedule(), "test", nil)
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "test", nil)
 	if err != nil {
 		panic(err)
 	}
@@ -74,6 +75,8 @@ func makeTestMpool() (*MessagePool, *testMpoolAPI) {
 }
 
 func TestMessageChains(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001
+	//stm: @CHAIN_MEMPOOL_CREATE_MSG_CHAINS_001
 	mp, tma := makeTestMpool()
 
 	// the actors
@@ -310,6 +313,8 @@ func TestMessageChains(t *testing.T) {
 }
 
 func TestMessageChainSkipping(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_CREATE_MSG_CHAINS_001
+
 	// regression test for chain skip bug
 
 	mp, tma := makeTestMpool()
@@ -382,6 +387,7 @@ func TestMessageChainSkipping(t *testing.T) {
 }
 
 func TestBasicMessageSelection(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	oldMaxNonceGap := MaxNonceGap
 	MaxNonceGap = 1000
 	defer func() {
@@ -532,6 +538,7 @@ func TestBasicMessageSelection(t *testing.T) {
 }
 
 func TestMessageSelectionTrimmingGas(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()
 
 	// the actors
@@ -595,6 +602,7 @@ func TestMessageSelectionTrimmingGas(t *testing.T) {
 }
 
 func TestMessageSelectionTrimmingMsgsBasic(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()
 
 	// the actors
@@ -641,6 +649,7 @@ func TestMessageSelectionTrimmingMsgsBasic(t *testing.T) {
 }
 
 func TestMessageSelectionTrimmingMsgsTwoSendersBasic(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()
 
 	// the actors
@@ -707,6 +716,7 @@ func TestMessageSelectionTrimmingMsgsTwoSendersBasic(t *testing.T) {
 }
 
 func TestMessageSelectionTrimmingMsgsTwoSendersAdvanced(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()
 
 	// the actors
@@ -788,6 +798,7 @@ func TestMessageSelectionTrimmingMsgsTwoSendersAdvanced(t *testing.T) {
 }
 
 func TestPriorityMessageSelection(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()
 
 	// the actors
@@ -867,6 +878,7 @@ func TestPriorityMessageSelection(t *testing.T) {
 }
 
 func TestPriorityMessageSelection2(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()
 
 	// the actors
@@ -934,6 +946,7 @@ func TestPriorityMessageSelection2(t *testing.T) {
 }
 
 func TestPriorityMessageSelection3(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()
 
 	// the actors
@@ -1028,6 +1041,8 @@ func TestPriorityMessageSelection3(t *testing.T) {
 }
 
 func TestOptimalMessageSelection1(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	// this test uses just a single actor sending messages with a low tq
 	// the chain depenent merging algorithm should pick messages from the actor
 	// from the start
@@ -1094,6 +1109,8 @@ func TestOptimalMessageSelection1(t *testing.T) {
 }
 
 func TestOptimalMessageSelection2(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	// this test uses two actors sending messages to each other, with the first
 	// actor paying (much) higher gas premium than the second.
 	// We select with a low ticket quality; the chain depenent merging algorithm should pick
@@ -1173,6 +1190,8 @@ func TestOptimalMessageSelection2(t *testing.T) {
 }
 
 func TestOptimalMessageSelection3(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	// this test uses 10 actors sending a block of messages to each other, with the the first
 	// actors paying higher gas premium than the subsequent actors.
 	// We select with a low ticket quality; the chain dependent merging algorithm should pick
@@ -1416,6 +1435,8 @@ func makeZipfPremiumDistribution(rng *rand.Rand) func() uint64 {
 }
 
 func TestCompetitiveMessageSelectionExp(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	if testing.Short() {
 		t.Skip("skipping in short mode")
 	}
@@ -1439,6 +1460,8 @@ func TestCompetitiveMessageSelectionExp(t *testing.T) {
 }
 
 func TestCompetitiveMessageSelectionZipf(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	if testing.Short() {
 		t.Skip("skipping in short mode")
 	}
@@ -1462,6 +1485,7 @@ func TestCompetitiveMessageSelectionZipf(t *testing.T) {
 }
 
 func TestGasReward(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_GET_GAS_REWARD_001
 	tests := []struct {
 		Premium   uint64
 		FeeCap    uint64
@@ -1494,6 +1518,8 @@ func TestGasReward(t *testing.T) {
 }
 
 func TestRealWorldSelection(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @TOKEN_WALLET_SIGN_001, @CHAIN_MEMPOOL_SELECT_001
+
 	// load test-messages.json.gz and rewrite the messages so that
 	// 1) we map each real actor to a test actor so that we can sign the messages
 	// 2) adjust the nonces so that they start from 0

chain/messagesigner/messagesigner.go

@@ -84,7 +84,7 @@ func (ms *MessageSigner) SignMessage(ctx context.Context, msg *types.Message, cb
 	}
 
 	// If the callback executed successfully, write the nonce to the datastore
-	if err := ms.saveNonce(msg.From, nonce); err != nil {
+	if err := ms.saveNonce(ctx, msg.From, nonce); err != nil {
 		return nil, xerrors.Errorf("failed to save nonce: %w", err)
 	}
 
@@ -105,7 +105,7 @@ func (ms *MessageSigner) nextNonce(ctx context.Context, addr address.Address) (u
 
 	// Get the next nonce for this address from the datastore
 	addrNonceKey := ms.dstoreKey(addr)
-	dsNonceBytes, err := ms.ds.Get(addrNonceKey)
+	dsNonceBytes, err := ms.ds.Get(ctx, addrNonceKey)
 
 	switch {
 	case xerrors.Is(err, datastore.ErrNotFound):
@@ -139,7 +139,7 @@ func (ms *MessageSigner) nextNonce(ctx context.Context, addr address.Address) (u
 
 // saveNonce increments the nonce for this address and writes it to the
 // datastore
-func (ms *MessageSigner) saveNonce(addr address.Address, nonce uint64) error {
+func (ms *MessageSigner) saveNonce(ctx context.Context, addr address.Address, nonce uint64) error {
 	// Increment the nonce
 	nonce++
 
@@ -150,7 +150,7 @@ func (ms *MessageSigner) saveNonce(addr address.Address, nonce uint64) error {
 	if err != nil {
 		return xerrors.Errorf("failed to marshall nonce: %w", err)
 	}
-	err = ms.ds.Put(addrNonceKey, buf.Bytes())
+	err = ms.ds.Put(ctx, addrNonceKey, buf.Bytes())
 	if err != nil {
 		return xerrors.Errorf("failed to write nonce to datastore: %w", err)
 	}

chain/messagesigner/messagesigner_test.go

@@ -1,3 +1,4 @@
+//stm: #unit
 package messagesigner
 
 import (
@@ -60,6 +61,7 @@ func TestMessageSignerSignMessage(t *testing.T) {
 	to2, err := w.WalletNew(ctx, types.KTSecp256k1)
 	require.NoError(t, err)
 
+	//stm: @CHAIN_MESSAGE_SIGNER_NEW_SIGNER_001, @CHAIN_MESSAGE_SIGNER_SIGN_MESSAGE_001, @CHAIN_MESSAGE_SIGNER_SIGN_MESSAGE_005
 	type msgSpec struct {
 		msg        *types.Message
 		mpoolNonce [1]uint64