feat: config: Move MakeNewSectorForDeals config into the Sealing sectoin

fix: worker: Fix default value handling
feat: miner cli: Separate proving workers command
2022-03-26 16:46:37 -04:00 · 2022-03-26 16:46:00 -04:00 · 2022-03-26 16:43:27 -04:00 · 2022-03-25 20:31:49 -04:00 · 2022-03-25 18:54:59 -04:00 · 2022-03-25 17:28:41 -04:00
663 changed files with 45131 additions and 10273 deletions
@@ -44,13 +44,13 @@ commands:
      - restore_cache:
          name: Restore parameters cache
          keys:
-            - 'v25-2k-lotus-params'
+            - 'v26-2k-lotus-params'
          paths:
            - /var/tmp/filecoin-proof-parameters/
      - run:  ./lotus fetch-params 2048
      - save_cache:
          name: Save parameters cache
-          key: 'v25-2k-lotus-params'
+          key: 'v26-2k-lotus-params'
          paths:
            - /var/tmp/filecoin-proof-parameters/
  install_ipfs:
@@ -318,7 +318,7 @@ jobs:
          command: pushd testplans/lotus-soup && mkdir -p $HOME/testground && cp env-ci.toml $HOME/testground/.env.toml && echo 'endpoint="https://ci.testground.ipfs.team"' >> $HOME/testground/.env.toml && echo 'user="circleci"' >> $HOME/testground/.env.toml
      - run:
          name: "prepare testground home dir and link test plans"
-          command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup && ln -s $(pwd)/testplans/graphsync $HOME/testground/plans/graphsync
+          command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup
      - run:
          name: "go get lotus@master"
          command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master
@@ -328,10 +328,6 @@ jobs:
      - run:
          name: "trigger payment channel stress testplan on taas"
          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-      - run:
-          name: "trigger graphsync testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/graphsync/_compositions/stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-

  build-macos:
    description: build darwin lotus binary
@@ -394,7 +390,7 @@ jobs:

  build-appimage:
    machine:
-      image: ubuntu-2004:202104-01
+      image: ubuntu-2004:202111-02
    steps:
      - checkout
      - attach_workspace:
@@ -402,6 +398,16 @@ jobs:
      - run:
          name: install appimage-builder
          command: |
+            # appimage-builder requires /dev/snd to exist. It creates containers during the testing phase
+            # that pass sound devices from the host to the testing container. (hard coded!)
+            # https://github.com/AppImageCrafters/appimage-builder/blob/master/appimagebuilder/modules/test/execution_test.py#L54
+            # Circleci doesn't provide a working sound device; this is enough to fake it.
+            if [ ! -e /dev/snd ]
+            then
+              sudo mkdir /dev/snd
+              sudo mknod /dev/snd/ControlC0 c 1 2
+            fi
+
            # docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html
            sudo apt update
            sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace
@@ -622,6 +628,11 @@ jobs:
        default: "latest"
        description: A comma-separated string containing docker image tags to build and push (default = latest)

+      target:
+        type: string
+        default: "lotus-all-in-one"
+        description: Docker target to build
+
    steps:
      - run:
          name: Confirm that environment variables are set
@@ -661,6 +672,7 @@ jobs:

            docker build \
              <<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
+              --target <<parameters.target>> \
              -f <<parameters.path>>/<<parameters.dockerfile>> \
              $docker_tag_args \
              <<parameters.path>>
@@ -709,6 +721,17 @@ jobs:
      - packer/build:
          template: tools/packer/lotus.pkr.hcl
          args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG"
+  publish-packer-snap:
+    description: build packer image with snap. mainnet only.
+    executor:
+      name: packer/default
+      packer-version: 1.6.6
+    steps:
+      - checkout
+      - attach_workspace:
+          at: "."
+      - packer/build:
+          template: tools/packer/lotus-snap.pkr.hcl
  publish-dockerhub:
    description: publish to dockerhub
    machine:
@@ -789,6 +812,11 @@ workflows:
          suite: itest-deals_512mb
          target: "./itests/deals_512mb_test.go"
      
+      - test:
+          name: test-itest-deals_anycid
+          suite: itest-deals_anycid
+          target: "./itests/deals_anycid_test.go"
+      
      - test:
          name: test-itest-deals_concurrent
          suite: itest-deals_concurrent
@@ -809,6 +837,11 @@ workflows:
          suite: itest-deals_padding
          target: "./itests/deals_padding_test.go"
      
+      - test:
+          name: test-itest-deals_partial_retrieval_dm-level
+          suite: itest-deals_partial_retrieval_dm-level
+          target: "./itests/deals_partial_retrieval_dm-level_test.go"
+      
      - test:
          name: test-itest-deals_partial_retrieval
          suite: itest-deals_partial_retrieval
@@ -829,6 +862,11 @@ workflows:
          suite: itest-deals_publish
          target: "./itests/deals_publish_test.go"
      
+      - test:
+          name: test-itest-deals_retry_deal_no_funds
+          suite: itest-deals_retry_deal_no_funds
+          target: "./itests/deals_retry_deal_no_funds_test.go"
+      
      - test:
          name: test-itest-deals
          suite: itest-deals
@@ -844,6 +882,11 @@ workflows:
          suite: itest-get_messages_in_ts
          target: "./itests/get_messages_in_ts_test.go"
      
+      - test:
+          name: test-itest-mempool
+          suite: itest-mempool
+          target: "./itests/mempool_test.go"
+      
      - test:
          name: test-itest-multisig
          suite: itest-multisig
@@ -874,6 +917,11 @@ workflows:
          suite: itest-sector_finalize_early
          target: "./itests/sector_finalize_early_test.go"
      
+      - test:
+          name: test-itest-sector_make_cc_avail
+          suite: itest-sector_make_cc_avail
+          target: "./itests/sector_make_cc_avail_test.go"
+      
      - test:
          name: test-itest-sector_miner_collateral
          suite: itest-sector_miner_collateral
@@ -884,11 +932,26 @@ workflows:
          suite: itest-sector_pledge
          target: "./itests/sector_pledge_test.go"
      
+      - test:
+          name: test-itest-sector_prefer_no_upgrade
+          suite: itest-sector_prefer_no_upgrade
+          target: "./itests/sector_prefer_no_upgrade_test.go"
+      
+      - test:
+          name: test-itest-sector_revert_available
+          suite: itest-sector_revert_available
+          target: "./itests/sector_revert_available_test.go"
+      
      - test:
          name: test-itest-sector_terminate
          suite: itest-sector_terminate
          target: "./itests/sector_terminate_test.go"
      
+      - test:
+          name: test-itest-self_sent_txn
+          suite: itest-self_sent_txn
+          target: "./itests/self_sent_txn_test.go"
+      
      - test:
          name: test-itest-tape
          suite: itest-tape
@@ -909,6 +972,11 @@ workflows:
          suite: itest-wdpost
          target: "./itests/wdpost_test.go"
      
+      - test:
+          name: test-itest-worker
+          suite: itest-worker
+          target: "./itests/worker_test.go"
+      
      - test:
          name: test-unit-cli
          suite: utest-unit-cli
@@ -939,7 +1007,7 @@ workflows:
          codecov-upload: false
          suite: conformance-bleeding-edge
          target: "./conformance"
-          vectors-branch: master
+          vectors-branch: specs-actors-v7
      - trigger-testplans:
          filters:
            branches:
@@ -995,6 +1063,13 @@ workflows:
          path: .
          repo: lotus-dev
          tag: '${CIRCLE_SHA1:0:8}'
+          target: lotus-all-in-one
+      - build-and-push-image:
+          dockerfile: Dockerfile.lotus
+          path: .
+          repo: lotus-test
+          tag: '${CIRCLE_SHA1:0:8}'
+          target: lotus-test
      - publish-packer-mainnet:
          requires:
            - build-all
@@ -1061,3 +1136,13 @@ workflows:
      - publish-dockerhub:
          name: publish-dockerhub-nightly
          tag: nightly
+  monthly:
+    triggers:
+      - schedule:
+          cron: "0 0 1 * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - publish-packer-snap
@@ -44,13 +44,13 @@ commands:
      - restore_cache:
          name: Restore parameters cache
          keys:
-            - 'v25-2k-lotus-params'
+            - 'v26-2k-lotus-params'
          paths:
            - /var/tmp/filecoin-proof-parameters/
      - run:  ./lotus fetch-params 2048
      - save_cache:
          name: Save parameters cache
-          key: 'v25-2k-lotus-params'
+          key: 'v26-2k-lotus-params'
          paths:
            - /var/tmp/filecoin-proof-parameters/
  install_ipfs:
@@ -318,7 +318,7 @@ jobs:
          command: pushd testplans/lotus-soup && mkdir -p $HOME/testground && cp env-ci.toml $HOME/testground/.env.toml && echo 'endpoint="https://ci.testground.ipfs.team"' >> $HOME/testground/.env.toml && echo 'user="circleci"' >> $HOME/testground/.env.toml
      - run:
          name: "prepare testground home dir and link test plans"
-          command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup && ln -s $(pwd)/testplans/graphsync $HOME/testground/plans/graphsync
+          command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup
      - run:
          name: "go get lotus@master"
          command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master
@@ -328,10 +328,6 @@ jobs:
      - run:
          name: "trigger payment channel stress testplan on taas"
          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-      - run:
-          name: "trigger graphsync testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/graphsync/_compositions/stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-

  build-macos:
    description: build darwin lotus binary
@@ -394,7 +390,7 @@ jobs:

  build-appimage:
    machine:
-      image: ubuntu-2004:202104-01
+      image: ubuntu-2004:202111-02
    steps:
      - checkout
      - attach_workspace:
@@ -402,6 +398,16 @@ jobs:
      - run:
          name: install appimage-builder
          command: |
+            # appimage-builder requires /dev/snd to exist. It creates containers during the testing phase
+            # that pass sound devices from the host to the testing container. (hard coded!)
+            # https://github.com/AppImageCrafters/appimage-builder/blob/master/appimagebuilder/modules/test/execution_test.py#L54
+            # Circleci doesn't provide a working sound device; this is enough to fake it.
+            if [ ! -e /dev/snd ]
+            then
+              sudo mkdir /dev/snd
+              sudo mknod /dev/snd/ControlC0 c 1 2
+            fi
+
            # docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html
            sudo apt update
            sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace
@@ -622,6 +628,11 @@ jobs:
        default: "latest"
        description: A comma-separated string containing docker image tags to build and push (default = latest)

+      target:
+        type: string
+        default: "lotus-all-in-one"
+        description: Docker target to build
+
    steps:
      - run:
          name: Confirm that environment variables are set
@@ -661,6 +672,7 @@ jobs:

            docker build \
              <<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
+              --target <<parameters.target>> \
              -f <<parameters.path>>/<<parameters.dockerfile>> \
              $docker_tag_args \
              <<parameters.path>>
@@ -709,6 +721,17 @@ jobs:
      - packer/build:
          template: tools/packer/lotus.pkr.hcl
          args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG"
+  publish-packer-snap:
+    description: build packer image with snap. mainnet only.
+    executor:
+      name: packer/default
+      packer-version: 1.6.6
+    steps:
+      - checkout
+      - attach_workspace:
+          at: "."
+      - packer/build:
+          template: tools/packer/lotus-snap.pkr.hcl
  publish-dockerhub:
    description: publish to dockerhub
    machine:
@@ -789,7 +812,7 @@ workflows:
          codecov-upload: false
          suite: conformance-bleeding-edge
          target: "./conformance"
-          vectors-branch: master
+          vectors-branch: specs-actors-v7
      - trigger-testplans:
          filters:
            branches:
@@ -845,6 +868,13 @@ workflows:
          path: .
          repo: lotus-dev
          tag: '${CIRCLE_SHA1:0:8}'
+          target: lotus-all-in-one
+      - build-and-push-image:
+          dockerfile: Dockerfile.lotus
+          path: .
+          repo: lotus-test
+          tag: '${CIRCLE_SHA1:0:8}'
+          target: lotus-test
      - publish-packer-mainnet:
          requires:
            - build-all
@@ -911,3 +941,13 @@ workflows:
      - publish-dockerhub:
          name: publish-dockerhub-nightly
          tag: nightly
+  monthly:
+    triggers:
+      - schedule:
+          cron: "0 0 1 * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - publish-packer-snap
@@ -0,0 +1,21 @@
+## Related Issues
+<!-- link all issues that this PR might resolve/fix. If an issue doesn't exist, include a brief motivation for the change being made.-->
+
+## Proposed Changes
+<!-- provide a clear list of the changes being made-->
+
+
+## Additional Info
+<!-- callouts, links to documentation, and etc-->
+
+## Checklist
+
+Before you mark the PR ready for review, please make sure that:
+- [ ] All commits have a clear commit message.
+- [ ] The PR title is in the form of of `<PR type>: <area>: <change being made>`
+    - example: ` fix: mempool: Introduce a cache for valid signatures`
+    - `PR type`: _fix_, _feat_, _INTERFACE BREAKING CHANGE_, _CONSENSUS BREAKING_, _build_, _chore_, _ci_, _docs_,_perf_, _refactor_, _revert_, _style_, _test_
+    - `area`: _api_, _chain_, _state_, _vm_, _data transfer_, _market_, _mempool_, _message_, _block production_, _multisig_, _networking_, _paychan_, _proving_, _sealing_, _wallet_, _deps_
+- [ ] This PR has tests for new functionality or change in behaviour
+- [ ] If new user-facing features are introduced, clear usage guidelines and / or documentation updates should be included in https://lotus.filecoin.io or [Discussion Tutorials.](https://github.com/filecoin-project/lotus/discussions/categories/tutorials)
+- [ ] CI is green
@@ -40,6 +40,7 @@ build/paramfetch.sh
 /bundle
 /darwin
 /linux
+*.snap

 *-fuzz.zip
 /chain/types/work_msg/
@@ -49,23 +49,23 @@ AppDir:
    fedora:
      image: appimagecrafters/tests-env:fedora-30
      command: ./AppRun
-      use_host_x: true
+      use_host_x: false
    debian:
      image: appimagecrafters/tests-env:debian-stable
      command: ./AppRun
-      use_host_x: true
+      use_host_x: false
    arch:
      image: appimagecrafters/tests-env:archlinux-latest
      command: ./AppRun
-      use_host_x: true
+      use_host_x: false
    centos:
      image: appimagecrafters/tests-env:centos-7
      command: ./AppRun
-      use_host_x: true
+      use_host_x: false
    ubuntu:
      image: appimagecrafters/tests-env:ubuntu-xenial
      command: ./AppRun
-      use_host_x: true
+      use_host_x: false
 AppImage:
  arch: x86_64
  update-information: guess
@@ -28,6 +28,14 @@ WORKDIR /opt/filecoin
 RUN make clean deps


+FROM builder-local AS builder-test
+MAINTAINER Lotus Development Team
+
+WORKDIR /opt/filecoin
+
+RUN make debug
+
+
 FROM builder-local AS builder
 MAINTAINER Lotus Development Team

@@ -36,7 +44,7 @@ WORKDIR /opt/filecoin
 ARG RUSTFLAGS=""
 ARG GOFLAGS=""

-RUN make lotus lotus-miner lotus-worker lotus-shed lotus-wallet lotus-gateway
+RUN make lotus lotus-miner lotus-worker lotus-shed lotus-wallet lotus-gateway lotus-stats


 FROM ubuntu:20.04 AS base
@@ -53,8 +61,9 @@ COPY --from=builder /usr/lib/x86_64-linux-gnu/libnuma.so.1   /lib/
 COPY --from=builder /usr/lib/x86_64-linux-gnu/libhwloc.so.5  /lib/
 COPY --from=builder /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /lib/

-RUN useradd -r -u 532 -U fc
-
+RUN useradd -r -u 532 -U fc \
+ && mkdir -p /etc/OpenCL/vendors \
+ && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd

 ###
 FROM base AS lotus
@@ -66,8 +75,6 @@ COPY scripts/docker-lotus-entrypoint.sh /

 ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
 ENV LOTUS_PATH /var/lib/lotus
-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831
 ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
 ENV DOCKER_LOTUS_IMPORT_WALLET ""

@@ -92,8 +99,6 @@ MAINTAINER Lotus Development Team
 COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/

 ENV WALLET_PATH /var/lib/lotus-wallet
-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831

 RUN mkdir /var/lib/lotus-wallet
 RUN chown fc: /var/lib/lotus-wallet
@@ -114,10 +119,6 @@ MAINTAINER Lotus Development Team

 COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/

-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831
-ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
-
 USER fc

 EXPOSE 1234
@@ -135,11 +136,7 @@ COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/
 COPY scripts/docker-lotus-miner-entrypoint.sh /

 ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
 ENV LOTUS_MINER_PATH /var/lib/lotus-miner
-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831
-ENV DOCKER_LOTUS_MINER_INIT true

 RUN mkdir /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters
 RUN chown fc: /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters
@@ -163,10 +160,7 @@ MAINTAINER Lotus Development Team
 COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/

 ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV MINER_API_INFO /ip4/127.0.0.1/tcp/2345/http
 ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831

 RUN mkdir /var/lib/lotus-worker
 RUN chown fc: /var/lib/lotus-worker
@@ -186,16 +180,11 @@ CMD ["-help"]
 from base as lotus-all-in-one

 ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831
 ENV LOTUS_MINER_PATH /var/lib/lotus-miner
 ENV LOTUS_PATH /var/lib/lotus
 ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
-ENV MINER_API_INFO /ip4/127.0.0.1/tcp/2345/http
 ENV WALLET_PATH /var/lib/lotus-wallet
 ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
-ENV DOCKER_LOTUS_MINER_INIT true

 COPY --from=builder /opt/filecoin/lotus      /usr/local/bin/
 COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/
@@ -203,6 +192,44 @@ COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/
 COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/
 COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/
 COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/
+COPY --from=builder /opt/filecoin/lotus-stats /usr/local/bin/
+
+RUN mkdir /var/tmp/filecoin-proof-parameters
+RUN mkdir /var/lib/lotus
+RUN mkdir /var/lib/lotus-miner
+RUN mkdir /var/lib/lotus-worker
+RUN mkdir /var/lib/lotus-wallet
+RUN chown fc: /var/tmp/filecoin-proof-parameters
+RUN chown fc: /var/lib/lotus
+RUN chown fc: /var/lib/lotus-miner
+RUN chown fc: /var/lib/lotus-worker
+RUN chown fc: /var/lib/lotus-wallet
+
+
+VOLUME /var/tmp/filecoin-proof-parameters
+VOLUME /var/lib/lotus
+VOLUME /var/lib/lotus-miner
+VOLUME /var/lib/lotus-worker
+VOLUME /var/lib/lotus-wallet
+
+EXPOSE 1234
+EXPOSE 2345
+EXPOSE 3456
+EXPOSE 1777
+
+###
+from base as lotus-test
+
+ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
+ENV LOTUS_MINER_PATH /var/lib/lotus-miner
+ENV LOTUS_PATH /var/lib/lotus
+ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
+ENV WALLET_PATH /var/lib/lotus-wallet
+
+COPY --from=builder-test /opt/filecoin/lotus      /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-miner /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-worker /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-seed /usr/local/bin/

 RUN mkdir /var/tmp/filecoin-proof-parameters
 RUN mkdir /var/lib/lotus
@@ -97,7 +97,7 @@ BINS+=lotus-miner

 lotus-worker: $(BUILD_DEPS)
 	rm -f lotus-worker
-	$(GOCC) build $(GOFLAGS) -o lotus-worker ./cmd/lotus-seal-worker
+	$(GOCC) build $(GOFLAGS) -o lotus-worker ./cmd/lotus-worker
 .PHONY: lotus-worker
 BINS+=lotus-worker

@@ -291,6 +291,7 @@ method-gen: api-gen
 	(cd ./lotuspond/front/src/chain && $(GOCC) run ./methodgen.go)

 actors-gen:
+	$(GOCC) run ./gen/inline-gen . gen/inlinegen-data.json
 	$(GOCC) run ./chain/actors/agen
 	$(GOCC) fmt ./...

@@ -344,6 +345,8 @@ gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci
 	@echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO MAKE docsgen-cli"
 .PHONY: gen

+jen: gen
+
 snap: lotus lotus-miner lotus-worker
 	snapcraft
 	# snapcraft upload ./lotus_*.snap
@@ -359,4 +362,4 @@ print-%:
 	@echo $*=$($*)

 circleci:
-	go generate -x ./.circleci
+	go generate -x ./.circleci
@@ -7,7 +7,6 @@ import (
 	"time"

 	"github.com/ipfs/go-cid"
-	textselector "github.com/ipld/go-ipld-selector-text-lite"
 	"github.com/libp2p/go-libp2p-core/peer"

 	"github.com/filecoin-project/go-address"
@@ -28,7 +27,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
 	"github.com/filecoin-project/lotus/chain/types"
-	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/repo/imports"
 )
@@ -291,7 +289,7 @@ type FullNode interface {

 	// // UX ?

-	// MethodGroup: Wallet
+	// MethodGroup: WalletF

 	// WalletNew creates a new address in the wallet with the given sigType.
 	// Available key types: bls, secp256k1, secp256k1-ledger
@@ -352,10 +350,11 @@ type FullNode interface {
 	// ClientMinerQueryOffer returns a QueryOffer for the specific miner and file.
 	ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (QueryOffer, error) //perm:read
 	// ClientRetrieve initiates the retrieval of a file, as specified in the order.
-	ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *FileRef) error //perm:admin
-	// ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel
-	// of status updates.
-	ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin
+	ClientRetrieve(ctx context.Context, params RetrievalOrder) (*RestrievalRes, error) //perm:admin
+	// ClientRetrieveWait waits for retrieval to be complete
+	ClientRetrieveWait(ctx context.Context, deal retrievalmarket.DealID) error //perm:admin
+	// ClientExport exports a file stored in the local filestore to a system file
+	ClientExport(ctx context.Context, exportRef ExportRef, fileRef FileRef) error //perm:admin
 	// ClientListRetrievals returns information about retrievals made by the local client
 	ClientListRetrievals(ctx context.Context) ([]RetrievalInfo, error) //perm:write
 	// ClientGetRetrievalUpdates returns status of updated retrieval deals
@@ -630,10 +629,14 @@ type FullNode interface {
 	// <sender address of the approve msg>, <method to call in the proposed message>, <params to include in the proposed message>
 	MsigApproveTxnHash(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (*MessagePrototype, error) //perm:sign

+	// MsigCancel cancels a previously-proposed multisig message
+	// It takes the following params: <multisig address>, <proposed transaction ID> <signer address>
+	MsigCancel(context.Context, address.Address, uint64, address.Address) (*MessagePrototype, error) //perm:sign
+
 	// MsigCancel cancels a previously-proposed multisig message
 	// It takes the following params: <multisig address>, <proposed transaction ID>, <recipient address>, <value to transfer>,
 	// <sender address of the cancel msg>, <method to call in the proposed message>, <params to include in the proposed message>
-	MsigCancel(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (*MessagePrototype, error) //perm:sign
+	MsigCancelTxnHash(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (*MessagePrototype, error) //perm:sign

 	// MsigAddPropose proposes adding a signer in the multisig
 	// It takes the following params: <multisig address>, <sender address of the propose msg>,
@@ -686,7 +689,17 @@ type FullNode interface {
 	// MethodGroup: Paych
 	// The Paych methods are for interacting with and managing payment channels

-	PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*ChannelInfo, error)                     //perm:sign
+	// PaychGet gets or creates a payment channel between address pair
+	//  The specified amount will be reserved for use. If there aren't enough non-reserved funds
+	//    available, funds will be added through an on-chain message.
+	//  - When opts.OffChain is true, this call will not cause any messages to be sent to the chain (no automatic
+	//    channel creation/funds adding). If the operation can't be performed without sending a message an error will be
+	//    returned. Note that even when this option is specified, this call can be blocked by previous operations on the
+	//    channel waiting for on-chain operations.
+	PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt, opts PaychGetOpts) (*ChannelInfo, error) //perm:sign
+	// PaychFund gets or creates a payment channel between address pair.
+	// The specified amount will be added to the channel through on-chain send for future use
+	PaychFund(ctx context.Context, from, to address.Address, amt types.BigInt) (*ChannelInfo, error)                    //perm:sign
 	PaychGetWaitReady(context.Context, cid.Cid) (address.Address, error)                                                //perm:sign
 	PaychAvailableFunds(ctx context.Context, ch address.Address) (*ChannelAvailableFunds, error)                        //perm:sign
 	PaychAvailableFundsByFromTo(ctx context.Context, from, to address.Address) (*ChannelAvailableFunds, error)          //perm:sign
@@ -825,6 +838,10 @@ const (
 	PCHOutbound
 )

+type PaychGetOpts struct {
+	OffChain bool
+}
+
 type PaychStatus struct {
 	ControlAddr address.Address
 	Direction   PCHDir
@@ -842,16 +859,23 @@ type ChannelAvailableFunds struct {
 	From address.Address
 	// To is the to address of the channel
 	To address.Address
-	// ConfirmedAmt is the amount of funds that have been confirmed on-chain
-	// for the channel
+
+	// ConfirmedAmt is the total amount of funds that have been confirmed on-chain for the channel
 	ConfirmedAmt types.BigInt
 	// PendingAmt is the amount of funds that are pending confirmation on-chain
 	PendingAmt types.BigInt
+
+	// NonReservedAmt is part of ConfirmedAmt that is available for use (e.g. when the payment channel was pre-funded)
+	NonReservedAmt types.BigInt
+	// PendingAvailableAmt is the amount of funds that are pending confirmation on-chain that will become available once confirmed
+	PendingAvailableAmt types.BigInt
+
 	// PendingWaitSentinel can be used with PaychGetWaitReady to wait for
 	// confirmation of pending funds
 	PendingWaitSentinel *cid.Cid
 	// QueuedAmt is the amount that is queued up behind a pending request
 	QueuedAmt types.BigInt
+
 	// VoucherRedeemedAmt is the amount that is redeemed by vouchers on-chain
 	// and in the local datastore
 	VoucherReedeemedAmt types.BigInt
@@ -897,6 +921,7 @@ type QueryOffer struct {
 	Size                    uint64
 	MinPrice                types.BigInt
 	UnsealPrice             types.BigInt
+	PricePerByte            abi.TokenAmount
 	PaymentInterval         uint64
 	PaymentIntervalIncrease uint64
 	Miner                   address.Address
@@ -930,15 +955,14 @@ type MarketDeal struct {
 }

 type RetrievalOrder struct {
-	// TODO: make this less unixfs specific
-	Root                  cid.Cid
-	Piece                 *cid.Cid
-	DatamodelPathSelector *textselector.Expression
-	Size                  uint64
+	Root         cid.Cid
+	Piece        *cid.Cid
+	DataSelector *Selector
+
+	// todo: Size/Total are only used for calculating price per byte; we should let users just pass that
+	Size  uint64
+	Total types.BigInt

-	FromLocalCAR string // if specified, get data from a local CARv2 file.
-	// TODO: support offset
-	Total                   types.BigInt
 	UnsealPrice             types.BigInt
 	PaymentInterval         uint64
 	PaymentIntervalIncrease uint64
@@ -1081,7 +1105,7 @@ type CirculatingSupply struct {
 type MiningBaseInfo struct {
 	MinerPower        types.BigInt
 	NetworkPower      types.BigInt
-	Sectors           []builtin.SectorInfo
+	Sectors           []builtin.ExtendedSectorInfo
 	WorkerKey         address.Address
 	SectorSize        abi.SectorSize
 	PrevBeaconEntry   types.BeaconEntry
@@ -31,6 +31,8 @@ import (
 type Gateway interface {
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
 	ChainHead(ctx context.Context) (*types.TipSet, error)
+	ChainGetParentMessages(context.Context, cid.Cid) ([]Message, error)
+	ChainGetParentReceipts(context.Context, cid.Cid) ([]*types.MessageReceipt, error)
 	ChainGetBlockMessages(context.Context, cid.Cid) (*BlockMessages, error)
 	ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
 	ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*HeadChange, error)
@@ -39,11 +41,13 @@ type Gateway interface {
 	ChainGetTipSetAfterHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error)
 	ChainNotify(context.Context) (<-chan []*HeadChange, error)
 	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
+	ChainGetGenesis(context.Context) (*types.TipSet, error)
 	GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
 	MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
 	MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
-	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
 	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
+	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
+	MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MsigVesting, error)
 	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
 	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
 	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
@@ -2,6 +2,7 @@ package api

 import (
 	"context"
+	"time"

 	metrics "github.com/libp2p/go-libp2p-core/metrics"
 	"github.com/libp2p/go-libp2p-core/network"
@@ -25,6 +26,7 @@ type Net interface {

 	NetConnectedness(context.Context, peer.ID) (network.Connectedness, error) //perm:read
 	NetPeers(context.Context) ([]peer.AddrInfo, error)                        //perm:read
+	NetPing(context.Context, peer.ID) (time.Duration, error)                  //perm:read
 	NetConnect(context.Context, peer.AddrInfo) error                          //perm:write
 	NetAddrsListen(context.Context) (peer.AddrInfo, error)                    //perm:read
 	NetDisconnect(context.Context, peer.ID) error                             //perm:write
@@ -51,6 +53,15 @@ type Net interface {
 	NetBlockRemove(ctx context.Context, acl NetBlockList) error //perm:admin
 	NetBlockList(ctx context.Context) (NetBlockList, error)     //perm:read

+	NetProtectAdd(ctx context.Context, acl []peer.ID) error    //perm:admin
+	NetProtectRemove(ctx context.Context, acl []peer.ID) error //perm:admin
+	NetProtectList(ctx context.Context) ([]peer.ID, error)     //perm:read
+
+	// ResourceManager API
+	NetStat(ctx context.Context, scope string) (NetStat, error)          //perm:read
+	NetLimit(ctx context.Context, scope string) (NetLimit, error)        //perm:read
+	NetSetLimit(ctx context.Context, scope string, limit NetLimit) error //perm:admin
+
 	// ID returns peerID of libp2p node backing this API
 	ID(context.Context) (peer.ID, error) //perm:read
 }
@@ -14,6 +14,7 @@ import (
 	"github.com/filecoin-project/go-address"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/filecoin-project/go-state-types/abi"
+	abinetwork "github.com/filecoin-project/go-state-types/network"
 	"github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
 	"github.com/filecoin-project/specs-storage/storage"

@@ -23,7 +24,6 @@ import (

 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 	"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
 )
@@ -91,7 +91,8 @@ type StorageMiner interface {
 	SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error   //perm:admin
 	// SectorRemove removes the sector from storage. It doesn't terminate it on-chain, which can
 	// be done with SectorTerminate. Removing and not terminating live sectors will cause additional penalties.
-	SectorRemove(context.Context, abi.SectorNumber) error //perm:admin
+	SectorRemove(context.Context, abi.SectorNumber) error                           //perm:admin
+	SectorMarkForUpgrade(ctx context.Context, id abi.SectorNumber, snap bool) error //perm:admin
 	// SectorTerminate terminates the sector on-chain (adding it to a termination batch first), then
 	// automatically removes it from storage
 	SectorTerminate(context.Context, abi.SectorNumber) error //perm:admin
@@ -99,8 +100,7 @@ type StorageMiner interface {
 	// Returns null if message wasn't sent
 	SectorTerminateFlush(ctx context.Context) (*cid.Cid, error) //perm:admin
 	// SectorTerminatePending returns a list of pending sector terminations to be sent in the next batch message
-	SectorTerminatePending(ctx context.Context) ([]abi.SectorID, error)  //perm:admin
-	SectorMarkForUpgrade(ctx context.Context, id abi.SectorNumber) error //perm:admin
+	SectorTerminatePending(ctx context.Context) ([]abi.SectorID, error) //perm:admin
 	// SectorPreCommitFlush immediately sends a PreCommit message with sectors batched for PreCommit.
 	// Returns null if message wasn't sent
 	SectorPreCommitFlush(ctx context.Context) ([]sealiface.PreCommitBatchRes, error) //perm:admin
@@ -111,6 +111,9 @@ type StorageMiner interface {
 	SectorCommitFlush(ctx context.Context) ([]sealiface.CommitBatchRes, error) //perm:admin
 	// SectorCommitPending returns a list of pending Commit sectors to be sent in the next aggregate message
 	SectorCommitPending(ctx context.Context) ([]abi.SectorID, error) //perm:admin
+	SectorMatchPendingPiecesToOpenSectors(ctx context.Context) error //perm:admin
+	// SectorAbortUpgrade can be called on sectors that are in the process of being upgraded to abort it
+	SectorAbortUpgrade(context.Context, abi.SectorNumber) error //perm:admin

 	// WorkerConnect tells the node to connect to workers RPC
 	WorkerConnect(context.Context, string) error                              //perm:admin retry:true
@@ -118,36 +121,42 @@ type StorageMiner interface {
 	WorkerJobs(context.Context) (map[uuid.UUID][]storiface.WorkerJob, error)  //perm:admin

 	//storiface.WorkerReturn
-	ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error                //perm:admin retry:true
-	ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storage.PreCommit1Out, err *storiface.CallError) error //perm:admin retry:true
-	ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storage.SectorCids, err *storiface.CallError) error //perm:admin retry:true
-	ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storage.Commit1Out, err *storiface.CallError) error       //perm:admin retry:true
-	ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storage.Proof, err *storiface.CallError) error          //perm:admin retry:true
-	ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                            //perm:admin retry:true
-	ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                           //perm:admin retry:true
-	ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                               //perm:admin retry:true
-	ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                               //perm:admin retry:true
-	ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err *storiface.CallError) error                        //perm:admin retry:true
-	ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                     //perm:admin retry:true
+	ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error                                      //perm:admin retry:true
+	ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storage.PreCommit1Out, err *storiface.CallError) error                       //perm:admin retry:true
+	ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storage.SectorCids, err *storiface.CallError) error                       //perm:admin retry:true
+	ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storage.Commit1Out, err *storiface.CallError) error                             //perm:admin retry:true
+	ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storage.Proof, err *storiface.CallError) error                                //perm:admin retry:true
+	ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                  //perm:admin retry:true
+	ReturnReplicaUpdate(ctx context.Context, callID storiface.CallID, out storage.ReplicaUpdateOut, err *storiface.CallError) error                     //perm:admin retry:true
+	ReturnProveReplicaUpdate1(ctx context.Context, callID storiface.CallID, vanillaProofs storage.ReplicaVanillaProofs, err *storiface.CallError) error //perm:admin retry:true
+	ReturnProveReplicaUpdate2(ctx context.Context, callID storiface.CallID, proof storage.ReplicaUpdateProof, err *storiface.CallError) error           //perm:admin retry:true
+	ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                       //perm:admin retry:true
+	ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                           //perm:admin retry:true
+	ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                 //perm:admin retry:true
+	ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                     //perm:admin retry:true
+	ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                     //perm:admin retry:true
+	ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err *storiface.CallError) error                                              //perm:admin retry:true
+	ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                           //perm:admin retry:true

 	// SealingSchedDiag dumps internal sealing scheduler state
 	SealingSchedDiag(ctx context.Context, doSched bool) (interface{}, error) //perm:admin
 	SealingAbort(ctx context.Context, call storiface.CallID) error           //perm:admin

-	//stores.SectorIndex
-	StorageAttach(context.Context, stores.StorageInfo, fsutil.FsStat) error                                                                                             //perm:admin
-	StorageInfo(context.Context, stores.ID) (stores.StorageInfo, error)                                                                                                 //perm:admin
-	StorageReportHealth(context.Context, stores.ID, stores.HealthReport) error                                                                                          //perm:admin
-	StorageDeclareSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error                                     //perm:admin
-	StorageDropSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType) error                                                      //perm:admin
-	StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]stores.SectorStorageInfo, error) //perm:admin
-	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]stores.StorageInfo, error)           //perm:admin
-	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                                          //perm:admin
-	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                               //perm:admin
-	StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error)                                                                                               //perm:admin
+	// SectorIndex
+	StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error                                                                                             //perm:admin
+	StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error)                                                                                              //perm:admin
+	StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error                                                                                       //perm:admin
+	StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error                                     //perm:admin
+	StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error                                                      //perm:admin
+	StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error) //perm:admin
+	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error)           //perm:admin
+	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                                             //perm:admin
+	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                                  //perm:admin
+	StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error)                                                                                            //perm:admin
+	StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error)                                                                                                    //perm:admin

-	StorageLocal(ctx context.Context) (map[stores.ID]string, error)       //perm:admin
-	StorageStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) //perm:admin
+	StorageLocal(ctx context.Context) (map[storiface.ID]string, error)       //perm:admin
+	StorageStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error) //perm:admin

 	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error                                                                                                        //perm:write
 	MarketListDeals(ctx context.Context) ([]MarketDeal, error)                                                                                                                           //perm:read
@@ -160,12 +169,15 @@ type StorageMiner interface {
 	MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error)                                                                                                             //perm:read
 	MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)                                                                                                          //perm:write
 	MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)                                                                                                   //perm:write
+	// MarketDataTransferDiagnostics generates debugging information about current data transfers over graphsync
+	MarketDataTransferDiagnostics(ctx context.Context, p peer.ID) (*TransferDiagnostics, error) //perm:write
 	// MarketRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer
 	MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
 	// MarketCancelDataTransfer cancels a data transfer with the given transfer ID and other peer
 	MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
 	MarketPendingDeals(ctx context.Context) (PendingDealInfo, error)                                                             //perm:write
 	MarketPublishPendingDeals(ctx context.Context) error                                                                         //perm:admin
+	MarketRetryPublishDeal(ctx context.Context, propcid cid.Cid) error                                                           //perm:admin

 	// DagstoreListShards returns information about all shards known to the
 	// DAG store. Only available on nodes running the markets subsystem.
@@ -209,6 +221,16 @@ type StorageMiner interface {
 	// DagstoreGC runs garbage collection on the DAG store.
 	DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin

+	// IndexerAnnounceDeal informs indexer nodes that a new deal was received,
+	// so they can download its index
+	IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin
+
+	// IndexerAnnounceAllDeals informs the indexer nodes aboutall active deals.
+	IndexerAnnounceAllDeals(ctx context.Context) error //perm:admin
+
+	// DagstoreLookupPieces returns information about shards that contain the given CID.
+	DagstoreLookupPieces(ctx context.Context, cid cid.Cid) ([]DagstoreShardInfo, error) //perm:admin
+
 	// RuntimeSubsystems returns the subsystems that are enabled
 	// in this instance.
 	RuntimeSubsystems(ctx context.Context) (MinerSubsystems, error) //perm:read
@@ -245,11 +267,10 @@ type StorageMiner interface {

 	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin

-	ComputeProof(ctx context.Context, ssi []builtin.SectorInfo, rand abi.PoStRandomness) ([]builtin.PoStProof, error) //perm:read
+	ComputeProof(ctx context.Context, ssi []builtin.ExtendedSectorInfo, rand abi.PoStRandomness, poStEpoch abi.ChainEpoch, nv abinetwork.Version) ([]builtin.PoStProof, error) //perm:read
 }

 var _ storiface.WorkerReturn = *new(StorageMiner)
-var _ stores.SectorIndex = *new(StorageMiner)

 type SealRes struct {
 	Err   string
@@ -273,19 +294,20 @@ type SectorPiece struct {
 }

 type SectorInfo struct {
-	SectorID     abi.SectorNumber
-	State        SectorState
-	CommD        *cid.Cid
-	CommR        *cid.Cid
-	Proof        []byte
-	Deals        []abi.DealID
-	Pieces       []SectorPiece
-	Ticket       SealTicket
-	Seed         SealSeed
-	PreCommitMsg *cid.Cid
-	CommitMsg    *cid.Cid
-	Retries      uint64
-	ToUpgrade    bool
+	SectorID             abi.SectorNumber
+	State                SectorState
+	CommD                *cid.Cid
+	CommR                *cid.Cid
+	Proof                []byte
+	Deals                []abi.DealID
+	Pieces               []SectorPiece
+	Ticket               SealTicket
+	Seed                 SealSeed
+	PreCommitMsg         *cid.Cid
+	CommitMsg            *cid.Cid
+	Retries              uint64
+	ToUpgrade            bool
+	ReplicaUpdateMessage *cid.Cid

 	LastErr string

@@ -1,3 +1,4 @@
+//stm: #unit
 package api

 import (
@@ -26,6 +27,7 @@ func goCmd() string {
 }

 func TestDoesntDependOnFFI(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_FFI_DEPENDENCE_001
 	deps, err := exec.Command(goCmd(), "list", "-deps", "github.com/filecoin-project/lotus/api").Output()
 	if err != nil {
 		t.Fatal(err)
@@ -38,6 +40,7 @@ func TestDoesntDependOnFFI(t *testing.T) {
 }

 func TestDoesntDependOnBuild(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_FFI_DEPENDENCE_002
 	deps, err := exec.Command(goCmd(), "list", "-deps", "github.com/filecoin-project/lotus/api").Output()
 	if err != nil {
 		t.Fatal(err)
@@ -50,6 +53,7 @@ func TestDoesntDependOnBuild(t *testing.T) {
 }

 func TestReturnTypes(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_001
 	errType := reflect.TypeOf(new(error)).Elem()
 	bareIface := reflect.TypeOf(new(interface{})).Elem()
 	jmarsh := reflect.TypeOf(new(json.Marshaler)).Elem()
@@ -115,6 +119,7 @@ func TestReturnTypes(t *testing.T) {
 }

 func TestPermTags(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_PERM_TAGS_001
 	_ = PermissionedFullAPI(&FullNodeStruct{})
 	_ = PermissionedStorMinerAPI(&StorageMinerStruct{})
 	_ = PermissionedWorkerAPI(&WorkerStruct{})
@@ -7,8 +7,9 @@ import (
 	"github.com/ipfs/go-cid"

 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-actors/v5/actors/runtime/proof"
+
 	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 	"github.com/filecoin-project/specs-storage/storage"
 )
@@ -29,7 +30,7 @@ type Worker interface {

 	// TaskType -> Weight
 	TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) //perm:admin
-	Paths(context.Context) ([]stores.StoragePath, error)                //perm:admin
+	Paths(context.Context) ([]storiface.StoragePath, error)             //perm:admin
 	Info(context.Context) (storiface.WorkerInfo, error)                 //perm:admin

 	// storiface.WorkerCalls
@@ -39,11 +40,19 @@ type Worker interface {
 	SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error) //perm:admin
 	SealCommit2(ctx context.Context, sector storage.SectorRef, c1o storage.Commit1Out) (storiface.CallID, error)                                                                                         //perm:admin
 	FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                                //perm:admin
+	FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                         //perm:admin
+	ReplicaUpdate(ctx context.Context, sector storage.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error)                                                                                       //perm:admin
+	ProveReplicaUpdate1(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error)                                                              //perm:admin
+	ProveReplicaUpdate2(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storage.ReplicaVanillaProofs) (storiface.CallID, error)                  //perm:admin
+	GenerateSectorKeyFromData(ctx context.Context, sector storage.SectorRef, commD cid.Cid) (storiface.CallID, error)                                                                                    //perm:admin
 	ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) (storiface.CallID, error)                                                                                 //perm:admin
 	MoveStorage(ctx context.Context, sector storage.SectorRef, types storiface.SectorFileType) (storiface.CallID, error)                                                                                 //perm:admin
 	UnsealPiece(context.Context, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error)                                           //perm:admin
 	Fetch(context.Context, storage.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error)                                                             //perm:admin

+	GenerateWinningPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, randomness abi.PoStRandomness) ([]proof.PoStProof, error)                           //perm:admin
+	GenerateWindowPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, partitionIdx int, randomness abi.PoStRandomness) (storiface.WindowPoStResult, error) //perm:admin
+
 	TaskDisable(ctx context.Context, tt sealtasks.TaskType) error //perm:admin
 	TaskEnable(ctx context.Context, tt sealtasks.TaskType) error  //perm:admin

@@ -1,6 +1,7 @@
 package docgen

 import (
+	"encoding/json"
 	"fmt"
 	"go/ast"
 	"go/parser"
@@ -15,7 +16,7 @@ import (
 	"github.com/filecoin-project/go-bitfield"
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-filestore"
+	"github.com/ipfs/go-graphsync"
 	"github.com/libp2p/go-libp2p-core/metrics"
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
@@ -24,7 +25,7 @@ import (
 	"github.com/multiformats/go-multiaddr"

 	datatransfer "github.com/filecoin-project/go-data-transfer"
-	filestore2 "github.com/filecoin-project/go-fil-markets/filestore"
+	filestore "github.com/filecoin-project/go-fil-markets/filestore"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-jsonrpc/auth"
 	textselector "github.com/ipld/go-ipld-selector-text-lite"
@@ -39,7 +40,6 @@ import (
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
@@ -92,6 +92,8 @@ func init() {

 	storeIDExample := imports.ID(50)
 	textSelExample := textselector.Expression("Links/21/Hash/Links/42/Hash")
+	apiSelExample := api.Selector("Links/21/Hash/Links/42/Hash")
+	clientEvent := retrievalmarket.ClientEventDealAccepted

 	addExample(bitfield.NewFromSet([]uint64{5}))
 	addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1)
@@ -110,7 +112,6 @@ func init() {
 	addExample(abi.UnpaddedPieceSize(1024))
 	addExample(abi.UnpaddedPieceSize(1024).Padded())
 	addExample(abi.DealID(5432))
-	addExample(filestore.StatusFileChanged)
 	addExample(abi.SectorNumber(9))
 	addExample(abi.SectorSize(32 * 1024 * 1024 * 1024))
 	addExample(api.MpoolChange(0))
@@ -120,13 +121,17 @@ func init() {
 	addExample(api.FullAPIVersion1)
 	addExample(api.PCHInbound)
 	addExample(time.Minute)
+	addExample(graphsync.NewRequestID())
 	addExample(datatransfer.TransferID(3))
 	addExample(datatransfer.Ongoing)
 	addExample(storeIDExample)
 	addExample(&storeIDExample)
+	addExample(clientEvent)
+	addExample(&clientEvent)
 	addExample(retrievalmarket.ClientEventDealAccepted)
 	addExample(retrievalmarket.DealStatusNew)
 	addExample(&textSelExample)
+	addExample(&apiSelExample)
 	addExample(network.ReachabilityPublic)
 	addExample(build.NewestNetworkVersion)
 	addExample(map[string]int{"name": 42})
@@ -178,7 +183,7 @@ func init() {
 	ExampleValues[reflect.TypeOf(struct{ A multiaddr.Multiaddr }{}).Field(0).Type] = maddr

 	// miner specific
-	addExample(filestore2.Path(".lotusminer/fstmp123"))
+	addExample(filestore.Path(".lotusminer/fstmp123"))
 	si := uint64(12)
 	addExample(&si)
 	addExample(retrievalmarket.DealID(5))
@@ -193,10 +198,10 @@ func init() {
 		},
 	})
 	addExample(api.SectorState(sealing.Proving))
-	addExample(stores.ID("76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8"))
+	addExample(storiface.ID("76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8"))
 	addExample(storiface.FTUnsealed)
 	addExample(storiface.PathSealing)
-	addExample(map[stores.ID][]stores.Decl{
+	addExample(map[storiface.ID][]storiface.Decl{
 		"76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8": {
 			{
 				SectorID:       abi.SectorID{Miner: 1000, Number: 100},
@@ -204,7 +209,7 @@ func init() {
 			},
 		},
 	})
-	addExample(map[stores.ID]string{
+	addExample(map[storiface.ID]string{
 		"76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8": "/data/path",
 	})
 	addExample(map[uuid.UUID][]storiface.WorkerJob{
@@ -228,16 +233,18 @@ func init() {
 				Hostname: "host",
 				Resources: storiface.WorkerResources{
 					MemPhysical: 256 << 30,
+					MemUsed:     2 << 30,
 					MemSwap:     120 << 30,
-					MemReserved: 2 << 30,
+					MemSwapUsed: 2 << 30,
 					CPUs:        64,
 					GPUs:        []string{"aGPU 1337"},
+					Resources:   storiface.ResourceTable,
 				},
 			},
 			Enabled:    true,
 			MemUsedMin: 0,
 			MemUsedMax: 0,
-			GpuUsed:    false,
+			GpuUsed:    0,
 			CpuUse:     0,
 		},
 	})
@@ -245,10 +252,18 @@ func init() {
 	addExample(map[abi.SectorNumber]string{
 		123: "can't acquire read lock",
 	})
+	addExample(json.RawMessage(`"json raw message"`))
 	addExample(map[api.SectorState]int{
 		api.SectorState(sealing.Proving): 120,
 	})
 	addExample([]abi.SectorNumber{123, 124})
+	addExample([]storiface.SectorLock{
+		{
+			Sector: abi.SectorID{Number: 123, Miner: 1000},
+			Write:  [storiface.FileTypes]uint{0, 0, 1},
+			Read:   [storiface.FileTypes]uint{2, 3, 0},
+		},
+	})

 	// worker specific
 	addExample(storiface.AcquireMove)
@@ -283,6 +298,35 @@ func init() {
 		State: "ShardStateAvailable",
 		Error: "<error>",
 	})
+	addExample(storiface.ResourceTable)
+	addExample(network.ScopeStat{
+		Memory:             123,
+		NumStreamsInbound:  1,
+		NumStreamsOutbound: 2,
+		NumConnsInbound:    3,
+		NumConnsOutbound:   4,
+		NumFD:              5,
+	})
+	addExample(map[string]network.ScopeStat{
+		"abc": {
+			Memory:             123,
+			NumStreamsInbound:  1,
+			NumStreamsOutbound: 2,
+			NumConnsInbound:    3,
+			NumConnsOutbound:   4,
+			NumFD:              5,
+		}})
+	addExample(api.NetLimit{
+		Memory:          123,
+		StreamsInbound:  1,
+		StreamsOutbound: 2,
+		Streams:         3,
+		ConnsInbound:    3,
+		ConnsOutbound:   4,
+		Conns:           4,
+		FD:              5,
+	})
+
 }

 func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) {
@@ -333,7 +377,7 @@ func ExampleValue(method string, t, parent reflect.Type) interface{} {
 	switch t.Kind() {
 	case reflect.Slice:
 		out := reflect.New(t).Elem()
-		reflect.Append(out, reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
+		out = reflect.Append(out, reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
 		return out.Interface()
 	case reflect.Chan:
 		return ExampleValue(method, t.Elem(), nil)
@@ -8,6 +8,7 @@ import (
 	context "context"
 	json "encoding/json"
 	reflect "reflect"
+	time "time"

 	address "github.com/filecoin-project/go-address"
 	bitfield "github.com/filecoin-project/go-bitfield"
@@ -25,7 +26,6 @@ import (
 	miner "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	types "github.com/filecoin-project/lotus/chain/types"
 	alerting "github.com/filecoin-project/lotus/journal/alerting"
-	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 	dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
 	imports "github.com/filecoin-project/lotus/node/repo/imports"
 	miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
@@ -537,6 +537,20 @@ func (mr *MockFullNodeMockRecorder) ClientDealSize(arg0, arg1 interface{}) *gomo
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientDealSize", reflect.TypeOf((*MockFullNode)(nil).ClientDealSize), arg0, arg1)
 }

+// ClientExport mocks base method.
+func (m *MockFullNode) ClientExport(arg0 context.Context, arg1 api.ExportRef, arg2 api.FileRef) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ClientExport", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ClientExport indicates an expected call of ClientExport.
+func (mr *MockFullNodeMockRecorder) ClientExport(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientExport", reflect.TypeOf((*MockFullNode)(nil).ClientExport), arg0, arg1, arg2)
+}
+
 // ClientFindData mocks base method.
 func (m *MockFullNode) ClientFindData(arg0 context.Context, arg1 cid.Cid, arg2 *cid.Cid) ([]api.QueryOffer, error) {
 	m.ctrl.T.Helper()
@@ -775,17 +789,18 @@ func (mr *MockFullNodeMockRecorder) ClientRestartDataTransfer(arg0, arg1, arg2,
 }

 // ClientRetrieve mocks base method.
-func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) error {
+func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 api.RetrievalOrder) (*api.RestrievalRes, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "ClientRetrieve", arg0, arg1, arg2)
-	ret0, _ := ret[0].(error)
-	return ret0
+	ret := m.ctrl.Call(m, "ClientRetrieve", arg0, arg1)
+	ret0, _ := ret[0].(*api.RestrievalRes)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
 }

 // ClientRetrieve indicates an expected call of ClientRetrieve.
-func (mr *MockFullNodeMockRecorder) ClientRetrieve(arg0, arg1, arg2 interface{}) *gomock.Call {
+func (mr *MockFullNodeMockRecorder) ClientRetrieve(arg0, arg1 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieve", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieve), arg0, arg1, arg2)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieve", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieve), arg0, arg1)
 }

 // ClientRetrieveTryRestartInsufficientFunds mocks base method.
@@ -802,19 +817,18 @@ func (mr *MockFullNodeMockRecorder) ClientRetrieveTryRestartInsufficientFunds(ar
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieveTryRestartInsufficientFunds", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieveTryRestartInsufficientFunds), arg0, arg1)
 }

-// ClientRetrieveWithEvents mocks base method.
-func (m *MockFullNode) ClientRetrieveWithEvents(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
+// ClientRetrieveWait mocks base method.
+func (m *MockFullNode) ClientRetrieveWait(arg0 context.Context, arg1 retrievalmarket.DealID) error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "ClientRetrieveWithEvents", arg0, arg1, arg2)
-	ret0, _ := ret[0].(<-chan marketevents.RetrievalEvent)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
+	ret := m.ctrl.Call(m, "ClientRetrieveWait", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
 }

-// ClientRetrieveWithEvents indicates an expected call of ClientRetrieveWithEvents.
-func (mr *MockFullNodeMockRecorder) ClientRetrieveWithEvents(arg0, arg1, arg2 interface{}) *gomock.Call {
+// ClientRetrieveWait indicates an expected call of ClientRetrieveWait.
+func (mr *MockFullNodeMockRecorder) ClientRetrieveWait(arg0, arg1 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieveWithEvents", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieveWithEvents), arg0, arg1, arg2)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieveWait", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieveWait), arg0, arg1)
 }

 // ClientStartDeal mocks base method.
@@ -1428,18 +1442,33 @@ func (mr *MockFullNodeMockRecorder) MsigApproveTxnHash(arg0, arg1, arg2, arg3, a
 }

 // MsigCancel mocks base method.
-func (m *MockFullNode) MsigCancel(arg0 context.Context, arg1 address.Address, arg2 uint64, arg3 address.Address, arg4 big.Int, arg5 address.Address, arg6 uint64, arg7 []byte) (*api.MessagePrototype, error) {
+func (m *MockFullNode) MsigCancel(arg0 context.Context, arg1 address.Address, arg2 uint64, arg3 address.Address) (*api.MessagePrototype, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "MsigCancel", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7)
+	ret := m.ctrl.Call(m, "MsigCancel", arg0, arg1, arg2, arg3)
 	ret0, _ := ret[0].(*api.MessagePrototype)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }

 // MsigCancel indicates an expected call of MsigCancel.
-func (mr *MockFullNodeMockRecorder) MsigCancel(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 interface{}) *gomock.Call {
+func (mr *MockFullNodeMockRecorder) MsigCancel(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MsigCancel", reflect.TypeOf((*MockFullNode)(nil).MsigCancel), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MsigCancel", reflect.TypeOf((*MockFullNode)(nil).MsigCancel), arg0, arg1, arg2, arg3)
+}
+
+// MsigCancelTxnHash mocks base method.
+func (m *MockFullNode) MsigCancelTxnHash(arg0 context.Context, arg1 address.Address, arg2 uint64, arg3 address.Address, arg4 big.Int, arg5 address.Address, arg6 uint64, arg7 []byte) (*api.MessagePrototype, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "MsigCancelTxnHash", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7)
+	ret0, _ := ret[0].(*api.MessagePrototype)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// MsigCancelTxnHash indicates an expected call of MsigCancelTxnHash.
+func (mr *MockFullNodeMockRecorder) MsigCancelTxnHash(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MsigCancelTxnHash", reflect.TypeOf((*MockFullNode)(nil).MsigCancelTxnHash), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7)
 }

 // MsigCreate mocks base method.
@@ -1783,6 +1812,21 @@ func (mr *MockFullNodeMockRecorder) NetFindPeer(arg0, arg1 interface{}) *gomock.
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetFindPeer", reflect.TypeOf((*MockFullNode)(nil).NetFindPeer), arg0, arg1)
 }

+// NetLimit mocks base method.
+func (m *MockFullNode) NetLimit(arg0 context.Context, arg1 string) (api.NetLimit, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetLimit", arg0, arg1)
+	ret0, _ := ret[0].(api.NetLimit)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetLimit indicates an expected call of NetLimit.
+func (mr *MockFullNodeMockRecorder) NetLimit(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetLimit", reflect.TypeOf((*MockFullNode)(nil).NetLimit), arg0, arg1)
+}
+
 // NetPeerInfo mocks base method.
 func (m *MockFullNode) NetPeerInfo(arg0 context.Context, arg1 peer.ID) (*api.ExtendedPeerInfo, error) {
 	m.ctrl.T.Helper()
@@ -1813,6 +1857,64 @@ func (mr *MockFullNodeMockRecorder) NetPeers(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPeers", reflect.TypeOf((*MockFullNode)(nil).NetPeers), arg0)
 }

+// NetPing mocks base method.
+func (m *MockFullNode) NetPing(arg0 context.Context, arg1 peer.ID) (time.Duration, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetPing", arg0, arg1)
+	ret0, _ := ret[0].(time.Duration)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetPing indicates an expected call of NetPing.
+func (mr *MockFullNodeMockRecorder) NetPing(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPing", reflect.TypeOf((*MockFullNode)(nil).NetPing), arg0, arg1)
+}
+
+// NetProtectAdd mocks base method.
+func (m *MockFullNode) NetProtectAdd(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectAdd", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectAdd indicates an expected call of NetProtectAdd.
+func (mr *MockFullNodeMockRecorder) NetProtectAdd(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectAdd", reflect.TypeOf((*MockFullNode)(nil).NetProtectAdd), arg0, arg1)
+}
+
+// NetProtectList mocks base method.
+func (m *MockFullNode) NetProtectList(arg0 context.Context) ([]peer.ID, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectList", arg0)
+	ret0, _ := ret[0].([]peer.ID)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetProtectList indicates an expected call of NetProtectList.
+func (mr *MockFullNodeMockRecorder) NetProtectList(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectList", reflect.TypeOf((*MockFullNode)(nil).NetProtectList), arg0)
+}
+
+// NetProtectRemove mocks base method.
+func (m *MockFullNode) NetProtectRemove(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectRemove", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectRemove indicates an expected call of NetProtectRemove.
+func (mr *MockFullNodeMockRecorder) NetProtectRemove(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectRemove", reflect.TypeOf((*MockFullNode)(nil).NetProtectRemove), arg0, arg1)
+}
+
 // NetPubsubScores mocks base method.
 func (m *MockFullNode) NetPubsubScores(arg0 context.Context) ([]api.PubsubScore, error) {
 	m.ctrl.T.Helper()
@@ -1828,6 +1930,35 @@ func (mr *MockFullNodeMockRecorder) NetPubsubScores(arg0 interface{}) *gomock.Ca
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPubsubScores", reflect.TypeOf((*MockFullNode)(nil).NetPubsubScores), arg0)
 }

+// NetSetLimit mocks base method.
+func (m *MockFullNode) NetSetLimit(arg0 context.Context, arg1 string, arg2 api.NetLimit) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetSetLimit", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetSetLimit indicates an expected call of NetSetLimit.
+func (mr *MockFullNodeMockRecorder) NetSetLimit(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetSetLimit", reflect.TypeOf((*MockFullNode)(nil).NetSetLimit), arg0, arg1, arg2)
+}
+
+// NetStat mocks base method.
+func (m *MockFullNode) NetStat(arg0 context.Context, arg1 string) (api.NetStat, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetStat", arg0, arg1)
+	ret0, _ := ret[0].(api.NetStat)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetStat indicates an expected call of NetStat.
+func (mr *MockFullNodeMockRecorder) NetStat(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetStat", reflect.TypeOf((*MockFullNode)(nil).NetStat), arg0, arg1)
+}
+
 // NodeStatus mocks base method.
 func (m *MockFullNode) NodeStatus(arg0 context.Context, arg1 bool) (api.NodeStatus, error) {
 	m.ctrl.T.Helper()
@@ -1903,19 +2034,34 @@ func (mr *MockFullNodeMockRecorder) PaychCollect(arg0, arg1 interface{}) *gomock
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychCollect", reflect.TypeOf((*MockFullNode)(nil).PaychCollect), arg0, arg1)
 }

-// PaychGet mocks base method.
-func (m *MockFullNode) PaychGet(arg0 context.Context, arg1, arg2 address.Address, arg3 big.Int) (*api.ChannelInfo, error) {
+// PaychFund mocks base method.
+func (m *MockFullNode) PaychFund(arg0 context.Context, arg1, arg2 address.Address, arg3 big.Int) (*api.ChannelInfo, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "PaychGet", arg0, arg1, arg2, arg3)
+	ret := m.ctrl.Call(m, "PaychFund", arg0, arg1, arg2, arg3)
+	ret0, _ := ret[0].(*api.ChannelInfo)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// PaychFund indicates an expected call of PaychFund.
+func (mr *MockFullNodeMockRecorder) PaychFund(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychFund", reflect.TypeOf((*MockFullNode)(nil).PaychFund), arg0, arg1, arg2, arg3)
+}
+
+// PaychGet mocks base method.
+func (m *MockFullNode) PaychGet(arg0 context.Context, arg1, arg2 address.Address, arg3 big.Int, arg4 api.PaychGetOpts) (*api.ChannelInfo, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "PaychGet", arg0, arg1, arg2, arg3, arg4)
 	ret0, _ := ret[0].(*api.ChannelInfo)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }

 // PaychGet indicates an expected call of PaychGet.
-func (mr *MockFullNodeMockRecorder) PaychGet(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+func (mr *MockFullNodeMockRecorder) PaychGet(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychGet", reflect.TypeOf((*MockFullNode)(nil).PaychGet), arg0, arg1, arg2, arg3)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychGet", reflect.TypeOf((*MockFullNode)(nil).PaychGet), arg0, arg1, arg2, arg3, arg4)
 }

 // PaychGetWaitReady mocks base method.
@@ -1,3 +1,4 @@
+//stm: #unit
 package api

 import (
@@ -29,6 +30,7 @@ type StrC struct {
 }

 func TestGetInternalStructs(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_API_STRUCTS_001
 	var proxy StrA

 	sts := GetInternalStructs(&proxy)
@@ -44,6 +46,7 @@ func TestGetInternalStructs(t *testing.T) {
 }

 func TestNestedInternalStructs(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_API_STRUCTS_001
 	var proxy StrC

 	// check that only the top-level internal struct gets picked up
@@ -5,12 +5,14 @@ import (
 	"fmt"
 	"time"

-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/libp2p/go-libp2p-core/network"

 	datatransfer "github.com/filecoin-project/go-data-transfer"
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-graphsync"

 	"github.com/libp2p/go-libp2p-core/peer"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
@@ -54,6 +56,30 @@ type MessageSendSpec struct {
 	MaxFee abi.TokenAmount
 }

+// GraphSyncDataTransfer provides diagnostics on a data transfer happening over graphsync
+type GraphSyncDataTransfer struct {
+	// GraphSync request id for this transfer
+	RequestID *graphsync.RequestID
+	// Graphsync state for this transfer
+	RequestState string
+	// If a channel ID is present, indicates whether this is the current graphsync request for this channel
+	// (could have changed in a restart)
+	IsCurrentChannelRequest bool
+	// Data transfer channel ID for this transfer
+	ChannelID *datatransfer.ChannelID
+	// Data transfer state for this transfer
+	ChannelState *DataTransferChannel
+	// Diagnostic information about this request -- and unexpected inconsistencies in
+	// request state
+	Diagnostics []string
+}
+
+// TransferDiagnostics give current information about transfers going over graphsync that may be helpful for debugging
+type TransferDiagnostics struct {
+	ReceivingTransfers []*GraphSyncDataTransfer
+	SendingTransfers   []*GraphSyncDataTransfer
+}
+
 type DataTransferChannel struct {
 	TransferID  datatransfer.TransferID
 	Status      datatransfer.Status
@@ -99,6 +125,28 @@ func NewDataTransferChannel(hostID peer.ID, channelState datatransfer.ChannelSta
 	return channel
 }

+type NetStat struct {
+	System    *network.ScopeStat           `json:",omitempty"`
+	Transient *network.ScopeStat           `json:",omitempty"`
+	Services  map[string]network.ScopeStat `json:",omitempty"`
+	Protocols map[string]network.ScopeStat `json:",omitempty"`
+	Peers     map[string]network.ScopeStat `json:",omitempty"`
+}
+
+type NetLimit struct {
+	Dynamic bool `json:",omitempty"`
+	// set if Dynamic is false
+	Memory int64 `json:",omitempty"`
+	// set if Dynamic is true
+	MemoryFraction float64 `json:",omitempty"`
+	MinMemory      int64   `json:",omitempty"`
+	MaxMemory      int64   `json:",omitempty"`
+
+	Streams, StreamsInbound, StreamsOutbound int
+	Conns, ConnsInbound, ConnsOutbound       int
+	FD                                       int
+}
+
 type NetBlockList struct {
 	Peers     []peer.ID
 	IPAddrs   []string
@@ -194,4 +242,47 @@ type RetrievalInfo struct {

 	TransferChannelID *datatransfer.ChannelID
 	DataTransfer      *DataTransferChannel
+
+	// optional event if part of ClientGetRetrievalUpdates
+	Event *retrievalmarket.ClientEvent
+}
+
+type RestrievalRes struct {
+	DealID retrievalmarket.DealID
+}
+
+// Selector specifies ipld selector string
+// - if the string starts with '{', it's interpreted as json selector string
+//   see https://ipld.io/specs/selectors/ and https://ipld.io/specs/selectors/fixtures/selector-fixtures-1/
+// - otherwise the string is interpreted as ipld-selector-text-lite (simple ipld path)
+//   see https://github.com/ipld/go-ipld-selector-text-lite
+type Selector string
+
+type DagSpec struct {
+	// DataSelector matches data to be retrieved
+	// - when using textselector, the path specifies subtree
+	// - the matched graph must have a single root
+	DataSelector *Selector
+
+	// ExportMerkleProof is applicable only when exporting to a CAR file via a path textselector
+	// When true, in addition to the selection target, the resulting CAR will contain every block along the
+	// path back to, and including the original root
+	// When false the resulting CAR contains only the blocks of the target subdag
+	ExportMerkleProof bool
+}
+
+type ExportRef struct {
+	Root cid.Cid
+
+	// DAGs array specifies a list of DAGs to export
+	// - If exporting into unixfs files, only one DAG is supported, DataSelector is only used to find the targeted root node
+	// - If exporting into a car file
+	//   - When exactly one text-path DataSelector is specified exports the subgraph and its full merkle-path from the original root
+	//   - Otherwise ( multiple paths and/or JSON selector specs) determines each individual subroot and exports the subtrees as a multi-root car
+	// - When not specified defaults to a single DAG:
+	//   - Data - the entire DAG: `{"R":{"l":{"none":{}},":>":{"a":{">":{"@":{}}}}}}`
+	DAGs []DagSpec
+
+	FromLocalCAR string // if specified, get data from a local CARv2 file.
+	DealID       retrievalmarket.DealID
 }
@@ -12,6 +12,7 @@ import (
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/dline"
 	"github.com/ipfs/go-cid"
+	textselector "github.com/ipld/go-ipld-selector-text-lite"
 	"github.com/libp2p/go-libp2p-core/peer"

 	"github.com/filecoin-project/lotus/api"
@@ -325,10 +326,10 @@ type FullNode interface {
 	// ClientMinerQueryOffer returns a QueryOffer for the specific miner and file.
 	ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error) //perm:read
 	// ClientRetrieve initiates the retrieval of a file, as specified in the order.
-	ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error //perm:admin
+	ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *api.FileRef) error //perm:admin
 	// ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel
 	// of status updates.
-	ClientRetrieveWithEvents(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin
+	ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin
 	// ClientQueryAsk returns a signed StorageAsk from the specified miner.
 	// ClientListRetrievals returns information about retrievals made by the local client
 	ClientListRetrievals(ctx context.Context) ([]api.RetrievalInfo, error) //perm:write
@@ -714,3 +715,37 @@ type FullNode interface {
 	// the path specified when calling CreateBackup is within the base path
 	CreateBackup(ctx context.Context, fpath string) error //perm:admin
 }
+
+func OfferOrder(o api.QueryOffer, client address.Address) RetrievalOrder {
+	return RetrievalOrder{
+		Root:                    o.Root,
+		Piece:                   o.Piece,
+		Size:                    o.Size,
+		Total:                   o.MinPrice,
+		UnsealPrice:             o.UnsealPrice,
+		PaymentInterval:         o.PaymentInterval,
+		PaymentIntervalIncrease: o.PaymentIntervalIncrease,
+		Client:                  client,
+
+		Miner:     o.Miner,
+		MinerPeer: &o.MinerPeer,
+	}
+}
+
+type RetrievalOrder struct {
+	// TODO: make this less unixfs specific
+	Root                  cid.Cid
+	Piece                 *cid.Cid
+	DatamodelPathSelector *textselector.Expression
+	Size                  uint64
+
+	FromLocalCAR string // if specified, get data from a local CARv2 file.
+	// TODO: support offset
+	Total                   types.BigInt
+	UnsealPrice             types.BigInt
+	PaymentInterval         uint64
+	PaymentIntervalIncrease uint64
+	Client                  address.Address
+	Miner                   address.Address
+	MinerPeer               *retrievalmarket.RetrievalPeer
+}
@@ -8,7 +8,7 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/dline"
-	"github.com/filecoin-project/go-state-types/network"
+	abinetwork "github.com/filecoin-project/go-state-types/network"

 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
@@ -57,7 +57,7 @@ type Gateway interface {
 	StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error)
 	StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
 	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)
-	StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error)
+	StateNetworkVersion(context.Context, types.TipSetKey) (abinetwork.Version, error)
 	StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error)
 	StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
 	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
@@ -13,7 +13,7 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/dline"
-	"github.com/filecoin-project/go-state-types/network"
+	abinetwork "github.com/filecoin-project/go-state-types/network"
 	"github.com/filecoin-project/lotus/api"
 	apitypes "github.com/filecoin-project/lotus/api/types"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
@@ -125,11 +125,11 @@ type FullNodeStruct struct {

 		ClientRestartDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"`

-		ClientRetrieve func(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) error `perm:"admin"`
+		ClientRetrieve func(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) error `perm:"admin"`

 		ClientRetrieveTryRestartInsufficientFunds func(p0 context.Context, p1 address.Address) error `perm:"write"`

-		ClientRetrieveWithEvents func(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) `perm:"admin"`
+		ClientRetrieveWithEvents func(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) `perm:"admin"`

 		ClientStartDeal func(p0 context.Context, p1 *api.StartDealParams) (*cid.Cid, error) `perm:"admin"`

@@ -451,7 +451,7 @@ type GatewayStruct struct {

 		StateMinerProvingDeadline func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*dline.Info, error) ``

-		StateNetworkVersion func(p0 context.Context, p1 types.TipSetKey) (network.Version, error) ``
+		StateNetworkVersion func(p0 context.Context, p1 types.TipSetKey) (abinetwork.Version, error) ``

 		StateSearchMsg func(p0 context.Context, p1 cid.Cid) (*api.MsgLookup, error) ``

@@ -965,14 +965,14 @@ func (s *FullNodeStub) ClientRestartDataTransfer(p0 context.Context, p1 datatran
 	return ErrNotSupported
 }

-func (s *FullNodeStruct) ClientRetrieve(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) error {
+func (s *FullNodeStruct) ClientRetrieve(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) error {
 	if s.Internal.ClientRetrieve == nil {
 		return ErrNotSupported
 	}
 	return s.Internal.ClientRetrieve(p0, p1, p2)
 }

-func (s *FullNodeStub) ClientRetrieve(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) error {
+func (s *FullNodeStub) ClientRetrieve(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) error {
 	return ErrNotSupported
 }

@@ -987,14 +987,14 @@ func (s *FullNodeStub) ClientRetrieveTryRestartInsufficientFunds(p0 context.Cont
 	return ErrNotSupported
 }

-func (s *FullNodeStruct) ClientRetrieveWithEvents(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
+func (s *FullNodeStruct) ClientRetrieveWithEvents(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
 	if s.Internal.ClientRetrieveWithEvents == nil {
 		return nil, ErrNotSupported
 	}
 	return s.Internal.ClientRetrieveWithEvents(p0, p1, p2)
 }

-func (s *FullNodeStub) ClientRetrieveWithEvents(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
+func (s *FullNodeStub) ClientRetrieveWithEvents(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
 	return nil, ErrNotSupported
 }

@@ -2703,15 +2703,15 @@ func (s *GatewayStub) StateMinerProvingDeadline(p0 context.Context, p1 address.A
 	return nil, ErrNotSupported
 }

-func (s *GatewayStruct) StateNetworkVersion(p0 context.Context, p1 types.TipSetKey) (network.Version, error) {
+func (s *GatewayStruct) StateNetworkVersion(p0 context.Context, p1 types.TipSetKey) (abinetwork.Version, error) {
 	if s.Internal.StateNetworkVersion == nil {
-		return *new(network.Version), ErrNotSupported
+		return *new(abinetwork.Version), ErrNotSupported
 	}
 	return s.Internal.StateNetworkVersion(p0, p1)
 }

-func (s *GatewayStub) StateNetworkVersion(p0 context.Context, p1 types.TipSetKey) (network.Version, error) {
-	return *new(network.Version), ErrNotSupported
+func (s *GatewayStub) StateNetworkVersion(p0 context.Context, p1 types.TipSetKey) (abinetwork.Version, error) {
+	return *new(abinetwork.Version), ErrNotSupported
 }

 func (s *GatewayStruct) StateSearchMsg(p0 context.Context, p1 cid.Cid) (*api.MsgLookup, error) {
@@ -7,6 +7,7 @@ package v0mocks
 import (
 	context "context"
 	reflect "reflect"
+	time "time"

 	address "github.com/filecoin-project/go-address"
 	bitfield "github.com/filecoin-project/go-bitfield"
@@ -21,6 +22,7 @@ import (
 	network "github.com/filecoin-project/go-state-types/network"
 	api "github.com/filecoin-project/lotus/api"
 	apitypes "github.com/filecoin-project/lotus/api/types"
+	v0api "github.com/filecoin-project/lotus/api/v0api"
 	miner "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	types "github.com/filecoin-project/lotus/chain/types"
 	alerting "github.com/filecoin-project/lotus/journal/alerting"
@@ -760,7 +762,7 @@ func (mr *MockFullNodeMockRecorder) ClientRestartDataTransfer(arg0, arg1, arg2,
 }

 // ClientRetrieve mocks base method.
-func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) error {
+func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 v0api.RetrievalOrder, arg2 *api.FileRef) error {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "ClientRetrieve", arg0, arg1, arg2)
 	ret0, _ := ret[0].(error)
@@ -788,7 +790,7 @@ func (mr *MockFullNodeMockRecorder) ClientRetrieveTryRestartInsufficientFunds(ar
 }

 // ClientRetrieveWithEvents mocks base method.
-func (m *MockFullNode) ClientRetrieveWithEvents(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
+func (m *MockFullNode) ClientRetrieveWithEvents(arg0 context.Context, arg1 v0api.RetrievalOrder, arg2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "ClientRetrieveWithEvents", arg0, arg1, arg2)
 	ret0, _ := ret[0].(<-chan marketevents.RetrievalEvent)
@@ -1723,6 +1725,21 @@ func (mr *MockFullNodeMockRecorder) NetFindPeer(arg0, arg1 interface{}) *gomock.
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetFindPeer", reflect.TypeOf((*MockFullNode)(nil).NetFindPeer), arg0, arg1)
 }

+// NetLimit mocks base method.
+func (m *MockFullNode) NetLimit(arg0 context.Context, arg1 string) (api.NetLimit, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetLimit", arg0, arg1)
+	ret0, _ := ret[0].(api.NetLimit)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetLimit indicates an expected call of NetLimit.
+func (mr *MockFullNodeMockRecorder) NetLimit(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetLimit", reflect.TypeOf((*MockFullNode)(nil).NetLimit), arg0, arg1)
+}
+
 // NetPeerInfo mocks base method.
 func (m *MockFullNode) NetPeerInfo(arg0 context.Context, arg1 peer.ID) (*api.ExtendedPeerInfo, error) {
 	m.ctrl.T.Helper()
@@ -1753,6 +1770,64 @@ func (mr *MockFullNodeMockRecorder) NetPeers(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPeers", reflect.TypeOf((*MockFullNode)(nil).NetPeers), arg0)
 }

+// NetPing mocks base method.
+func (m *MockFullNode) NetPing(arg0 context.Context, arg1 peer.ID) (time.Duration, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetPing", arg0, arg1)
+	ret0, _ := ret[0].(time.Duration)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetPing indicates an expected call of NetPing.
+func (mr *MockFullNodeMockRecorder) NetPing(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPing", reflect.TypeOf((*MockFullNode)(nil).NetPing), arg0, arg1)
+}
+
+// NetProtectAdd mocks base method.
+func (m *MockFullNode) NetProtectAdd(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectAdd", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectAdd indicates an expected call of NetProtectAdd.
+func (mr *MockFullNodeMockRecorder) NetProtectAdd(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectAdd", reflect.TypeOf((*MockFullNode)(nil).NetProtectAdd), arg0, arg1)
+}
+
+// NetProtectList mocks base method.
+func (m *MockFullNode) NetProtectList(arg0 context.Context) ([]peer.ID, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectList", arg0)
+	ret0, _ := ret[0].([]peer.ID)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetProtectList indicates an expected call of NetProtectList.
+func (mr *MockFullNodeMockRecorder) NetProtectList(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectList", reflect.TypeOf((*MockFullNode)(nil).NetProtectList), arg0)
+}
+
+// NetProtectRemove mocks base method.
+func (m *MockFullNode) NetProtectRemove(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectRemove", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectRemove indicates an expected call of NetProtectRemove.
+func (mr *MockFullNodeMockRecorder) NetProtectRemove(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectRemove", reflect.TypeOf((*MockFullNode)(nil).NetProtectRemove), arg0, arg1)
+}
+
 // NetPubsubScores mocks base method.
 func (m *MockFullNode) NetPubsubScores(arg0 context.Context) ([]api.PubsubScore, error) {
 	m.ctrl.T.Helper()
@@ -1768,6 +1843,35 @@ func (mr *MockFullNodeMockRecorder) NetPubsubScores(arg0 interface{}) *gomock.Ca
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPubsubScores", reflect.TypeOf((*MockFullNode)(nil).NetPubsubScores), arg0)
 }

+// NetSetLimit mocks base method.
+func (m *MockFullNode) NetSetLimit(arg0 context.Context, arg1 string, arg2 api.NetLimit) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetSetLimit", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetSetLimit indicates an expected call of NetSetLimit.
+func (mr *MockFullNodeMockRecorder) NetSetLimit(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetSetLimit", reflect.TypeOf((*MockFullNode)(nil).NetSetLimit), arg0, arg1, arg2)
+}
+
+// NetStat mocks base method.
+func (m *MockFullNode) NetStat(arg0 context.Context, arg1 string) (api.NetStat, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetStat", arg0, arg1)
+	ret0, _ := ret[0].(api.NetStat)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetStat indicates an expected call of NetStat.
+func (mr *MockFullNodeMockRecorder) NetStat(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetStat", reflect.TypeOf((*MockFullNode)(nil).NetStat), arg0, arg1)
+}
+
 // PaychAllocateLane mocks base method.
 func (m *MockFullNode) PaychAllocateLane(arg0 context.Context, arg1 address.Address) (uint64, error) {
 	m.ctrl.T.Helper()
@@ -3,7 +3,10 @@ package v0api
 import (
 	"context"

+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/crypto"
+	marketevents "github.com/filecoin-project/lotus/markets/loggers"

 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -108,7 +111,7 @@ func (w *WrapperV1Full) MsigApproveTxnHash(ctx context.Context, msig address.Add
 }

 func (w *WrapperV1Full) MsigCancel(ctx context.Context, msig address.Address, txID uint64, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) {
-	p, err := w.FullNode.MsigCancel(ctx, msig, txID, to, amt, src, method, params)
+	p, err := w.FullNode.MsigCancelTxnHash(ctx, msig, txID, to, amt, src, method, params)
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
 	}
@@ -194,4 +197,148 @@ func (w *WrapperV1Full) ChainGetRandomnessFromBeacon(ctx context.Context, tsk ty
 	return w.StateGetRandomnessFromBeacon(ctx, personalization, randEpoch, entropy, tsk)
 }

+func (w *WrapperV1Full) ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *api.FileRef) error {
+	events := make(chan marketevents.RetrievalEvent)
+	go w.clientRetrieve(ctx, order, ref, events)
+
+	for {
+		select {
+		case evt, ok := <-events:
+			if !ok { // done successfully
+				return nil
+			}
+
+			if evt.Err != "" {
+				return xerrors.Errorf("retrieval failed: %s", evt.Err)
+			}
+		case <-ctx.Done():
+			return xerrors.Errorf("retrieval timed out")
+		}
+	}
+}
+
+func (w *WrapperV1Full) ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
+	events := make(chan marketevents.RetrievalEvent)
+	go w.clientRetrieve(ctx, order, ref, events)
+	return events, nil
+}
+
+func readSubscribeEvents(ctx context.Context, dealID retrievalmarket.DealID, subscribeEvents <-chan api.RetrievalInfo, events chan marketevents.RetrievalEvent) error {
+	for {
+		var subscribeEvent api.RetrievalInfo
+		var evt retrievalmarket.ClientEvent
+		select {
+		case <-ctx.Done():
+			return xerrors.New("Retrieval Timed Out")
+		case subscribeEvent = <-subscribeEvents:
+			if subscribeEvent.ID != dealID {
+				// we can't check the deal ID ahead of time because:
+				// 1. We need to subscribe before retrieving.
+				// 2. We won't know the deal ID until after retrieving.
+				continue
+			}
+			if subscribeEvent.Event != nil {
+				evt = *subscribeEvent.Event
+			}
+		}
+
+		select {
+		case <-ctx.Done():
+			return xerrors.New("Retrieval Timed Out")
+		case events <- marketevents.RetrievalEvent{
+			Event:         evt,
+			Status:        subscribeEvent.Status,
+			BytesReceived: subscribeEvent.BytesReceived,
+			FundsSpent:    subscribeEvent.TotalPaid,
+		}:
+		}
+
+		switch subscribeEvent.Status {
+		case retrievalmarket.DealStatusCompleted:
+			return nil
+		case retrievalmarket.DealStatusRejected:
+			return xerrors.Errorf("Retrieval Proposal Rejected: %s", subscribeEvent.Message)
+		case
+			retrievalmarket.DealStatusDealNotFound,
+			retrievalmarket.DealStatusErrored:
+			return xerrors.Errorf("Retrieval Error: %s", subscribeEvent.Message)
+		}
+	}
+}
+
+func (w *WrapperV1Full) clientRetrieve(ctx context.Context, order RetrievalOrder, ref *api.FileRef, events chan marketevents.RetrievalEvent) {
+	defer close(events)
+
+	finish := func(e error) {
+		if e != nil {
+			events <- marketevents.RetrievalEvent{Err: e.Error(), FundsSpent: big.Zero()}
+		}
+	}
+
+	var dealID retrievalmarket.DealID
+	if order.FromLocalCAR == "" {
+		// Subscribe to events before retrieving to avoid losing events.
+		subscribeCtx, cancel := context.WithCancel(ctx)
+		defer cancel()
+		retrievalEvents, err := w.ClientGetRetrievalUpdates(subscribeCtx)
+
+		if err != nil {
+			finish(xerrors.Errorf("GetRetrievalUpdates failed: %w", err))
+			return
+		}
+
+		retrievalRes, err := w.FullNode.ClientRetrieve(ctx, api.RetrievalOrder{
+			Root:                    order.Root,
+			Piece:                   order.Piece,
+			Size:                    order.Size,
+			Total:                   order.Total,
+			UnsealPrice:             order.UnsealPrice,
+			PaymentInterval:         order.PaymentInterval,
+			PaymentIntervalIncrease: order.PaymentIntervalIncrease,
+			Client:                  order.Client,
+			Miner:                   order.Miner,
+			MinerPeer:               order.MinerPeer,
+		})
+
+		if err != nil {
+			finish(xerrors.Errorf("Retrieve failed: %w", err))
+			return
+		}
+
+		dealID = retrievalRes.DealID
+
+		err = readSubscribeEvents(ctx, retrievalRes.DealID, retrievalEvents, events)
+		if err != nil {
+			finish(xerrors.Errorf("Retrieve: %w", err))
+			return
+		}
+	}
+
+	// If ref is nil, it only fetches the data into the configured blockstore.
+	if ref == nil {
+		finish(nil)
+		return
+	}
+
+	eref := api.ExportRef{
+		Root:         order.Root,
+		FromLocalCAR: order.FromLocalCAR,
+		DealID:       dealID,
+	}
+
+	if order.DatamodelPathSelector != nil {
+		s := api.Selector(*order.DatamodelPathSelector)
+		eref.DAGs = append(eref.DAGs, api.DagSpec{
+			DataSelector:      &s,
+			ExportMerkleProof: true,
+		})
+	}
+
+	finish(w.ClientExport(ctx, eref, *ref))
+}
+
+func (w *WrapperV1Full) PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*api.ChannelInfo, error) {
+	return w.FullNode.PaychFund(ctx, from, to, amt)
+}
+
 var _ FullNode = &WrapperV1Full{}
@@ -54,11 +54,11 @@ func VersionForType(nodeType NodeType) (Version, error) {

 // semver versions of the rpc api exposed
 var (
-	FullAPIVersion0 = newVer(1, 4, 0)
-	FullAPIVersion1 = newVer(2, 1, 0)
+	FullAPIVersion0 = newVer(1, 5, 0)
+	FullAPIVersion1 = newVer(2, 2, 0)

-	MinerAPIVersion0  = newVer(1, 2, 0)
-	WorkerAPIVersion0 = newVer(1, 1, 0)
+	MinerAPIVersion0  = newVer(1, 5, 0)
+	WorkerAPIVersion0 = newVer(1, 6, 0)
 )

 //nolint:varcheck,deadcode
@@ -25,35 +25,35 @@ func NewAPIBlockstore(cio ChainIO) Blockstore {
 	return Adapt(bs) // return an adapted blockstore.
 }

-func (a *apiBlockstore) DeleteBlock(cid.Cid) error {
+func (a *apiBlockstore) DeleteBlock(context.Context, cid.Cid) error {
 	return xerrors.New("not supported")
 }

-func (a *apiBlockstore) Has(c cid.Cid) (bool, error) {
-	return a.api.ChainHasObj(context.TODO(), c)
+func (a *apiBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
+	return a.api.ChainHasObj(ctx, c)
 }

-func (a *apiBlockstore) Get(c cid.Cid) (blocks.Block, error) {
-	bb, err := a.api.ChainReadObj(context.TODO(), c)
+func (a *apiBlockstore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
+	bb, err := a.api.ChainReadObj(ctx, c)
 	if err != nil {
 		return nil, err
 	}
 	return blocks.NewBlockWithCid(bb, c)
 }

-func (a *apiBlockstore) GetSize(c cid.Cid) (int, error) {
-	bb, err := a.api.ChainReadObj(context.TODO(), c)
+func (a *apiBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
+	bb, err := a.api.ChainReadObj(ctx, c)
 	if err != nil {
 		return 0, err
 	}
 	return len(bb), nil
 }

-func (a *apiBlockstore) Put(blocks.Block) error {
+func (a *apiBlockstore) Put(context.Context, blocks.Block) error {
 	return xerrors.New("not supported")
 }

-func (a *apiBlockstore) PutMany([]blocks.Block) error {
+func (a *apiBlockstore) PutMany(context.Context, []blocks.Block) error {
 	return xerrors.New("not supported")
 }

@@ -0,0 +1,262 @@
+package blockstore
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"golang.org/x/xerrors"
+
+	block "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+)
+
+// autolog is a logger for the autobatching blockstore. It is subscoped from the
+// blockstore logger.
+var autolog = log.Named("auto")
+
+// contains the same set of blocks twice, once as an ordered list for flushing, and as a map for fast access
+type blockBatch struct {
+	blockList []block.Block
+	blockMap  map[cid.Cid]block.Block
+}
+
+type AutobatchBlockstore struct {
+	// TODO: drop if memory consumption is too high
+	addedCids map[cid.Cid]struct{}
+
+	stateLock     sync.Mutex
+	bufferedBatch blockBatch
+
+	flushingBatch blockBatch
+	flushErr      error
+
+	flushCh chan struct{}
+
+	doFlushLock     sync.Mutex
+	flushRetryDelay time.Duration
+	doneCh          chan struct{}
+	shutdown        context.CancelFunc
+
+	backingBs Blockstore
+
+	bufferCapacity int
+	bufferSize     int
+}
+
+func NewAutobatch(ctx context.Context, backingBs Blockstore, bufferCapacity int) *AutobatchBlockstore {
+	ctx, cancel := context.WithCancel(ctx)
+	bs := &AutobatchBlockstore{
+		addedCids:      make(map[cid.Cid]struct{}),
+		backingBs:      backingBs,
+		bufferCapacity: bufferCapacity,
+		flushCh:        make(chan struct{}, 1),
+		doneCh:         make(chan struct{}),
+		// could be made configable
+		flushRetryDelay: time.Millisecond * 100,
+		shutdown:        cancel,
+	}
+
+	bs.bufferedBatch.blockMap = make(map[cid.Cid]block.Block)
+
+	go bs.flushWorker(ctx)
+
+	return bs
+}
+
+func (bs *AutobatchBlockstore) Put(ctx context.Context, blk block.Block) error {
+	bs.stateLock.Lock()
+	defer bs.stateLock.Unlock()
+
+	_, ok := bs.addedCids[blk.Cid()]
+	if !ok {
+		bs.addedCids[blk.Cid()] = struct{}{}
+		bs.bufferedBatch.blockList = append(bs.bufferedBatch.blockList, blk)
+		bs.bufferedBatch.blockMap[blk.Cid()] = blk
+		bs.bufferSize += len(blk.RawData())
+		if bs.bufferSize >= bs.bufferCapacity {
+			// signal that a flush is appropriate, may be ignored
+			select {
+			case bs.flushCh <- struct{}{}:
+			default:
+				// do nothing
+			}
+		}
+	}
+
+	return nil
+}
+
+func (bs *AutobatchBlockstore) flushWorker(ctx context.Context) {
+	defer close(bs.doneCh)
+	for {
+		select {
+		case <-bs.flushCh:
+			// TODO: check if we _should_ actually flush. We could get a spurious wakeup
+			// here.
+			putErr := bs.doFlush(ctx, false)
+			for putErr != nil {
+				select {
+				case <-ctx.Done():
+					return
+				case <-time.After(bs.flushRetryDelay):
+					autolog.Errorf("FLUSH ERRORED: %w, retrying after %v", putErr, bs.flushRetryDelay)
+					putErr = bs.doFlush(ctx, true)
+				}
+			}
+		case <-ctx.Done():
+			// Do one last flush.
+			_ = bs.doFlush(ctx, false)
+			return
+		}
+	}
+}
+
+// caller must NOT hold stateLock
+// set retryOnly to true to only retry a failed flush and not flush anything new.
+func (bs *AutobatchBlockstore) doFlush(ctx context.Context, retryOnly bool) error {
+	bs.doFlushLock.Lock()
+	defer bs.doFlushLock.Unlock()
+
+	// If we failed to flush last time, try flushing again.
+	if bs.flushErr != nil {
+		bs.flushErr = bs.backingBs.PutMany(ctx, bs.flushingBatch.blockList)
+	}
+
+	// If we failed, or we're _only_ retrying, bail.
+	if retryOnly || bs.flushErr != nil {
+		return bs.flushErr
+	}
+
+	// Then take the current batch...
+	bs.stateLock.Lock()
+	// We do NOT clear addedCids here, because its purpose is to expedite Puts
+	bs.flushingBatch = bs.bufferedBatch
+	bs.bufferedBatch.blockList = make([]block.Block, 0, len(bs.flushingBatch.blockList))
+	bs.bufferedBatch.blockMap = make(map[cid.Cid]block.Block, len(bs.flushingBatch.blockMap))
+	bs.stateLock.Unlock()
+
+	// And try to flush it.
+	bs.flushErr = bs.backingBs.PutMany(ctx, bs.flushingBatch.blockList)
+
+	// If we succeeded, reset the batch. Otherwise, we'll try again next time.
+	if bs.flushErr == nil {
+		bs.stateLock.Lock()
+		bs.flushingBatch = blockBatch{}
+		bs.stateLock.Unlock()
+	}
+
+	return bs.flushErr
+}
+
+// caller must NOT hold stateLock
+func (bs *AutobatchBlockstore) Flush(ctx context.Context) error {
+	return bs.doFlush(ctx, false)
+}
+
+func (bs *AutobatchBlockstore) Shutdown(ctx context.Context) error {
+	// TODO: Prevent puts after we call this to avoid losing data.
+	bs.shutdown()
+	select {
+	case <-bs.doneCh:
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+
+	bs.doFlushLock.Lock()
+	defer bs.doFlushLock.Unlock()
+
+	return bs.flushErr
+}
+
+func (bs *AutobatchBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block, error) {
+	// may seem backward to check the backingBs first, but that is the likeliest case
+	blk, err := bs.backingBs.Get(ctx, c)
+	if err == nil {
+		return blk, nil
+	}
+
+	if err != ErrNotFound {
+		return blk, err
+	}
+
+	bs.stateLock.Lock()
+	defer bs.stateLock.Unlock()
+	v, ok := bs.flushingBatch.blockMap[c]
+	if ok {
+		return v, nil
+	}
+
+	v, ok = bs.bufferedBatch.blockMap[c]
+	if ok {
+		return v, nil
+	}
+
+	return bs.Get(ctx, c)
+}
+
+func (bs *AutobatchBlockstore) DeleteBlock(context.Context, cid.Cid) error {
+	// if we wanted to support this, we would have to:
+	// - flush
+	// - delete from the backingBs (if present)
+	// - remove from addedCids (if present)
+	// - if present in addedCids, also walk the ordered lists and remove if present
+	return xerrors.New("deletion is unsupported")
+}
+
+func (bs *AutobatchBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
+	// see note in DeleteBlock()
+	return xerrors.New("deletion is unsupported")
+}
+
+func (bs *AutobatchBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
+	_, err := bs.Get(ctx, c)
+	if err == nil {
+		return true, nil
+	}
+	if err == ErrNotFound {
+		return false, nil
+	}
+
+	return false, err
+}
+
+func (bs *AutobatchBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
+	blk, err := bs.Get(ctx, c)
+	if err != nil {
+		return 0, err
+	}
+
+	return len(blk.RawData()), nil
+}
+
+func (bs *AutobatchBlockstore) PutMany(ctx context.Context, blks []block.Block) error {
+	for _, blk := range blks {
+		if err := bs.Put(ctx, blk); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (bs *AutobatchBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+	if err := bs.Flush(ctx); err != nil {
+		return nil, err
+	}
+
+	return bs.backingBs.AllKeysChan(ctx)
+}
+
+func (bs *AutobatchBlockstore) HashOnRead(enabled bool) {
+	bs.backingBs.HashOnRead(enabled)
+}
+
+func (bs *AutobatchBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error {
+	blk, err := bs.Get(ctx, cid)
+	if err != nil {
+		return err
+	}
+
+	return callback(blk.RawData())
+}
@@ -0,0 +1,34 @@
+package blockstore
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestAutobatchBlockstore(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	ab := NewAutobatch(ctx, NewMemory(), len(b0.RawData())+len(b1.RawData())-1)
+
+	require.NoError(t, ab.Put(ctx, b0))
+	require.NoError(t, ab.Put(ctx, b1))
+	require.NoError(t, ab.Put(ctx, b2))
+
+	v0, err := ab.Get(ctx, b0.Cid())
+	require.NoError(t, err)
+	require.Equal(t, b0.RawData(), v0.RawData())
+
+	v1, err := ab.Get(ctx, b1.Cid())
+	require.NoError(t, err)
+	require.Equal(t, b1.RawData(), v1.RawData())
+
+	v2, err := ab.Get(ctx, b2.Cid())
+	require.NoError(t, err)
+	require.Equal(t, b2.RawData(), v2.RawData())
+
+	require.NoError(t, ab.Flush(ctx))
+	require.NoError(t, ab.Shutdown(ctx))
+}
@@ -525,7 +525,7 @@ func (b *Blockstore) Size() (int64, error) {

 // View implements blockstore.Viewer, which leverages zero-copy read-only
 // access to values.
-func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
+func (b *Blockstore) View(ctx context.Context, cid cid.Cid, fn func([]byte) error) error {
 	if err := b.access(); err != nil {
 		return err
 	}
@@ -552,7 +552,7 @@ func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
 }

 // Has implements Blockstore.Has.
-func (b *Blockstore) Has(cid cid.Cid) (bool, error) {
+func (b *Blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 	if err := b.access(); err != nil {
 		return false, err
 	}
@@ -582,7 +582,7 @@ func (b *Blockstore) Has(cid cid.Cid) (bool, error) {
 }

 // Get implements Blockstore.Get.
-func (b *Blockstore) Get(cid cid.Cid) (blocks.Block, error) {
+func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
 	if !cid.Defined() {
 		return nil, blockstore.ErrNotFound
 	}
@@ -619,7 +619,7 @@ func (b *Blockstore) Get(cid cid.Cid) (blocks.Block, error) {
 }

 // GetSize implements Blockstore.GetSize.
-func (b *Blockstore) GetSize(cid cid.Cid) (int, error) {
+func (b *Blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
 	if err := b.access(); err != nil {
 		return 0, err
 	}
@@ -652,7 +652,7 @@ func (b *Blockstore) GetSize(cid cid.Cid) (int, error) {
 }

 // Put implements Blockstore.Put.
-func (b *Blockstore) Put(block blocks.Block) error {
+func (b *Blockstore) Put(ctx context.Context, block blocks.Block) error {
 	if err := b.access(); err != nil {
 		return err
 	}
@@ -691,7 +691,7 @@ func (b *Blockstore) Put(block blocks.Block) error {
 }

 // PutMany implements Blockstore.PutMany.
-func (b *Blockstore) PutMany(blocks []blocks.Block) error {
+func (b *Blockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
 	if err := b.access(); err != nil {
 		return err
 	}
@@ -755,7 +755,7 @@ func (b *Blockstore) PutMany(blocks []blocks.Block) error {
 }

 // DeleteBlock implements Blockstore.DeleteBlock.
-func (b *Blockstore) DeleteBlock(cid cid.Cid) error {
+func (b *Blockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
 	if err := b.access(); err != nil {
 		return err
 	}
@@ -774,7 +774,7 @@ func (b *Blockstore) DeleteBlock(cid cid.Cid) error {
 	})
 }

-func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
+func (b *Blockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
 	if err := b.access(); err != nil {
 		return err
 	}
@@ -1,9 +1,10 @@
+//stm: #unit
 package badgerbs

 import (
 	"bytes"
+	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -19,6 +20,8 @@ import (
 )

 func TestBadgerBlockstore(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
 	(&Suite{
 		NewBlockstore:  newBlockstore(DefaultOptions),
 		OpenBlockstore: openBlockstore(DefaultOptions),
@@ -37,6 +40,8 @@ func TestBadgerBlockstore(t *testing.T) {
 }

 func TestStorageKey(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_STORAGE_KEY_001
 	bs, _ := newBlockstore(DefaultOptions)(t)
 	bbs := bs.(*Blockstore)
 	defer bbs.Close() //nolint:errcheck
@@ -72,20 +77,13 @@ func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (
 	return func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
 		tb.Helper()

-		path, err := ioutil.TempDir("", "")
-		if err != nil {
-			tb.Fatal(err)
-		}
+		path = tb.TempDir()

 		db, err := Open(optsSupplier(path))
 		if err != nil {
 			tb.Fatal(err)
 		}

-		tb.Cleanup(func() {
-			_ = os.RemoveAll(path)
-		})
-
 		return db, path
 	}
 }
@@ -98,17 +96,11 @@ func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB,
 }

 func testMove(t *testing.T, optsF func(string) Options) {
-	basePath, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
+	ctx := context.Background()
+	basePath := t.TempDir()

 	dbPath := filepath.Join(basePath, "db")

-	t.Cleanup(func() {
-		_ = os.RemoveAll(basePath)
-	})
-
 	db, err := Open(optsF(dbPath))
 	if err != nil {
 		t.Fatal(err)
@@ -122,7 +114,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
 	// add some blocks
 	for i := 0; i < 10; i++ {
 		blk := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
-		err := db.Put(blk)
+		err := db.Put(ctx, blk)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -132,7 +124,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
 	// delete some of them
 	for i := 5; i < 10; i++ {
 		c := have[i].Cid()
-		err := db.DeleteBlock(c)
+		err := db.DeleteBlock(ctx, c)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -145,7 +137,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
 	g.Go(func() error {
 		for i := 10; i < 1000; i++ {
 			blk := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
-			err := db.Put(blk)
+			err := db.Put(ctx, blk)
 			if err != nil {
 				return err
 			}
@@ -165,7 +157,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
 	// now check that we have all the blocks in have and none in the deleted lists
 	checkBlocks := func() {
 		for _, blk := range have {
-			has, err := db.Has(blk.Cid())
+			has, err := db.Has(ctx, blk.Cid())
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -174,7 +166,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
 				t.Fatal("missing block")
 			}

-			blk2, err := db.Get(blk.Cid())
+			blk2, err := db.Get(ctx, blk.Cid())
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -185,7 +177,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
 		}

 		for _, c := range deleted {
-			has, err := db.Has(c)
+			has, err := db.Has(ctx, c)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -263,10 +255,16 @@ func testMove(t *testing.T, optsF func(string) Options) {
 }

 func TestMoveNoPrefix(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_COLLECT_GARBAGE_001
 	testMove(t, DefaultOptions)
 }

 func TestMoveWithPrefix(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_COLLECT_GARBAGE_001
 	testMove(t, func(path string) Options {
 		opts := DefaultOptions(path)
 		opts.Prefix = "/prefixed/"
@@ -1,3 +1,4 @@
+//stm: #unit
 package badgerbs

 import (
@@ -44,28 +45,38 @@ func (s *Suite) RunTests(t *testing.T, prefix string) {
 }

 func (s *Suite) TestGetWhenKeyNotPresent(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_GET_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
 	}

 	c := cid.NewCidV0(u.Hash([]byte("stuff")))
-	bl, err := bs.Get(c)
+	bl, err := bs.Get(ctx, c)
 	require.Nil(t, bl)
 	require.Equal(t, blockstore.ErrNotFound, err)
 }

 func (s *Suite) TestGetWhenKeyIsNil(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_GET_001
+	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
 	}

-	_, err := bs.Get(cid.Undef)
+	_, err := bs.Get(ctx, cid.Undef)
 	require.Equal(t, blockstore.ErrNotFound, err)
 }

 func (s *Suite) TestPutThenGetBlock(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_GET_001
+	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
@@ -73,15 +84,18 @@ func (s *Suite) TestPutThenGetBlock(t *testing.T) {

 	orig := blocks.NewBlock([]byte("some data"))

-	err := bs.Put(orig)
+	err := bs.Put(ctx, orig)
 	require.NoError(t, err)

-	fetched, err := bs.Get(orig.Cid())
+	fetched, err := bs.Get(ctx, orig.Cid())
 	require.NoError(t, err)
 	require.Equal(t, orig.RawData(), fetched.RawData())
 }

 func (s *Suite) TestHas(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_HAS_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
@@ -89,19 +103,23 @@ func (s *Suite) TestHas(t *testing.T) {

 	orig := blocks.NewBlock([]byte("some data"))

-	err := bs.Put(orig)
+	err := bs.Put(ctx, orig)
 	require.NoError(t, err)

-	ok, err := bs.Has(orig.Cid())
+	ok, err := bs.Has(ctx, orig.Cid())
 	require.NoError(t, err)
 	require.True(t, ok)

-	ok, err = bs.Has(blocks.NewBlock([]byte("another thing")).Cid())
+	ok, err = bs.Has(ctx, blocks.NewBlock([]byte("another thing")).Cid())
 	require.NoError(t, err)
 	require.False(t, ok)
 }

 func (s *Suite) TestCidv0v1(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_GET_001
+	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
@@ -109,15 +127,20 @@ func (s *Suite) TestCidv0v1(t *testing.T) {

 	orig := blocks.NewBlock([]byte("some data"))

-	err := bs.Put(orig)
+	err := bs.Put(ctx, orig)
 	require.NoError(t, err)

-	fetched, err := bs.Get(cid.NewCidV1(cid.DagProtobuf, orig.Cid().Hash()))
+	fetched, err := bs.Get(ctx, cid.NewCidV1(cid.DagProtobuf, orig.Cid().Hash()))
 	require.NoError(t, err)
 	require.Equal(t, orig.RawData(), fetched.RawData())
 }

 func (s *Suite) TestPutThenGetSizeBlock(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_GET_SIZE_001
+	ctx := context.Background()
+
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
@@ -127,26 +150,28 @@ func (s *Suite) TestPutThenGetSizeBlock(t *testing.T) {
 	missingBlock := blocks.NewBlock([]byte("missingBlock"))
 	emptyBlock := blocks.NewBlock([]byte{})

-	err := bs.Put(block)
+	err := bs.Put(ctx, block)
 	require.NoError(t, err)

-	blockSize, err := bs.GetSize(block.Cid())
+	blockSize, err := bs.GetSize(ctx, block.Cid())
 	require.NoError(t, err)
 	require.Len(t, block.RawData(), blockSize)

-	err = bs.Put(emptyBlock)
+	err = bs.Put(ctx, emptyBlock)
 	require.NoError(t, err)

-	emptySize, err := bs.GetSize(emptyBlock.Cid())
+	emptySize, err := bs.GetSize(ctx, emptyBlock.Cid())
 	require.NoError(t, err)
 	require.Zero(t, emptySize)

-	missingSize, err := bs.GetSize(missingBlock.Cid())
+	missingSize, err := bs.GetSize(ctx, missingBlock.Cid())
 	require.Equal(t, blockstore.ErrNotFound, err)
 	require.Equal(t, -1, missingSize)
 }

 func (s *Suite) TestAllKeysSimple(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
@@ -163,6 +188,9 @@ func (s *Suite) TestAllKeysSimple(t *testing.T) {
 }

 func (s *Suite) TestAllKeysRespectsContext(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_ALL_KEYS_CHAN_001
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
@@ -193,6 +221,7 @@ func (s *Suite) TestAllKeysRespectsContext(t *testing.T) {
 }

 func (s *Suite) TestDoubleClose(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
 	bs, _ := s.NewBlockstore(t)
 	c, ok := bs.(io.Closer)
 	if !ok {
@@ -203,6 +232,10 @@ func (s *Suite) TestDoubleClose(t *testing.T) {
 }

 func (s *Suite) TestReopenPutGet(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_GET_001
+	ctx := context.Background()
 	bs, path := s.NewBlockstore(t)
 	c, ok := bs.(io.Closer)
 	if !ok {
@@ -210,7 +243,7 @@ func (s *Suite) TestReopenPutGet(t *testing.T) {
 	}

 	orig := blocks.NewBlock([]byte("some data"))
-	err := bs.Put(orig)
+	err := bs.Put(ctx, orig)
 	require.NoError(t, err)

 	err = c.Close()
@@ -219,7 +252,7 @@ func (s *Suite) TestReopenPutGet(t *testing.T) {
 	bs, err = s.OpenBlockstore(t, path)
 	require.NoError(t, err)

-	fetched, err := bs.Get(orig.Cid())
+	fetched, err := bs.Get(ctx, orig.Cid())
 	require.NoError(t, err)
 	require.Equal(t, orig.RawData(), fetched.RawData())

@@ -228,6 +261,11 @@ func (s *Suite) TestReopenPutGet(t *testing.T) {
 }

 func (s *Suite) TestPutMany(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_HAS_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_GET_001, @SPLITSTORE_BADGER_PUT_MANY_001
+	//stm: @SPLITSTORE_BADGER_ALL_KEYS_CHAN_001
+	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
@@ -238,15 +276,15 @@ func (s *Suite) TestPutMany(t *testing.T) {
 		blocks.NewBlock([]byte("foo2")),
 		blocks.NewBlock([]byte("foo3")),
 	}
-	err := bs.PutMany(blks)
+	err := bs.PutMany(ctx, blks)
 	require.NoError(t, err)

 	for _, blk := range blks {
-		fetched, err := bs.Get(blk.Cid())
+		fetched, err := bs.Get(ctx, blk.Cid())
 		require.NoError(t, err)
 		require.Equal(t, blk.RawData(), fetched.RawData())

-		ok, err := bs.Has(blk.Cid())
+		ok, err := bs.Has(ctx, blk.Cid())
 		require.NoError(t, err)
 		require.True(t, ok)
 	}
@@ -259,6 +297,12 @@ func (s *Suite) TestPutMany(t *testing.T) {
 }

 func (s *Suite) TestDelete(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_POOLED_STORAGE_HAS_001
+	//stm: @SPLITSTORE_BADGER_ALL_KEYS_CHAN_001, @SPLITSTORE_BADGER_HAS_001
+	//stm: @SPLITSTORE_BADGER_PUT_MANY_001
+
+	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
@@ -269,10 +313,10 @@ func (s *Suite) TestDelete(t *testing.T) {
 		blocks.NewBlock([]byte("foo2")),
 		blocks.NewBlock([]byte("foo3")),
 	}
-	err := bs.PutMany(blks)
+	err := bs.PutMany(ctx, blks)
 	require.NoError(t, err)

-	err = bs.DeleteBlock(blks[1].Cid())
+	err = bs.DeleteBlock(ctx, blks[1].Cid())
 	require.NoError(t, err)

 	ch, err := bs.AllKeysChan(context.Background())
@@ -285,17 +329,17 @@ func (s *Suite) TestDelete(t *testing.T) {
 		cid.NewCidV1(cid.Raw, blks[2].Cid().Hash()),
 	})

-	has, err := bs.Has(blks[1].Cid())
+	has, err := bs.Has(ctx, blks[1].Cid())
 	require.NoError(t, err)
 	require.False(t, has)
-
 }

 func insertBlocks(t *testing.T, bs blockstore.BasicBlockstore, count int) []cid.Cid {
+	ctx := context.Background()
 	keys := make([]cid.Cid, count)
 	for i := 0; i < count; i++ {
 		block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
-		err := bs.Put(block)
+		err := bs.Put(ctx, block)
 		require.NoError(t, err)
 		// NewBlock assigns a CIDv0; we convert it to CIDv1 because that's what
 		// the store returns.
@@ -1,6 +1,8 @@
 package blockstore

 import (
+	"context"
+
 	cid "github.com/ipfs/go-cid"
 	ds "github.com/ipfs/go-datastore"
 	logging "github.com/ipfs/go-log/v2"
@@ -27,7 +29,7 @@ type BasicBlockstore = blockstore.Blockstore
 type Viewer = blockstore.Viewer

 type BatchDeleter interface {
-	DeleteMany(cids []cid.Cid) error
+	DeleteMany(ctx context.Context, cids []cid.Cid) error
 }

 // BlockstoreIterator is a trait for efficient iteration
@@ -93,17 +95,17 @@ type adaptedBlockstore struct {

 var _ Blockstore = (*adaptedBlockstore)(nil)

-func (a *adaptedBlockstore) View(cid cid.Cid, callback func([]byte) error) error {
-	blk, err := a.Get(cid)
+func (a *adaptedBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error {
+	blk, err := a.Get(ctx, cid)
 	if err != nil {
 		return err
 	}
 	return callback(blk.RawData())
 }

-func (a *adaptedBlockstore) DeleteMany(cids []cid.Cid) error {
+func (a *adaptedBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
 	for _, cid := range cids {
-		err := a.DeleteBlock(cid)
+		err := a.DeleteBlock(ctx, cid)
 		if err != nil {
 			return err
 		}
@@ -88,34 +88,34 @@ func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid,
 	return out, nil
 }

-func (bs *BufferedBlockstore) DeleteBlock(c cid.Cid) error {
-	if err := bs.read.DeleteBlock(c); err != nil {
+func (bs *BufferedBlockstore) DeleteBlock(ctx context.Context, c cid.Cid) error {
+	if err := bs.read.DeleteBlock(ctx, c); err != nil {
 		return err
 	}

-	return bs.write.DeleteBlock(c)
+	return bs.write.DeleteBlock(ctx, c)
 }

-func (bs *BufferedBlockstore) DeleteMany(cids []cid.Cid) error {
-	if err := bs.read.DeleteMany(cids); err != nil {
+func (bs *BufferedBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
+	if err := bs.read.DeleteMany(ctx, cids); err != nil {
 		return err
 	}

-	return bs.write.DeleteMany(cids)
+	return bs.write.DeleteMany(ctx, cids)
 }

-func (bs *BufferedBlockstore) View(c cid.Cid, callback func([]byte) error) error {
+func (bs *BufferedBlockstore) View(ctx context.Context, c cid.Cid, callback func([]byte) error) error {
 	// both stores are viewable.
-	if err := bs.write.View(c, callback); err == ErrNotFound {
+	if err := bs.write.View(ctx, c, callback); err == ErrNotFound {
 		// not found in write blockstore; fall through.
 	} else {
 		return err // propagate errors, or nil, i.e. found.
 	}
-	return bs.read.View(c, callback)
+	return bs.read.View(ctx, c, callback)
 }

-func (bs *BufferedBlockstore) Get(c cid.Cid) (block.Block, error) {
-	if out, err := bs.write.Get(c); err != nil {
+func (bs *BufferedBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block, error) {
+	if out, err := bs.write.Get(ctx, c); err != nil {
 		if err != ErrNotFound {
 			return nil, err
 		}
@@ -123,20 +123,20 @@ func (bs *BufferedBlockstore) Get(c cid.Cid) (block.Block, error) {
 		return out, nil
 	}

-	return bs.read.Get(c)
+	return bs.read.Get(ctx, c)
 }

-func (bs *BufferedBlockstore) GetSize(c cid.Cid) (int, error) {
-	s, err := bs.read.GetSize(c)
+func (bs *BufferedBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
+	s, err := bs.read.GetSize(ctx, c)
 	if err == ErrNotFound || s == 0 {
-		return bs.write.GetSize(c)
+		return bs.write.GetSize(ctx, c)
 	}

 	return s, err
 }

-func (bs *BufferedBlockstore) Put(blk block.Block) error {
-	has, err := bs.read.Has(blk.Cid()) // TODO: consider dropping this check
+func (bs *BufferedBlockstore) Put(ctx context.Context, blk block.Block) error {
+	has, err := bs.read.Has(ctx, blk.Cid()) // TODO: consider dropping this check
 	if err != nil {
 		return err
 	}
@@ -145,11 +145,11 @@ func (bs *BufferedBlockstore) Put(blk block.Block) error {
 		return nil
 	}

-	return bs.write.Put(blk)
+	return bs.write.Put(ctx, blk)
 }

-func (bs *BufferedBlockstore) Has(c cid.Cid) (bool, error) {
-	has, err := bs.write.Has(c)
+func (bs *BufferedBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
+	has, err := bs.write.Has(ctx, c)
 	if err != nil {
 		return false, err
 	}
@@ -157,7 +157,7 @@ func (bs *BufferedBlockstore) Has(c cid.Cid) (bool, error) {
 		return true, nil
 	}

-	return bs.read.Has(c)
+	return bs.read.Has(ctx, c)
 }

 func (bs *BufferedBlockstore) HashOnRead(hor bool) {
@@ -165,8 +165,8 @@ func (bs *BufferedBlockstore) HashOnRead(hor bool) {
 	bs.write.HashOnRead(hor)
 }

-func (bs *BufferedBlockstore) PutMany(blks []block.Block) error {
-	return bs.write.PutMany(blks)
+func (bs *BufferedBlockstore) PutMany(ctx context.Context, blks []block.Block) error {
+	return bs.write.PutMany(ctx, blks)
 }

 func (bs *BufferedBlockstore) Read() Blockstore {
@@ -0,0 +1,21 @@
+package blockstore
+
+import (
+	"context"
+)
+
+type hotViewKey struct{}
+
+var hotView = hotViewKey{}
+
+// WithHotView constructs a new context with an option that provides a hint to the blockstore
+// (e.g. the splitstore) that the object (and its ipld references) should be kept hot.
+func WithHotView(ctx context.Context) context.Context {
+	return context.WithValue(ctx, hotView, struct{}{})
+}
+
+// IsHotView returns true if the hot view option is set in the context
+func IsHotView(ctx context.Context) bool {
+	v := ctx.Value(hotView)
+	return v != nil
+}
@@ -18,39 +18,39 @@ func NewDiscardStore(bs Blockstore) Blockstore {
 	return &discardstore{bs: bs}
 }

-func (b *discardstore) Has(cid cid.Cid) (bool, error) {
-	return b.bs.Has(cid)
+func (b *discardstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
+	return b.bs.Has(ctx, cid)
 }

 func (b *discardstore) HashOnRead(hor bool) {
 	b.bs.HashOnRead(hor)
 }

-func (b *discardstore) Get(cid cid.Cid) (blocks.Block, error) {
-	return b.bs.Get(cid)
+func (b *discardstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
+	return b.bs.Get(ctx, cid)
 }

-func (b *discardstore) GetSize(cid cid.Cid) (int, error) {
-	return b.bs.GetSize(cid)
+func (b *discardstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
+	return b.bs.GetSize(ctx, cid)
 }

-func (b *discardstore) View(cid cid.Cid, f func([]byte) error) error {
-	return b.bs.View(cid, f)
+func (b *discardstore) View(ctx context.Context, cid cid.Cid, f func([]byte) error) error {
+	return b.bs.View(ctx, cid, f)
 }

-func (b *discardstore) Put(blk blocks.Block) error {
+func (b *discardstore) Put(ctx context.Context, blk blocks.Block) error {
 	return nil
 }

-func (b *discardstore) PutMany(blks []blocks.Block) error {
+func (b *discardstore) PutMany(ctx context.Context, blks []blocks.Block) error {
 	return nil
 }

-func (b *discardstore) DeleteBlock(cid cid.Cid) error {
+func (b *discardstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
 	return nil
 }

-func (b *discardstore) DeleteMany(cids []cid.Cid) error {
+func (b *discardstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
 	return nil
 }

@@ -71,14 +71,14 @@ func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
 	// chain bitswap puts blocks in temp blockstore which is cleaned up
 	// every few min (to drop any messages we fetched but don't want)
 	// in this case we want to keep this block around
-	if err := fbs.Put(b); err != nil {
+	if err := fbs.Put(ctx, b); err != nil {
 		return nil, xerrors.Errorf("persisting fallback-fetched block: %w", err)
 	}
 	return b, nil
 }

-func (fbs *FallbackStore) Get(c cid.Cid) (blocks.Block, error) {
-	b, err := fbs.Blockstore.Get(c)
+func (fbs *FallbackStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
+	b, err := fbs.Blockstore.Get(ctx, c)
 	switch err {
 	case nil:
 		return b, nil
@@ -89,8 +89,8 @@ func (fbs *FallbackStore) Get(c cid.Cid) (blocks.Block, error) {
 	}
 }

-func (fbs *FallbackStore) GetSize(c cid.Cid) (int, error) {
-	sz, err := fbs.Blockstore.GetSize(c)
+func (fbs *FallbackStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
+	sz, err := fbs.Blockstore.GetSize(ctx, c)
 	switch err {
 	case nil:
 		return sz, nil
@@ -38,7 +38,7 @@ func decodeCid(cid cid.Cid) (inline bool, data []byte, err error) {
 	return false, nil, err
 }

-func (b *idstore) Has(cid cid.Cid) (bool, error) {
+func (b *idstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 	inline, _, err := decodeCid(cid)
 	if err != nil {
 		return false, xerrors.Errorf("error decoding Cid: %w", err)
@@ -48,10 +48,10 @@ func (b *idstore) Has(cid cid.Cid) (bool, error) {
 		return true, nil
 	}

-	return b.bs.Has(cid)
+	return b.bs.Has(ctx, cid)
 }

-func (b *idstore) Get(cid cid.Cid) (blocks.Block, error) {
+func (b *idstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
 	inline, data, err := decodeCid(cid)
 	if err != nil {
 		return nil, xerrors.Errorf("error decoding Cid: %w", err)
@@ -61,10 +61,10 @@ func (b *idstore) Get(cid cid.Cid) (blocks.Block, error) {
 		return blocks.NewBlockWithCid(data, cid)
 	}

-	return b.bs.Get(cid)
+	return b.bs.Get(ctx, cid)
 }

-func (b *idstore) GetSize(cid cid.Cid) (int, error) {
+func (b *idstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
 	inline, data, err := decodeCid(cid)
 	if err != nil {
 		return 0, xerrors.Errorf("error decoding Cid: %w", err)
@@ -74,10 +74,10 @@ func (b *idstore) GetSize(cid cid.Cid) (int, error) {
 		return len(data), err
 	}

-	return b.bs.GetSize(cid)
+	return b.bs.GetSize(ctx, cid)
 }

-func (b *idstore) View(cid cid.Cid, cb func([]byte) error) error {
+func (b *idstore) View(ctx context.Context, cid cid.Cid, cb func([]byte) error) error {
 	inline, data, err := decodeCid(cid)
 	if err != nil {
 		return xerrors.Errorf("error decoding Cid: %w", err)
@@ -87,10 +87,10 @@ func (b *idstore) View(cid cid.Cid, cb func([]byte) error) error {
 		return cb(data)
 	}

-	return b.bs.View(cid, cb)
+	return b.bs.View(ctx, cid, cb)
 }

-func (b *idstore) Put(blk blocks.Block) error {
+func (b *idstore) Put(ctx context.Context, blk blocks.Block) error {
 	inline, _, err := decodeCid(blk.Cid())
 	if err != nil {
 		return xerrors.Errorf("error decoding Cid: %w", err)
@@ -100,10 +100,10 @@ func (b *idstore) Put(blk blocks.Block) error {
 		return nil
 	}

-	return b.bs.Put(blk)
+	return b.bs.Put(ctx, blk)
 }

-func (b *idstore) PutMany(blks []blocks.Block) error {
+func (b *idstore) PutMany(ctx context.Context, blks []blocks.Block) error {
 	toPut := make([]blocks.Block, 0, len(blks))
 	for _, blk := range blks {
 		inline, _, err := decodeCid(blk.Cid())
@@ -118,13 +118,13 @@ func (b *idstore) PutMany(blks []blocks.Block) error {
 	}

 	if len(toPut) > 0 {
-		return b.bs.PutMany(toPut)
+		return b.bs.PutMany(ctx, toPut)
 	}

 	return nil
 }

-func (b *idstore) DeleteBlock(cid cid.Cid) error {
+func (b *idstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
 	inline, _, err := decodeCid(cid)
 	if err != nil {
 		return xerrors.Errorf("error decoding Cid: %w", err)
@@ -134,10 +134,10 @@ func (b *idstore) DeleteBlock(cid cid.Cid) error {
 		return nil
 	}

-	return b.bs.DeleteBlock(cid)
+	return b.bs.DeleteBlock(ctx, cid)
 }

-func (b *idstore) DeleteMany(cids []cid.Cid) error {
+func (b *idstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
 	toDelete := make([]cid.Cid, 0, len(cids))
 	for _, cid := range cids {
 		inline, _, err := decodeCid(cid)
@@ -152,7 +152,7 @@ func (b *idstore) DeleteMany(cids []cid.Cid) error {
 	}

 	if len(toDelete) > 0 {
-		return b.bs.DeleteMany(toDelete)
+		return b.bs.DeleteMany(ctx, toDelete)
 	}

 	return nil
@@ -79,12 +79,12 @@ func NewRemoteIPFSBlockstore(ctx context.Context, maddr multiaddr.Multiaddr, onl
 	return Adapt(bs), nil
 }

-func (i *IPFSBlockstore) DeleteBlock(cid cid.Cid) error {
+func (i *IPFSBlockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
 	return xerrors.Errorf("not supported")
 }

-func (i *IPFSBlockstore) Has(cid cid.Cid) (bool, error) {
-	_, err := i.offlineAPI.Block().Stat(i.ctx, path.IpldPath(cid))
+func (i *IPFSBlockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
+	_, err := i.offlineAPI.Block().Stat(ctx, path.IpldPath(cid))
 	if err != nil {
 		// The underlying client is running in Offline mode.
 		// Stat() will fail with an err if the block isn't in the
@@ -99,8 +99,8 @@ func (i *IPFSBlockstore) Has(cid cid.Cid) (bool, error) {
 	return true, nil
 }

-func (i *IPFSBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
-	rd, err := i.api.Block().Get(i.ctx, path.IpldPath(cid))
+func (i *IPFSBlockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
+	rd, err := i.api.Block().Get(ctx, path.IpldPath(cid))
 	if err != nil {
 		return nil, xerrors.Errorf("getting ipfs block: %w", err)
 	}
@@ -113,8 +113,8 @@ func (i *IPFSBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
 	return blocks.NewBlockWithCid(data, cid)
 }

-func (i *IPFSBlockstore) GetSize(cid cid.Cid) (int, error) {
-	st, err := i.api.Block().Stat(i.ctx, path.IpldPath(cid))
+func (i *IPFSBlockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
+	st, err := i.api.Block().Stat(ctx, path.IpldPath(cid))
 	if err != nil {
 		return 0, xerrors.Errorf("getting ipfs block: %w", err)
 	}
@@ -122,23 +122,23 @@ func (i *IPFSBlockstore) GetSize(cid cid.Cid) (int, error) {
 	return st.Size(), nil
 }

-func (i *IPFSBlockstore) Put(block blocks.Block) error {
+func (i *IPFSBlockstore) Put(ctx context.Context, block blocks.Block) error {
 	mhd, err := multihash.Decode(block.Cid().Hash())
 	if err != nil {
 		return err
 	}

-	_, err = i.api.Block().Put(i.ctx, bytes.NewReader(block.RawData()),
+	_, err = i.api.Block().Put(ctx, bytes.NewReader(block.RawData()),
 		options.Block.Hash(mhd.Code, mhd.Length),
 		options.Block.Format(cid.CodecToStr[block.Cid().Type()]))
 	return err
 }

-func (i *IPFSBlockstore) PutMany(blocks []blocks.Block) error {
+func (i *IPFSBlockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
 	// TODO: could be done in parallel

 	for _, block := range blocks {
-		if err := i.Put(block); err != nil {
+		if err := i.Put(ctx, block); err != nil {
 			return err
 		}
 	}
@@ -15,24 +15,24 @@ func NewMemory() MemBlockstore {
 // MemBlockstore is a terminal blockstore that keeps blocks in memory.
 type MemBlockstore map[cid.Cid]blocks.Block

-func (m MemBlockstore) DeleteBlock(k cid.Cid) error {
+func (m MemBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
 	delete(m, k)
 	return nil
 }

-func (m MemBlockstore) DeleteMany(ks []cid.Cid) error {
+func (m MemBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error {
 	for _, k := range ks {
 		delete(m, k)
 	}
 	return nil
 }

-func (m MemBlockstore) Has(k cid.Cid) (bool, error) {
+func (m MemBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error) {
 	_, ok := m[k]
 	return ok, nil
 }

-func (m MemBlockstore) View(k cid.Cid, callback func([]byte) error) error {
+func (m MemBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error {
 	b, ok := m[k]
 	if !ok {
 		return ErrNotFound
@@ -40,7 +40,7 @@ func (m MemBlockstore) View(k cid.Cid, callback func([]byte) error) error {
 	return callback(b.RawData())
 }

-func (m MemBlockstore) Get(k cid.Cid) (blocks.Block, error) {
+func (m MemBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) {
 	b, ok := m[k]
 	if !ok {
 		return nil, ErrNotFound
@@ -49,7 +49,7 @@ func (m MemBlockstore) Get(k cid.Cid) (blocks.Block, error) {
 }

 // GetSize returns the CIDs mapped BlockSize
-func (m MemBlockstore) GetSize(k cid.Cid) (int, error) {
+func (m MemBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
 	b, ok := m[k]
 	if !ok {
 		return 0, ErrNotFound
@@ -58,7 +58,7 @@ func (m MemBlockstore) GetSize(k cid.Cid) (int, error) {
 }

 // Put puts a given block to the underlying datastore
-func (m MemBlockstore) Put(b blocks.Block) error {
+func (m MemBlockstore) Put(ctx context.Context, b blocks.Block) error {
 	// Convert to a basic block for safety, but try to reuse the existing
 	// block if it's already a basic block.
 	k := b.Cid()
@@ -76,9 +76,9 @@ func (m MemBlockstore) Put(b blocks.Block) error {

 // PutMany puts a slice of blocks at the same time using batching
 // capabilities of the underlying datastore whenever possible.
-func (m MemBlockstore) PutMany(bs []blocks.Block) error {
+func (m MemBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
 	for _, b := range bs {
-		_ = m.Put(b) // can't fail
+		_ = m.Put(ctx, b) // can't fail
 	}
 	return nil
 }
@@ -49,10 +49,11 @@ These are options in the `[Chainstore.Splitstore]` section of the configuration:
  blockstore and discards writes; this is necessary to support syncing from a snapshot.
 - `MarkSetType` -- specifies the type of markset to use during compaction.
  The markset is the data structure used by compaction/gc to track live objects.
-  The default value is `"map"`, which will use an in-memory map; if you are limited
-  in memory (or indeed see compaction run out of memory), you can also specify
-  `"badger"` which will use an disk backed markset, using badger. This will use
-  much less memory, but will also make compaction slower.
+  The default value is "badger", which will use a disk backed markset using badger.
+  If you have a lot of memory (48G or more) you can also use "map", which will use
+  an in memory markset, speeding up compaction at the cost of higher memory usage.
+  Note: If you are using a VPS with a network volume, you need to provision at least
+  3000 IOPs with the badger markset.
 - `HotStoreMessageRetention` -- specifies how many finalities, beyond the 4
  finalities maintained by default, to maintain messages and message receipts in the
  hotstore. This is useful for assistive nodes that want to support syncing for other
@@ -105,6 +106,12 @@ Compaction works transactionally with the following algorithm:
  - We delete in small batches taking a lock; each batch is checked again for marks, from the concurrent transactional mark, so as to never delete anything live
 - We then end the transaction and compact/gc the hotstore.

+As of [#8008](https://github.com/filecoin-project/lotus/pull/8008) the compaction algorithm has been
+modified to eliminate sorting and maintain the cold object set on disk. This drastically reduces
+memory usage; in fact, when using badger as the markset compaction uses very little memory, and
+it should be now possible to run splitstore with 32GB of RAM or less without danger of running out of
+memory during compaction.
+
 ## Garbage Collection

 TBD -- see [#6577](https://github.com/filecoin-project/lotus/issues/6577)
@@ -0,0 +1,118 @@
+package splitstore
+
+import (
+	"bufio"
+	"io"
+	"os"
+
+	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
+	mh "github.com/multiformats/go-multihash"
+)
+
+type Checkpoint struct {
+	file *os.File
+	buf  *bufio.Writer
+}
+
+func NewCheckpoint(path string) (*Checkpoint, error) {
+	file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_SYNC, 0644)
+	if err != nil {
+		return nil, xerrors.Errorf("error creating checkpoint: %w", err)
+	}
+	buf := bufio.NewWriter(file)
+
+	return &Checkpoint{
+		file: file,
+		buf:  buf,
+	}, nil
+}
+
+func OpenCheckpoint(path string) (*Checkpoint, cid.Cid, error) {
+	filein, err := os.Open(path)
+	if err != nil {
+		return nil, cid.Undef, xerrors.Errorf("error opening checkpoint for reading: %w", err)
+	}
+	defer filein.Close() //nolint:errcheck
+
+	bufin := bufio.NewReader(filein)
+	start, err := readRawCid(bufin, nil)
+	if err != nil && err != io.EOF {
+		return nil, cid.Undef, xerrors.Errorf("error reading cid from checkpoint: %w", err)
+	}
+
+	fileout, err := os.OpenFile(path, os.O_WRONLY|os.O_SYNC, 0644)
+	if err != nil {
+		return nil, cid.Undef, xerrors.Errorf("error opening checkpoint for writing: %w", err)
+	}
+	bufout := bufio.NewWriter(fileout)
+
+	return &Checkpoint{
+		file: fileout,
+		buf:  bufout,
+	}, start, nil
+}
+
+func (cp *Checkpoint) Set(c cid.Cid) error {
+	if _, err := cp.file.Seek(0, io.SeekStart); err != nil {
+		return xerrors.Errorf("error seeking beginning of checkpoint: %w", err)
+	}
+
+	if err := writeRawCid(cp.buf, c, true); err != nil {
+		return xerrors.Errorf("error writing cid to checkpoint: %w", err)
+	}
+
+	return nil
+}
+
+func (cp *Checkpoint) Close() error {
+	if cp.file == nil {
+		return nil
+	}
+
+	err := cp.file.Close()
+	cp.file = nil
+	cp.buf = nil
+
+	return err
+}
+
+func readRawCid(buf *bufio.Reader, hbuf []byte) (cid.Cid, error) {
+	sz, err := buf.ReadByte()
+	if err != nil {
+		return cid.Undef, err // don't wrap EOF as it is not an error here
+	}
+
+	if hbuf == nil {
+		hbuf = make([]byte, int(sz))
+	} else {
+		hbuf = hbuf[:int(sz)]
+	}
+
+	if _, err := io.ReadFull(buf, hbuf); err != nil {
+		return cid.Undef, xerrors.Errorf("error reading hash: %w", err) // wrap EOF, it's corrupt
+	}
+
+	hash, err := mh.Cast(hbuf)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("error casting multihash: %w", err)
+	}
+
+	return cid.NewCidV1(cid.Raw, hash), nil
+}
+
+func writeRawCid(buf *bufio.Writer, c cid.Cid, flush bool) error {
+	hash := c.Hash()
+	if err := buf.WriteByte(byte(len(hash))); err != nil {
+		return err
+	}
+	if _, err := buf.Write(hash); err != nil {
+		return err
+	}
+	if flush {
+		return buf.Flush()
+	}
+
+	return nil
+}
@@ -0,0 +1,138 @@
+package splitstore
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+)
+
+func TestCheckpoint(t *testing.T) {
+	dir := t.TempDir()
+
+	path := filepath.Join(dir, "checkpoint")
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	cp, err := NewCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Set(k1); err != nil {
+		t.Fatal(err)
+	}
+	if err := cp.Set(k2); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err := OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !start.Equals(k2) {
+		t.Fatalf("expected start to be %s; got %s", k2, start)
+	}
+
+	if err := cp.Set(k3); err != nil {
+		t.Fatal(err)
+	}
+	if err := cp.Set(k4); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err = OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !start.Equals(k4) {
+		t.Fatalf("expected start to be %s; got %s", k4, start)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// also test correct operation with an empty checkpoint
+	cp, err = NewCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err = OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if start.Defined() {
+		t.Fatal("expected start to be undefined")
+	}
+
+	if err := cp.Set(k1); err != nil {
+		t.Fatal(err)
+	}
+	if err := cp.Set(k2); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err = OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !start.Equals(k2) {
+		t.Fatalf("expected start to be %s; got %s", k2, start)
+	}
+
+	if err := cp.Set(k3); err != nil {
+		t.Fatal(err)
+	}
+	if err := cp.Set(k4); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err = OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !start.Equals(k4) {
+		t.Fatalf("expected start to be %s; got %s", k4, start)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+}
@@ -0,0 +1,102 @@
+package splitstore
+
+import (
+	"bufio"
+	"io"
+	"os"
+
+	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
+)
+
+type ColdSetWriter struct {
+	file *os.File
+	buf  *bufio.Writer
+}
+
+type ColdSetReader struct {
+	file *os.File
+	buf  *bufio.Reader
+}
+
+func NewColdSetWriter(path string) (*ColdSetWriter, error) {
+	file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return nil, xerrors.Errorf("error creating coldset: %w", err)
+	}
+	buf := bufio.NewWriter(file)
+
+	return &ColdSetWriter{
+		file: file,
+		buf:  buf,
+	}, nil
+}
+
+func NewColdSetReader(path string) (*ColdSetReader, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, xerrors.Errorf("error opening coldset: %w", err)
+	}
+	buf := bufio.NewReader(file)
+
+	return &ColdSetReader{
+		file: file,
+		buf:  buf,
+	}, nil
+}
+
+func (s *ColdSetWriter) Write(c cid.Cid) error {
+	return writeRawCid(s.buf, c, false)
+}
+
+func (s *ColdSetWriter) Close() error {
+	if s.file == nil {
+		return nil
+	}
+
+	err1 := s.buf.Flush()
+	err2 := s.file.Close()
+	s.buf = nil
+	s.file = nil
+
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
+
+func (s *ColdSetReader) ForEach(f func(cid.Cid) error) error {
+	hbuf := make([]byte, 256)
+	for {
+		next, err := readRawCid(s.buf, hbuf)
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+
+			return xerrors.Errorf("error reading coldset: %w", err)
+		}
+
+		if err := f(next); err != nil {
+			return err
+		}
+	}
+}
+
+func (s *ColdSetReader) Reset() error {
+	_, err := s.file.Seek(0, io.SeekStart)
+	return err
+}
+
+func (s *ColdSetReader) Close() error {
+	if s.file == nil {
+		return nil
+	}
+
+	err := s.file.Close()
+	s.file = nil
+	s.buf = nil
+
+	return err
+}
@@ -0,0 +1,90 @@
+package splitstore
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+)
+
+func TestColdSet(t *testing.T) {
+	dir := t.TempDir()
+
+	path := filepath.Join(dir, "coldset")
+
+	makeCid := func(i int) cid.Cid {
+		h, err := multihash.Sum([]byte(fmt.Sprintf("cid.%d", i)), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	const count = 1000
+	cids := make([]cid.Cid, 0, count)
+	for i := 0; i < count; i++ {
+		cids = append(cids, makeCid(i))
+	}
+
+	cw, err := NewColdSetWriter(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, c := range cids {
+		if err := cw.Write(c); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if err := cw.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cr, err := NewColdSetReader(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	index := 0
+	err = cr.ForEach(func(c cid.Cid) error {
+		if index >= count {
+			t.Fatal("too many cids")
+		}
+
+		if !c.Equals(cids[index]) {
+			t.Fatalf("wrong cid %d; expected %s but got %s", index, cids[index], c)
+		}
+
+		index++
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cr.Reset(); err != nil {
+		t.Fatal(err)
+	}
+
+	index = 0
+	err = cr.ForEach(func(c cid.Cid) error {
+		if index >= count {
+			t.Fatal("too many cids")
+		}
+
+		if !c.Equals(cids[index]) {
+			t.Fatalf("wrong cid; expected %s but got %s", cids[index], c)
+		}
+
+		index++
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+}
@@ -10,39 +10,36 @@ import (

 var errMarkSetClosed = errors.New("markset closed")

-// MarkSet is a utility to keep track of seen CID, and later query for them.
-//
-// * If the expected dataset is large, it can be backed by a datastore (e.g. bbolt).
-// * If a probabilistic result is acceptable, it can be backed by a bloom filter
+// MarkSet is an interface for tracking CIDs during chain and object walks
 type MarkSet interface {
+	ObjectVisitor
 	Mark(cid.Cid) error
+	MarkMany([]cid.Cid) error
 	Has(cid.Cid) (bool, error)
 	Close() error
-	SetConcurrent()
-}

-type MarkSetVisitor interface {
-	MarkSet
-	ObjectVisitor
+	// BeginCriticalSection ensures that the markset is persisted to disk for recovery in case
+	// of abnormal termination during the critical section span.
+	BeginCriticalSection() error
+	// EndCriticalSection ends the critical section span.
+	EndCriticalSection()
 }

 type MarkSetEnv interface {
-	// Create creates a new markset within the environment.
-	// name is a unique name for this markset, mapped to the filesystem in disk-backed environments
+	// New creates a new markset within the environment.
+	// name is a unique name for this markset, mapped to the filesystem for on-disk persistence.
 	// sizeHint is a hint about the expected size of the markset
-	Create(name string, sizeHint int64) (MarkSet, error)
-	// CreateVisitor is like Create, but returns a wider interface that supports atomic visits.
-	// It may not be supported by some markset types (e.g. bloom).
-	CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error)
-	// SupportsVisitor returns true if the marksets created by this environment support the visitor interface.
-	SupportsVisitor() bool
+	New(name string, sizeHint int64) (MarkSet, error)
+	// Recover recovers an existing markset persisted on-disk.
+	Recover(name string) (MarkSet, error)
+	// Close closes the markset
 	Close() error
 }

 func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) {
 	switch mtype {
 	case "map":
-		return NewMapMarkSetEnv()
+		return NewMapMarkSetEnv(path)
 	case "badger":
 		return NewBadgerMarkSetEnv(path)
 	default:
@@ -3,6 +3,7 @@ package splitstore
 import (
 	"os"
 	"path/filepath"
+	"runtime"
 	"sync"

 	"golang.org/x/xerrors"
@@ -28,13 +29,13 @@ type BadgerMarkSet struct {
 	writers int
 	seqno   int
 	version int
+	persist bool

 	db   *badger.DB
 	path string
 }

 var _ MarkSet = (*BadgerMarkSet)(nil)
-var _ MarkSetVisitor = (*BadgerMarkSet)(nil)

 var badgerMarkSetBatchSize = 16384

@@ -48,11 +49,10 @@ func NewBadgerMarkSetEnv(path string) (MarkSetEnv, error) {
 	return &BadgerMarkSetEnv{path: msPath}, nil
 }

-func (e *BadgerMarkSetEnv) create(name string, sizeHint int64) (*BadgerMarkSet, error) {
-	name += ".tmp"
+func (e *BadgerMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
 	path := filepath.Join(e.path, name)

-	db, err := openTransientBadgerDB(path)
+	db, err := openBadgerDB(path, false)
 	if err != nil {
 		return nil, xerrors.Errorf("error creating badger db: %w", err)
 	}
@@ -68,18 +68,72 @@ func (e *BadgerMarkSetEnv) create(name string, sizeHint int64) (*BadgerMarkSet,
 	return ms, nil
 }

-func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
-	return e.create(name, sizeHint)
-}
+func (e *BadgerMarkSetEnv) Recover(name string) (MarkSet, error) {
+	path := filepath.Join(e.path, name)

-func (e *BadgerMarkSetEnv) CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error) {
-	return e.create(name, sizeHint)
-}
+	if _, err := os.Stat(path); err != nil {
+		return nil, xerrors.Errorf("error stating badger db path: %w", err)
+	}

-func (e *BadgerMarkSetEnv) SupportsVisitor() bool { return true }
+	db, err := openBadgerDB(path, true)
+	if err != nil {
+		return nil, xerrors.Errorf("error creating badger db: %w", err)
+	}
+
+	ms := &BadgerMarkSet{
+		pend:    make(map[string]struct{}),
+		writing: make(map[int]map[string]struct{}),
+		db:      db,
+		path:    path,
+		persist: true,
+	}
+	ms.cond.L = &ms.mx
+
+	return ms, nil
+}

 func (e *BadgerMarkSetEnv) Close() error {
-	return os.RemoveAll(e.path)
+	return nil
+}
+
+func (s *BadgerMarkSet) BeginCriticalSection() error {
+	s.mx.Lock()
+
+	if s.persist {
+		s.mx.Unlock()
+		return nil
+	}
+
+	var write bool
+	var seqno int
+	if len(s.pend) > 0 {
+		write = true
+		seqno = s.nextBatch()
+	}
+
+	s.persist = true
+	s.mx.Unlock()
+
+	if write {
+		// all writes sync once perist is true
+		return s.write(seqno)
+	}
+
+	// wait for any pending writes and sync
+	s.mx.Lock()
+	for s.writers > 0 {
+		s.cond.Wait()
+	}
+	s.mx.Unlock()
+
+	return s.db.Sync()
+}
+
+func (s *BadgerMarkSet) EndCriticalSection() {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	s.persist = false
 }

 func (s *BadgerMarkSet) Mark(c cid.Cid) error {
@@ -99,6 +153,23 @@ func (s *BadgerMarkSet) Mark(c cid.Cid) error {
 	return nil
 }

+func (s *BadgerMarkSet) MarkMany(batch []cid.Cid) error {
+	s.mx.Lock()
+	if s.pend == nil {
+		s.mx.Unlock()
+		return errMarkSetClosed
+	}
+
+	write, seqno := s.putMany(batch)
+	s.mx.Unlock()
+
+	if write {
+		return s.write(seqno)
+	}
+
+	return nil
+}
+
 func (s *BadgerMarkSet) Has(c cid.Cid) (bool, error) {
 	s.mx.RLock()
 	defer s.mx.RUnlock()
@@ -204,16 +275,34 @@ func (s *BadgerMarkSet) tryDB(key []byte) (has bool, err error) {
 // writer holds the exclusive lock
 func (s *BadgerMarkSet) put(key string) (write bool, seqno int) {
 	s.pend[key] = struct{}{}
-	if len(s.pend) < badgerMarkSetBatchSize {
+	if !s.persist && len(s.pend) < badgerMarkSetBatchSize {
 		return false, 0
 	}

-	seqno = s.seqno
+	seqno = s.nextBatch()
+	return true, seqno
+}
+
+func (s *BadgerMarkSet) putMany(batch []cid.Cid) (write bool, seqno int) {
+	for _, c := range batch {
+		key := string(c.Hash())
+		s.pend[key] = struct{}{}
+	}
+
+	if !s.persist && len(s.pend) < badgerMarkSetBatchSize {
+		return false, 0
+	}
+
+	seqno = s.nextBatch()
+	return true, seqno
+}
+
+func (s *BadgerMarkSet) nextBatch() int {
+	seqno := s.seqno
 	s.seqno++
 	s.writing[seqno] = s.pend
 	s.pend = make(map[string]struct{})
-
-	return true, seqno
+	return seqno
 }

 func (s *BadgerMarkSet) write(seqno int) (err error) {
@@ -258,6 +347,14 @@ func (s *BadgerMarkSet) write(seqno int) (err error) {
 		return xerrors.Errorf("error flushing batch to badger markset: %w", err)
 	}

+	s.mx.RLock()
+	persist := s.persist
+	s.mx.RUnlock()
+
+	if persist {
+		return s.db.Sync()
+	}
+
 	return nil
 }

@@ -277,26 +374,29 @@ func (s *BadgerMarkSet) Close() error {
 	db := s.db
 	s.db = nil

-	return closeTransientBadgerDB(db, s.path)
+	return closeBadgerDB(db, s.path, s.persist)
 }

-func (s *BadgerMarkSet) SetConcurrent() {}
+func openBadgerDB(path string, recover bool) (*badger.DB, error) {
+	// if it is not a recovery, clean up first
+	if !recover {
+		err := os.RemoveAll(path)
+		if err != nil {
+			return nil, xerrors.Errorf("error clearing markset directory: %w", err)
+		}

-func openTransientBadgerDB(path string) (*badger.DB, error) {
-	// clean up first
-	err := os.RemoveAll(path)
-	if err != nil {
-		return nil, xerrors.Errorf("error clearing markset directory: %w", err)
-	}
-
-	err = os.MkdirAll(path, 0755) //nolint:gosec
-	if err != nil {
-		return nil, xerrors.Errorf("error creating markset directory: %w", err)
+		err = os.MkdirAll(path, 0755) //nolint:gosec
+		if err != nil {
+			return nil, xerrors.Errorf("error creating markset directory: %w", err)
+		}
 	}

 	opts := badger.DefaultOptions(path)
+	// we manually sync when we are in critical section
 	opts.SyncWrites = false
+	// no need to do that
 	opts.CompactL0OnClose = false
+	// we store hashes, not much to gain by compression
 	opts.Compression = options.None
 	// Note: We use FileIO for loading modes to avoid memory thrashing and interference
 	//       between the system blockstore and the markset.
@@ -305,6 +405,15 @@ func openTransientBadgerDB(path string) (*badger.DB, error) {
 	//       exceeded 1GB in size.
 	opts.TableLoadingMode = options.FileIO
 	opts.ValueLogLoadingMode = options.FileIO
+	// We increase the number of L0 tables before compaction to make it unlikely to
+	// be necessary.
+	opts.NumLevelZeroTables = 20      // default is 5
+	opts.NumLevelZeroTablesStall = 30 // default is 10
+	// increase the number of compactors from default 2 so that if we ever have to
+	// compact, it is fast
+	if runtime.NumCPU()/2 > opts.NumCompactors {
+		opts.NumCompactors = runtime.NumCPU() / 2
+	}
 	opts.Logger = &badgerLogger{
 		SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(),
 		skip2:         log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(),
@@ -313,12 +422,16 @@ func openTransientBadgerDB(path string) (*badger.DB, error) {
 	return badger.Open(opts)
 }

-func closeTransientBadgerDB(db *badger.DB, path string) error {
+func closeBadgerDB(db *badger.DB, path string, persist bool) error {
 	err := db.Close()
 	if err != nil {
 		return xerrors.Errorf("error closing badger markset: %w", err)
 	}

+	if persist {
+		return nil
+	}
+
 	err = os.RemoveAll(path)
 	if err != nil {
 		return xerrors.Errorf("error deleting badger markset: %w", err)
@@ -1,12 +1,20 @@
 package splitstore

 import (
+	"bufio"
+	"io"
+	"os"
+	"path/filepath"
 	"sync"

+	"golang.org/x/xerrors"
+
 	cid "github.com/ipfs/go-cid"
 )

-type MapMarkSetEnv struct{}
+type MapMarkSetEnv struct {
+	path string
+}

 var _ MarkSetEnv = (*MapMarkSetEnv)(nil)

@@ -14,55 +22,194 @@ type MapMarkSet struct {
 	mx  sync.RWMutex
 	set map[string]struct{}

-	ts bool
+	persist bool
+	file    *os.File
+	buf     *bufio.Writer
+
+	path string
 }

 var _ MarkSet = (*MapMarkSet)(nil)
-var _ MarkSetVisitor = (*MapMarkSet)(nil)

-func NewMapMarkSetEnv() (*MapMarkSetEnv, error) {
-	return &MapMarkSetEnv{}, nil
+func NewMapMarkSetEnv(path string) (*MapMarkSetEnv, error) {
+	msPath := filepath.Join(path, "markset.map")
+	err := os.MkdirAll(msPath, 0755) //nolint:gosec
+	if err != nil {
+		return nil, xerrors.Errorf("error creating markset directory: %w", err)
+	}
+
+	return &MapMarkSetEnv{path: msPath}, nil
 }

-func (e *MapMarkSetEnv) create(name string, sizeHint int64) (*MapMarkSet, error) {
+func (e *MapMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
+	path := filepath.Join(e.path, name)
 	return &MapMarkSet{
-		set: make(map[string]struct{}, sizeHint),
+		set:  make(map[string]struct{}, sizeHint),
+		path: path,
 	}, nil
 }

-func (e *MapMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
-	return e.create(name, sizeHint)
-}
+func (e *MapMarkSetEnv) Recover(name string) (MarkSet, error) {
+	path := filepath.Join(e.path, name)
+	s := &MapMarkSet{
+		set:  make(map[string]struct{}),
+		path: path,
+	}

-func (e *MapMarkSetEnv) CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error) {
-	return e.create(name, sizeHint)
-}
+	in, err := os.Open(path)
+	if err != nil {
+		return nil, xerrors.Errorf("error opening markset file for read: %w", err)
+	}
+	defer in.Close() //nolint:errcheck

-func (e *MapMarkSetEnv) SupportsVisitor() bool { return true }
+	// wrap a buffered reader to make this faster
+	buf := bufio.NewReader(in)
+	for {
+		var sz byte
+		if sz, err = buf.ReadByte(); err != nil {
+			break
+		}
+
+		key := make([]byte, int(sz))
+		if _, err = io.ReadFull(buf, key); err != nil {
+			break
+		}
+
+		s.set[string(key)] = struct{}{}
+	}
+
+	if err != io.EOF {
+		return nil, xerrors.Errorf("error reading markset file: %w", err)
+	}
+
+	file, err := os.OpenFile(s.path, os.O_WRONLY|os.O_APPEND, 0)
+	if err != nil {
+		return nil, xerrors.Errorf("error opening markset file for write: %w", err)
+	}
+
+	s.persist = true
+	s.file = file
+	s.buf = bufio.NewWriter(file)
+
+	return s, nil
+}

 func (e *MapMarkSetEnv) Close() error {
 	return nil
 }

-func (s *MapMarkSet) Mark(cid cid.Cid) error {
-	if s.ts {
-		s.mx.Lock()
-		defer s.mx.Unlock()
-	}
+func (s *MapMarkSet) BeginCriticalSection() error {
+	s.mx.Lock()
+	defer s.mx.Unlock()

 	if s.set == nil {
 		return errMarkSetClosed
 	}

-	s.set[string(cid.Hash())] = struct{}{}
+	if s.persist {
+		return nil
+	}
+
+	file, err := os.OpenFile(s.path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return xerrors.Errorf("error opening markset file: %w", err)
+	}
+
+	// wrap a buffered writer to make this faster
+	s.buf = bufio.NewWriter(file)
+	for key := range s.set {
+		if err := s.writeKey([]byte(key), false); err != nil {
+			_ = file.Close()
+			s.buf = nil
+			return err
+		}
+	}
+	if err := s.buf.Flush(); err != nil {
+		_ = file.Close()
+		s.buf = nil
+		return xerrors.Errorf("error flushing markset file buffer: %w", err)
+	}
+
+	s.file = file
+	s.persist = true
+
+	return nil
+}
+
+func (s *MapMarkSet) EndCriticalSection() {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	if !s.persist {
+		return
+	}
+
+	_ = s.file.Close()
+	_ = os.Remove(s.path)
+	s.file = nil
+	s.buf = nil
+	s.persist = false
+}
+
+func (s *MapMarkSet) Mark(c cid.Cid) error {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	if s.set == nil {
+		return errMarkSetClosed
+	}
+
+	hash := c.Hash()
+	s.set[string(hash)] = struct{}{}
+
+	if s.persist {
+		if err := s.writeKey(hash, true); err != nil {
+			return err
+		}
+
+		if err := s.file.Sync(); err != nil {
+			return xerrors.Errorf("error syncing markset: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (s *MapMarkSet) MarkMany(batch []cid.Cid) error {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	if s.set == nil {
+		return errMarkSetClosed
+	}
+
+	for _, c := range batch {
+		hash := c.Hash()
+		s.set[string(hash)] = struct{}{}
+
+		if s.persist {
+			if err := s.writeKey(hash, false); err != nil {
+				return err
+			}
+		}
+	}
+
+	if s.persist {
+		if err := s.buf.Flush(); err != nil {
+			return xerrors.Errorf("error flushing markset buffer to disk: %w", err)
+		}
+
+		if err := s.file.Sync(); err != nil {
+			return xerrors.Errorf("error syncing markset: %w", err)
+		}
+	}
+
 	return nil
 }

 func (s *MapMarkSet) Has(cid cid.Cid) (bool, error) {
-	if s.ts {
-		s.mx.RLock()
-		defer s.mx.RUnlock()
-	}
+	s.mx.RLock()
+	defer s.mx.RUnlock()

 	if s.set == nil {
 		return false, errMarkSetClosed
@@ -73,33 +220,70 @@ func (s *MapMarkSet) Has(cid cid.Cid) (bool, error) {
 }

 func (s *MapMarkSet) Visit(c cid.Cid) (bool, error) {
-	if s.ts {
-		s.mx.Lock()
-		defer s.mx.Unlock()
-	}
+	s.mx.Lock()
+	defer s.mx.Unlock()

 	if s.set == nil {
 		return false, errMarkSetClosed
 	}

-	key := string(c.Hash())
+	hash := c.Hash()
+	key := string(hash)
 	if _, ok := s.set[key]; ok {
 		return false, nil
 	}

 	s.set[key] = struct{}{}
+
+	if s.persist {
+		if err := s.writeKey(hash, true); err != nil {
+			return false, err
+		}
+		if err := s.file.Sync(); err != nil {
+			return false, xerrors.Errorf("error syncing markset: %w", err)
+		}
+	}
+
 	return true, nil
 }

 func (s *MapMarkSet) Close() error {
-	if s.ts {
-		s.mx.Lock()
-		defer s.mx.Unlock()
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	if s.set == nil {
+		return nil
 	}
+
 	s.set = nil
+
+	if s.file != nil {
+		if err := s.file.Close(); err != nil {
+			log.Warnf("error closing markset file: %s", err)
+		}
+
+		if !s.persist {
+			if err := os.Remove(s.path); err != nil {
+				log.Warnf("error removing markset file: %s", err)
+			}
+		}
+	}
+
 	return nil
 }

-func (s *MapMarkSet) SetConcurrent() {
-	s.ts = true
+func (s *MapMarkSet) writeKey(k []byte, flush bool) error {
+	if err := s.buf.WriteByte(byte(len(k))); err != nil {
+		return xerrors.Errorf("error writing markset key length to disk: %w", err)
+	}
+	if _, err := s.buf.Write(k); err != nil {
+		return xerrors.Errorf("error writing markset key to disk: %w", err)
+	}
+	if flush {
+		if err := s.buf.Flush(); err != nil {
+			return xerrors.Errorf("error flushing markset buffer to disk: %w", err)
+		}
+	}
+
+	return nil
 }
@@ -1,8 +1,7 @@
+//stm: #unit
 package splitstore

 import (
-	"io/ioutil"
-	"os"
 	"testing"

 	cid "github.com/ipfs/go-cid"
@@ -10,31 +9,32 @@ import (
 )

 func TestMapMarkSet(t *testing.T) {
+	//stm: @SPLITSTORE_MARKSET_CREATE_001, @SPLITSTORE_MARKSET_HAS_001, @@SPLITSTORE_MARKSET_MARK_001
+	//stm: @SPLITSTORE_MARKSET_CLOSE_001, @SPLITSTORE_MARKSET_CREATE_VISITOR_001
 	testMarkSet(t, "map")
+	testMarkSetRecovery(t, "map")
+	testMarkSetMarkMany(t, "map")
 	testMarkSetVisitor(t, "map")
+	testMarkSetVisitorRecovery(t, "map")
 }

 func TestBadgerMarkSet(t *testing.T) {
+	//stm: @SPLITSTORE_MARKSET_CREATE_001, @SPLITSTORE_MARKSET_HAS_001, @@SPLITSTORE_MARKSET_MARK_001
+	//stm: @SPLITSTORE_MARKSET_CLOSE_001, @SPLITSTORE_MARKSET_CREATE_VISITOR_001
 	bs := badgerMarkSetBatchSize
 	badgerMarkSetBatchSize = 1
 	t.Cleanup(func() {
 		badgerMarkSetBatchSize = bs
 	})
 	testMarkSet(t, "badger")
+	testMarkSetRecovery(t, "badger")
+	testMarkSetMarkMany(t, "badger")
 	testMarkSetVisitor(t, "badger")
+	testMarkSetVisitorRecovery(t, "badger")
 }

 func testMarkSet(t *testing.T, lsType string) {
-	t.Helper()
-
-	path, err := ioutil.TempDir("", "markset.*")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() {
-		_ = os.RemoveAll(path)
-	})
+	path := t.TempDir()

 	env, err := OpenMarkSetEnv(path, lsType)
 	if err != nil {
@@ -42,12 +42,13 @@ func testMarkSet(t *testing.T, lsType string) {
 	}
 	defer env.Close() //nolint:errcheck

-	hotSet, err := env.Create("hot", 0)
+	// stm: @SPLITSTORE_MARKSET_CREATE_001
+	hotSet, err := env.New("hot", 0)
 	if err != nil {
 		t.Fatal(err)
 	}

-	coldSet, err := env.Create("cold", 0)
+	coldSet, err := env.New("cold", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -61,7 +62,9 @@ func testMarkSet(t *testing.T, lsType string) {
 		return cid.NewCidV1(cid.Raw, h)
 	}

+	// stm: @SPLITSTORE_MARKSET_HAS_001
 	mustHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
 		has, err := s.Has(cid)
 		if err != nil {
 			t.Fatal(err)
@@ -73,6 +76,7 @@ func testMarkSet(t *testing.T, lsType string) {
 	}

 	mustNotHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
 		has, err := s.Has(cid)
 		if err != nil {
 			t.Fatal(err)
@@ -88,6 +92,7 @@ func testMarkSet(t *testing.T, lsType string) {
 	k3 := makeCid("c")
 	k4 := makeCid("d")

+	// stm: @SPLITSTORE_MARKSET_MARK_001
 	hotSet.Mark(k1)  //nolint
 	hotSet.Mark(k2)  //nolint
 	coldSet.Mark(k3) //nolint
@@ -114,12 +119,12 @@ func testMarkSet(t *testing.T, lsType string) {
 		t.Fatal(err)
 	}

-	hotSet, err = env.Create("hot", 0)
+	hotSet, err = env.New("hot", 0)
 	if err != nil {
 		t.Fatal(err)
 	}

-	coldSet, err = env.Create("cold", 0)
+	coldSet, err = env.New("cold", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -138,6 +143,7 @@ func testMarkSet(t *testing.T, lsType string) {
 	mustNotHave(coldSet, k3)
 	mustNotHave(coldSet, k4)

+	//stm: @SPLITSTORE_MARKSET_CLOSE_001
 	err = hotSet.Close()
 	if err != nil {
 		t.Fatal(err)
@@ -150,16 +156,7 @@ func testMarkSet(t *testing.T, lsType string) {
 }

 func testMarkSetVisitor(t *testing.T, lsType string) {
-	t.Helper()
-
-	path, err := ioutil.TempDir("", "markset.*")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() {
-		_ = os.RemoveAll(path)
-	})
+	path := t.TempDir()

 	env, err := OpenMarkSetEnv(path, lsType)
 	if err != nil {
@@ -167,7 +164,8 @@ func testMarkSetVisitor(t *testing.T, lsType string) {
 	}
 	defer env.Close() //nolint:errcheck

-	visitor, err := env.CreateVisitor("test", 0)
+	//stm: @SPLITSTORE_MARKSET_CREATE_VISITOR_001
+	visitor, err := env.New("test", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -219,3 +217,301 @@ func testMarkSetVisitor(t *testing.T, lsType string) {
 	mustNotVisit(visitor, k3)
 	mustNotVisit(visitor, k4)
 }
+
+func testMarkSetVisitorRecovery(t *testing.T, lsType string) {
+	path := t.TempDir()
+
+	env, err := OpenMarkSetEnv(path, lsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer env.Close() //nolint:errcheck
+
+	visitor, err := env.New("test", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer visitor.Close() //nolint:errcheck
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	mustVisit := func(v ObjectVisitor, cid cid.Cid) {
+		visit, err := v.Visit(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !visit {
+			t.Fatal("object should be visited")
+		}
+	}
+
+	mustNotVisit := func(v ObjectVisitor, cid cid.Cid) {
+		visit, err := v.Visit(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if visit {
+			t.Fatal("unexpected visit")
+		}
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	mustVisit(visitor, k1)
+	mustVisit(visitor, k2)
+
+	if err := visitor.BeginCriticalSection(); err != nil {
+		t.Fatal(err)
+	}
+
+	mustVisit(visitor, k3)
+	mustVisit(visitor, k4)
+
+	mustNotVisit(visitor, k1)
+	mustNotVisit(visitor, k2)
+	mustNotVisit(visitor, k3)
+	mustNotVisit(visitor, k4)
+
+	if err := visitor.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	visitor, err = env.Recover("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mustNotVisit(visitor, k1)
+	mustNotVisit(visitor, k2)
+	mustNotVisit(visitor, k3)
+	mustNotVisit(visitor, k4)
+
+	visitor.EndCriticalSection()
+
+	if err := visitor.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = env.Recover("test")
+	if err == nil {
+		t.Fatal("expected recovery to fail")
+	}
+}
+
+func testMarkSetRecovery(t *testing.T, lsType string) {
+	path := t.TempDir()
+
+	env, err := OpenMarkSetEnv(path, lsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer env.Close() //nolint:errcheck
+
+	markSet, err := env.New("test", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	mustHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !has {
+			t.Fatal("mark not found")
+		}
+	}
+
+	mustNotHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("unexpected mark")
+		}
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	if err := markSet.Mark(k1); err != nil {
+		t.Fatal(err)
+	}
+	if err := markSet.Mark(k2); err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustNotHave(markSet, k3)
+	mustNotHave(markSet, k4)
+
+	if err := markSet.BeginCriticalSection(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := markSet.Mark(k3); err != nil {
+		t.Fatal(err)
+	}
+	if err := markSet.Mark(k4); err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustHave(markSet, k3)
+	mustHave(markSet, k4)
+
+	if err := markSet.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	markSet, err = env.Recover("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustHave(markSet, k3)
+	mustHave(markSet, k4)
+
+	markSet.EndCriticalSection()
+
+	if err := markSet.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = env.Recover("test")
+	if err == nil {
+		t.Fatal("expected recovery to fail")
+	}
+}
+
+func testMarkSetMarkMany(t *testing.T, lsType string) {
+	path := t.TempDir()
+
+	env, err := OpenMarkSetEnv(path, lsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer env.Close() //nolint:errcheck
+
+	markSet, err := env.New("test", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	mustHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !has {
+			t.Fatal("mark not found")
+		}
+	}
+
+	mustNotHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("unexpected mark")
+		}
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	if err := markSet.MarkMany([]cid.Cid{k1, k2}); err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustNotHave(markSet, k3)
+	mustNotHave(markSet, k4)
+
+	if err := markSet.BeginCriticalSection(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := markSet.MarkMany([]cid.Cid{k3, k4}); err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustHave(markSet, k3)
+	mustHave(markSet, k4)
+
+	if err := markSet.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	markSet, err = env.Recover("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustHave(markSet, k3)
+	mustHave(markSet, k4)
+
+	markSet.EndCriticalSection()
+
+	if err := markSet.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = env.Recover("test")
+	if err == nil {
+		t.Fatal("expected recovery to fail")
+	}
+}
@@ -18,6 +18,8 @@ import (

 	"github.com/filecoin-project/go-state-types/abi"
 	bstore "github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/metrics"

@@ -47,6 +49,9 @@ var (
 	enableDebugLog = false
 	// set this to true if you want to track origin stack traces in the write log
 	enableDebugLogWriteTraces = false
+
+	// upgradeBoundary is the boundary before and after an upgrade where we suppress compaction
+	upgradeBoundary = build.Finality
 )

 func init() {
@@ -98,6 +103,12 @@ type ChainAccessor interface {
 	SubscribeHeadChanges(change func(revert []*types.TipSet, apply []*types.TipSet) error)
 }

+// upgradeRange is a precomputed epoch range during which we shouldn't compact so as to not
+// interfere with an upgrade
+type upgradeRange struct {
+	start, end abi.ChainEpoch
+}
+
 // hotstore is the interface that must be satisfied by the hot blockstore; it is an extension
 // of the Blockstore interface with the traits we need for compaction.
 type hotstore interface {
@@ -118,13 +129,13 @@ type SplitStore struct {

 	headChangeMx sync.Mutex

-	coldPurgeSize int
-
 	chain ChainAccessor
 	ds    dstore.Datastore
 	cold  bstore.Blockstore
 	hot   hotstore

+	upgrades []upgradeRange
+
 	markSetEnv  MarkSetEnv
 	markSetSize int64

@@ -145,6 +156,17 @@ type SplitStore struct {
 	txnRefsMx       sync.Mutex
 	txnRefs         map[cid.Cid]struct{}
 	txnMissing      map[cid.Cid]struct{}
+	txnMarkSet      MarkSet
+	txnSyncMx       sync.Mutex
+	txnSyncCond     sync.Cond
+	txnSync         bool
+
+	// background cold object reification
+	reifyWorkers    sync.WaitGroup
+	reifyMx         sync.Mutex
+	reifyCond       sync.Cond
+	reifyPend       map[cid.Cid]struct{}
+	reifyInProgress map[cid.Cid]struct{}

 	// registered protectors
 	protectors []func(func(cid.Cid) error) error
@@ -173,10 +195,6 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 		return nil, err
 	}

-	if !markSetEnv.SupportsVisitor() {
-		return nil, xerrors.Errorf("markset type does not support atomic visitors")
-	}
-
 	// and now we can make a SplitStore
 	ss := &SplitStore{
 		cfg:        cfg,
@@ -185,13 +203,16 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 		cold:       cold,
 		hot:        hots,
 		markSetEnv: markSetEnv,
-
-		coldPurgeSize: defaultColdPurgeSize,
 	}

 	ss.txnViewsCond.L = &ss.txnViewsMx
+	ss.txnSyncCond.L = &ss.txnSyncMx
 	ss.ctx, ss.cancel = context.WithCancel(context.Background())

+	ss.reifyCond.L = &ss.reifyMx
+	ss.reifyPend = make(map[cid.Cid]struct{})
+	ss.reifyInProgress = make(map[cid.Cid]struct{})
+
 	if enableDebugLog {
 		ss.debug, err = openDebugLog(path)
 		if err != nil {
@@ -199,21 +220,29 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 		}
 	}

+	if ss.checkpointExists() {
+		log.Info("found compaction checkpoint; resuming compaction")
+		if err := ss.completeCompaction(); err != nil {
+			markSetEnv.Close() //nolint:errcheck
+			return nil, xerrors.Errorf("error resuming compaction: %w", err)
+		}
+	}
+
 	return ss, nil
 }

 // Blockstore interface
-func (s *SplitStore) DeleteBlock(_ cid.Cid) error {
+func (s *SplitStore) DeleteBlock(_ context.Context, _ cid.Cid) error {
 	// afaict we don't seem to be using this method, so it's not implemented
 	return errors.New("DeleteBlock not implemented on SplitStore; don't do this Luke!") //nolint
 }

-func (s *SplitStore) DeleteMany(_ []cid.Cid) error {
+func (s *SplitStore) DeleteMany(_ context.Context, _ []cid.Cid) error {
 	// afaict we don't seem to be using this method, so it's not implemented
 	return errors.New("DeleteMany not implemented on SplitStore; don't do this Luke!") //nolint
 }

-func (s *SplitStore) Has(cid cid.Cid) (bool, error) {
+func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 	if isIdentiyCid(cid) {
 		return true, nil
 	}
@@ -221,7 +250,21 @@ func (s *SplitStore) Has(cid cid.Cid) (bool, error) {
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()

-	has, err := s.hot.Has(cid)
+	// critical section
+	if s.txnMarkSet != nil {
+		has, err := s.txnMarkSet.Has(cid)
+		if err != nil {
+			return false, err
+		}
+
+		if has {
+			return s.has(cid)
+		}
+
+		return s.cold.Has(ctx, cid)
+	}
+
+	has, err := s.hot.Has(ctx, cid)

 	if err != nil {
 		return has, err
@@ -232,10 +275,16 @@ func (s *SplitStore) Has(cid cid.Cid) (bool, error) {
 		return true, nil
 	}

-	return s.cold.Has(cid)
+	has, err = s.cold.Has(ctx, cid)
+	if has && bstore.IsHotView(ctx) {
+		s.reifyColdObject(cid)
+	}
+
+	return has, err
+
 }

-func (s *SplitStore) Get(cid cid.Cid) (blocks.Block, error) {
+func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
 	if isIdentiyCid(cid) {
 		data, err := decodeIdentityCid(cid)
 		if err != nil {
@@ -248,7 +297,21 @@ func (s *SplitStore) Get(cid cid.Cid) (blocks.Block, error) {
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()

-	blk, err := s.hot.Get(cid)
+	// critical section
+	if s.txnMarkSet != nil {
+		has, err := s.txnMarkSet.Has(cid)
+		if err != nil {
+			return nil, err
+		}
+
+		if has {
+			return s.get(cid)
+		}
+
+		return s.cold.Get(ctx, cid)
+	}
+
+	blk, err := s.hot.Get(ctx, cid)

 	switch err {
 	case nil:
@@ -260,10 +323,13 @@ func (s *SplitStore) Get(cid cid.Cid) (blocks.Block, error) {
 			s.debug.LogReadMiss(cid)
 		}

-		blk, err = s.cold.Get(cid)
+		blk, err = s.cold.Get(ctx, cid)
 		if err == nil {
-			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
+			if bstore.IsHotView(ctx) {
+				s.reifyColdObject(cid)
+			}

+			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return blk, err

@@ -272,7 +338,7 @@ func (s *SplitStore) Get(cid cid.Cid) (blocks.Block, error) {
 	}
 }

-func (s *SplitStore) GetSize(cid cid.Cid) (int, error) {
+func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
 	if isIdentiyCid(cid) {
 		data, err := decodeIdentityCid(cid)
 		if err != nil {
@@ -285,7 +351,21 @@ func (s *SplitStore) GetSize(cid cid.Cid) (int, error) {
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()

-	size, err := s.hot.GetSize(cid)
+	// critical section
+	if s.txnMarkSet != nil {
+		has, err := s.txnMarkSet.Has(cid)
+		if err != nil {
+			return 0, err
+		}
+
+		if has {
+			return s.getSize(cid)
+		}
+
+		return s.cold.GetSize(ctx, cid)
+	}
+
+	size, err := s.hot.GetSize(ctx, cid)

 	switch err {
 	case nil:
@@ -297,8 +377,12 @@ func (s *SplitStore) GetSize(cid cid.Cid) (int, error) {
 			s.debug.LogReadMiss(cid)
 		}

-		size, err = s.cold.GetSize(cid)
+		size, err = s.cold.GetSize(ctx, cid)
 		if err == nil {
+			if bstore.IsHotView(ctx) {
+				s.reifyColdObject(cid)
+			}
+
 			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return size, err
@@ -308,7 +392,7 @@ func (s *SplitStore) GetSize(cid cid.Cid) (int, error) {
 	}
 }

-func (s *SplitStore) Put(blk blocks.Block) error {
+func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error {
 	if isIdentiyCid(blk.Cid()) {
 		return nil
 	}
@@ -316,18 +400,24 @@ func (s *SplitStore) Put(blk blocks.Block) error {
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()

-	err := s.hot.Put(blk)
+	err := s.hot.Put(ctx, blk)
 	if err != nil {
 		return err
 	}

 	s.debug.LogWrite(blk)

+	// critical section
+	if s.txnMarkSet != nil {
+		s.markLiveRefs([]cid.Cid{blk.Cid()})
+		return nil
+	}
+
 	s.trackTxnRef(blk.Cid())
 	return nil
 }

-func (s *SplitStore) PutMany(blks []blocks.Block) error {
+func (s *SplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {
 	// filter identites
 	idcids := 0
 	for _, blk := range blks {
@@ -361,13 +451,19 @@ func (s *SplitStore) PutMany(blks []blocks.Block) error {
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()

-	err := s.hot.PutMany(blks)
+	err := s.hot.PutMany(ctx, blks)
 	if err != nil {
 		return err
 	}

 	s.debug.LogWriteMany(blks)

+	// critical section
+	if s.txnMarkSet != nil {
+		s.markLiveRefs(batch)
+		return nil
+	}
+
 	s.trackTxnRefMany(batch)
 	return nil
 }
@@ -417,7 +513,7 @@ func (s *SplitStore) HashOnRead(enabled bool) {
 	s.cold.HashOnRead(enabled)
 }

-func (s *SplitStore) View(cid cid.Cid, cb func([]byte) error) error {
+func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) error) error {
 	if isIdentiyCid(cid) {
 		data, err := decodeIdentityCid(cid)
 		if err != nil {
@@ -427,6 +523,23 @@ func (s *SplitStore) View(cid cid.Cid, cb func([]byte) error) error {
 		return cb(data)
 	}

+	// critical section
+	s.txnLk.RLock() // the lock is released in protectView if we are not in critical section
+	if s.txnMarkSet != nil {
+		has, err := s.txnMarkSet.Has(cid)
+		s.txnLk.RUnlock()
+
+		if err != nil {
+			return err
+		}
+
+		if has {
+			return s.view(cid, cb)
+		}
+
+		return s.cold.View(ctx, cid, cb)
+	}
+
 	// views are (optimistically) protected two-fold:
 	// - if there is an active transaction, then the reference is protected.
 	// - if there is no active transaction, active views are tracked in a
@@ -438,15 +551,19 @@ func (s *SplitStore) View(cid cid.Cid, cb func([]byte) error) error {
 	s.protectView(cid)
 	defer s.viewDone()

-	err := s.hot.View(cid, cb)
+	err := s.hot.View(ctx, cid, cb)
 	switch err {
 	case bstore.ErrNotFound:
 		if s.isWarm() {
 			s.debug.LogReadMiss(cid)
 		}

-		err = s.cold.View(cid, cb)
+		err = s.cold.View(ctx, cid, cb)
 		if err == nil {
+			if bstore.IsHotView(ctx) {
+				s.reifyColdObject(cid)
+			}
+
 			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return err
@@ -463,16 +580,33 @@ func (s *SplitStore) isWarm() bool {
 }

 // State tracking
-func (s *SplitStore) Start(chain ChainAccessor) error {
+func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error {
 	s.chain = chain
 	curTs := chain.GetHeaviestTipSet()

+	// precompute the upgrade boundaries
+	s.upgrades = make([]upgradeRange, 0, len(us))
+	for _, upgrade := range us {
+		boundary := upgrade.Height
+		for _, pre := range upgrade.PreMigrations {
+			preMigrationBoundary := upgrade.Height - pre.StartWithin
+			if preMigrationBoundary < boundary {
+				boundary = preMigrationBoundary
+			}
+		}
+
+		upgradeStart := boundary - upgradeBoundary
+		upgradeEnd := upgrade.Height + upgradeBoundary
+
+		s.upgrades = append(s.upgrades, upgradeRange{start: upgradeStart, end: upgradeEnd})
+	}
+
 	// should we warmup
 	warmup := false

 	// load base epoch from metadata ds
 	// if none, then use current epoch because it's a fresh start
-	bs, err := s.ds.Get(baseEpochKey)
+	bs, err := s.ds.Get(s.ctx, baseEpochKey)
 	switch err {
 	case nil:
 		s.baseEpoch = bytesToEpoch(bs)
@@ -493,7 +627,7 @@ func (s *SplitStore) Start(chain ChainAccessor) error {
 	}

 	// load warmup epoch from metadata ds
-	bs, err = s.ds.Get(warmupEpochKey)
+	bs, err = s.ds.Get(s.ctx, warmupEpochKey)
 	switch err {
 	case nil:
 		s.warmupEpoch = bytesToEpoch(bs)
@@ -506,7 +640,7 @@ func (s *SplitStore) Start(chain ChainAccessor) error {
 	}

 	// load markSetSize from metadata ds to provide a size hint for marksets
-	bs, err = s.ds.Get(markSetSizeKey)
+	bs, err = s.ds.Get(s.ctx, markSetSizeKey)
 	switch err {
 	case nil:
 		s.markSetSize = bytesToInt64(bs)
@@ -517,7 +651,7 @@ func (s *SplitStore) Start(chain ChainAccessor) error {
 	}

 	// load compactionIndex from metadata ds to provide a hint as to when to perform moving gc
-	bs, err = s.ds.Get(compactionIndexKey)
+	bs, err = s.ds.Get(s.ctx, compactionIndexKey)
 	switch err {
 	case nil:
 		s.compactionIndex = bytesToInt64(bs)
@@ -539,6 +673,9 @@ func (s *SplitStore) Start(chain ChainAccessor) error {
 		}
 	}

+	// spawn the reifier
+	go s.reifyOrchestrator()
+
 	// watch the chain
 	chain.SubscribeHeadChanges(s.HeadChange)

@@ -559,12 +696,19 @@ func (s *SplitStore) Close() error {
 	}

 	if atomic.LoadInt32(&s.compacting) == 1 {
+		s.txnSyncMx.Lock()
+		s.txnSync = true
+		s.txnSyncCond.Broadcast()
+		s.txnSyncMx.Unlock()
+
 		log.Warn("close with ongoing compaction in progress; waiting for it to finish...")
 		for atomic.LoadInt32(&s.compacting) == 1 {
 			time.Sleep(time.Second)
 		}
 	}

+	s.reifyCond.Broadcast()
+	s.reifyWorkers.Wait()
 	s.cancel()
 	return multierr.Combine(s.markSetEnv.Close(), s.debug.Close())
 }
@@ -579,5 +723,5 @@ func (s *SplitStore) checkClosing() error {

 func (s *SplitStore) setBaseEpoch(epoch abi.ChainEpoch) error {
 	s.baseEpoch = epoch
-	return s.ds.Put(baseEpochKey, epochToBytes(epoch))
+	return s.ds.Put(s.ctx, baseEpochKey, epochToBytes(epoch))
 }
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sync"
 	"sync/atomic"
 	"time"

@@ -67,7 +68,10 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 	}
 	defer output.Close() //nolint:errcheck

+	var mx sync.Mutex
 	write := func(format string, args ...interface{}) {
+		mx.Lock()
+		defer mx.Unlock()
 		_, err := fmt.Fprintf(output, format+"\n", args...)
 		if err != nil {
 			log.Warnf("error writing check output: %s", err)
@@ -82,9 +86,10 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 	write("compaction index: %d", s.compactionIndex)
 	write("--")

-	var coldCnt, missingCnt int64
+	coldCnt := new(int64)
+	missingCnt := new(int64)

-	visitor, err := s.markSetEnv.CreateVisitor("check", 0)
+	visitor, err := s.markSetEnv.New("check", 0)
 	if err != nil {
 		return xerrors.Errorf("error creating visitor: %w", err)
 	}
@@ -96,7 +101,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 				return errStopWalk
 			}

-			has, err := s.hot.Has(c)
+			has, err := s.hot.Has(s.ctx, c)
 			if err != nil {
 				return xerrors.Errorf("error checking hotstore: %w", err)
 			}
@@ -105,16 +110,16 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 				return nil
 			}

-			has, err = s.cold.Has(c)
+			has, err = s.cold.Has(s.ctx, c)
 			if err != nil {
 				return xerrors.Errorf("error checking coldstore: %w", err)
 			}

 			if has {
-				coldCnt++
+				atomic.AddInt64(coldCnt, 1)
 				write("cold object reference: %s", c)
 			} else {
-				missingCnt++
+				atomic.AddInt64(missingCnt, 1)
 				write("missing object reference: %s", c)
 				return errStopWalk
 			}
@@ -128,9 +133,9 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 		return err
 	}

-	log.Infow("check done", "cold", coldCnt, "missing", missingCnt)
+	log.Infow("check done", "cold", *coldCnt, "missing", *missingCnt)
 	write("--")
-	write("cold: %d missing: %d", coldCnt, missingCnt)
+	write("cold: %d missing: %d", *coldCnt, *missingCnt)
 	write("DONE")

 	return nil
@@ -20,28 +20,28 @@ func (s *SplitStore) Expose() bstore.Blockstore {
 	return &exposedSplitStore{s: s}
 }

-func (es *exposedSplitStore) DeleteBlock(_ cid.Cid) error {
+func (es *exposedSplitStore) DeleteBlock(_ context.Context, _ cid.Cid) error {
 	return errors.New("DeleteBlock: operation not supported")
 }

-func (es *exposedSplitStore) DeleteMany(_ []cid.Cid) error {
+func (es *exposedSplitStore) DeleteMany(_ context.Context, _ []cid.Cid) error {
 	return errors.New("DeleteMany: operation not supported")
 }

-func (es *exposedSplitStore) Has(c cid.Cid) (bool, error) {
+func (es *exposedSplitStore) Has(ctx context.Context, c cid.Cid) (bool, error) {
 	if isIdentiyCid(c) {
 		return true, nil
 	}

-	has, err := es.s.hot.Has(c)
+	has, err := es.s.hot.Has(ctx, c)
 	if has || err != nil {
 		return has, err
 	}

-	return es.s.cold.Has(c)
+	return es.s.cold.Has(ctx, c)
 }

-func (es *exposedSplitStore) Get(c cid.Cid) (blocks.Block, error) {
+func (es *exposedSplitStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
 	if isIdentiyCid(c) {
 		data, err := decodeIdentityCid(c)
 		if err != nil {
@@ -51,16 +51,16 @@ func (es *exposedSplitStore) Get(c cid.Cid) (blocks.Block, error) {
 		return blocks.NewBlockWithCid(data, c)
 	}

-	blk, err := es.s.hot.Get(c)
+	blk, err := es.s.hot.Get(ctx, c)
 	switch err {
 	case bstore.ErrNotFound:
-		return es.s.cold.Get(c)
+		return es.s.cold.Get(ctx, c)
 	default:
 		return blk, err
 	}
 }

-func (es *exposedSplitStore) GetSize(c cid.Cid) (int, error) {
+func (es *exposedSplitStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
 	if isIdentiyCid(c) {
 		data, err := decodeIdentityCid(c)
 		if err != nil {
@@ -70,21 +70,21 @@ func (es *exposedSplitStore) GetSize(c cid.Cid) (int, error) {
 		return len(data), nil
 	}

-	size, err := es.s.hot.GetSize(c)
+	size, err := es.s.hot.GetSize(ctx, c)
 	switch err {
 	case bstore.ErrNotFound:
-		return es.s.cold.GetSize(c)
+		return es.s.cold.GetSize(ctx, c)
 	default:
 		return size, err
 	}
 }

-func (es *exposedSplitStore) Put(blk blocks.Block) error {
-	return es.s.Put(blk)
+func (es *exposedSplitStore) Put(ctx context.Context, blk blocks.Block) error {
+	return es.s.Put(ctx, blk)
 }

-func (es *exposedSplitStore) PutMany(blks []blocks.Block) error {
-	return es.s.PutMany(blks)
+func (es *exposedSplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {
+	return es.s.PutMany(ctx, blks)
 }

 func (es *exposedSplitStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
@@ -93,7 +93,7 @@ func (es *exposedSplitStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, e

 func (es *exposedSplitStore) HashOnRead(enabled bool) {}

-func (es *exposedSplitStore) View(c cid.Cid, f func([]byte) error) error {
+func (es *exposedSplitStore) View(ctx context.Context, c cid.Cid, f func([]byte) error) error {
 	if isIdentiyCid(c) {
 		data, err := decodeIdentityCid(c)
 		if err != nil {
@@ -103,10 +103,10 @@ func (es *exposedSplitStore) View(c cid.Cid, f func([]byte) error) error {
 		return f(data)
 	}

-	err := es.s.hot.View(c, f)
+	err := es.s.hot.View(ctx, c, f)
 	switch err {
 	case bstore.ErrNotFound:
-		return es.s.cold.View(c, f)
+		return es.s.cold.View(ctx, c, f)

 	default:
 		return err
@@ -0,0 +1,214 @@
+package splitstore
+
+import (
+	"errors"
+	"runtime"
+	"sync/atomic"
+
+	"golang.org/x/xerrors"
+
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
+)
+
+var (
+	errReifyLimit = errors.New("reification limit reached")
+	ReifyLimit    = 16384
+)
+
+func (s *SplitStore) reifyColdObject(c cid.Cid) {
+	if !s.isWarm() {
+		return
+	}
+
+	if isUnitaryObject(c) {
+		return
+	}
+
+	s.reifyMx.Lock()
+	defer s.reifyMx.Unlock()
+
+	_, ok := s.reifyInProgress[c]
+	if ok {
+		return
+	}
+
+	s.reifyPend[c] = struct{}{}
+	s.reifyCond.Broadcast()
+}
+
+func (s *SplitStore) reifyOrchestrator() {
+	workers := runtime.NumCPU() / 4
+	if workers < 2 {
+		workers = 2
+	}
+
+	workch := make(chan cid.Cid, workers)
+	defer close(workch)
+
+	for i := 0; i < workers; i++ {
+		s.reifyWorkers.Add(1)
+		go s.reifyWorker(workch)
+	}
+
+	for {
+		s.reifyMx.Lock()
+		for len(s.reifyPend) == 0 && atomic.LoadInt32(&s.closing) == 0 {
+			s.reifyCond.Wait()
+		}
+
+		if atomic.LoadInt32(&s.closing) != 0 {
+			s.reifyMx.Unlock()
+			return
+		}
+
+		reifyPend := s.reifyPend
+		s.reifyPend = make(map[cid.Cid]struct{})
+		s.reifyMx.Unlock()
+
+		for c := range reifyPend {
+			select {
+			case workch <- c:
+			case <-s.ctx.Done():
+				return
+			}
+		}
+	}
+}
+
+func (s *SplitStore) reifyWorker(workch chan cid.Cid) {
+	defer s.reifyWorkers.Done()
+	for c := range workch {
+		s.doReify(c)
+	}
+}
+
+func (s *SplitStore) doReify(c cid.Cid) {
+	var toreify, totrack, toforget []cid.Cid
+
+	defer func() {
+		s.reifyMx.Lock()
+		defer s.reifyMx.Unlock()
+
+		for _, c := range toreify {
+			delete(s.reifyInProgress, c)
+		}
+		for _, c := range totrack {
+			delete(s.reifyInProgress, c)
+		}
+		for _, c := range toforget {
+			delete(s.reifyInProgress, c)
+		}
+	}()
+
+	s.txnLk.RLock()
+	defer s.txnLk.RUnlock()
+
+	count := 0
+	err := s.walkObjectIncomplete(c, newTmpVisitor(),
+		func(c cid.Cid) error {
+			if isUnitaryObject(c) {
+				return errStopWalk
+			}
+
+			count++
+			if count > ReifyLimit {
+				return errReifyLimit
+			}
+
+			s.reifyMx.Lock()
+			_, inProgress := s.reifyInProgress[c]
+			if !inProgress {
+				s.reifyInProgress[c] = struct{}{}
+			}
+			s.reifyMx.Unlock()
+
+			if inProgress {
+				return errStopWalk
+			}
+
+			has, err := s.hot.Has(s.ctx, c)
+			if err != nil {
+				return xerrors.Errorf("error checking hotstore: %w", err)
+			}
+
+			if has {
+				if s.txnMarkSet != nil {
+					hasMark, err := s.txnMarkSet.Has(c)
+					if err != nil {
+						log.Warnf("error checking markset: %s", err)
+					} else if hasMark {
+						toforget = append(toforget, c)
+						return errStopWalk
+					}
+				} else {
+					totrack = append(totrack, c)
+					return errStopWalk
+				}
+			}
+
+			toreify = append(toreify, c)
+			return nil
+		},
+		func(missing cid.Cid) error {
+			log.Warnf("missing reference while reifying %s: %s", c, missing)
+			return errStopWalk
+		})
+
+	if err != nil {
+		if xerrors.Is(err, errReifyLimit) {
+			log.Debug("reification aborted; reify limit reached")
+			return
+		}
+
+		log.Warnf("error walking cold object for reification (cid: %s): %s", c, err)
+		return
+	}
+
+	log.Debugf("reifying %d objects rooted at %s", len(toreify), c)
+
+	// this should not get too big, maybe some 100s of objects.
+	batch := make([]blocks.Block, 0, len(toreify))
+	for _, c := range toreify {
+		blk, err := s.cold.Get(s.ctx, c)
+		if err != nil {
+			log.Warnf("error retrieving cold object for reification (cid: %s): %s", c, err)
+			continue
+		}
+
+		if err := s.checkClosing(); err != nil {
+			return
+		}
+
+		batch = append(batch, blk)
+	}
+
+	if len(batch) > 0 {
+		err = s.hot.PutMany(s.ctx, batch)
+		if err != nil {
+			log.Warnf("error reifying cold object (cid: %s): %s", c, err)
+			return
+		}
+	}
+
+	if s.txnMarkSet != nil {
+		if len(toreify) > 0 {
+			if err := s.txnMarkSet.MarkMany(toreify); err != nil {
+				log.Warnf("error marking reified objects: %s", err)
+			}
+		}
+		if len(totrack) > 0 {
+			if err := s.txnMarkSet.MarkMany(totrack); err != nil {
+				log.Warnf("error marking tracked objects: %s", err)
+			}
+		}
+	} else {
+		// if txnActive is false these are noops
+		if len(toreify) > 0 {
+			s.trackTxnRefMany(toreify)
+		}
+		if len(totrack) > 0 {
+			s.trackTxnRefMany(totrack)
+		}
+	}
+}
@@ -1,9 +1,11 @@
+//stm: #unit
 package splitstore

 import (
 	"context"
 	"errors"
 	"fmt"
+	"math/rand"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -11,6 +13,7 @@ import (

 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/types/mock"

@@ -19,16 +22,19 @@ import (
 	datastore "github.com/ipfs/go-datastore"
 	dssync "github.com/ipfs/go-datastore/sync"
 	logging "github.com/ipfs/go-log/v2"
+	mh "github.com/multiformats/go-multihash"
 )

 func init() {
 	CompactionThreshold = 5
 	CompactionBoundary = 2
 	WarmupBoundary = 0
+	SyncWaitTime = time.Millisecond
 	logging.SetLogLevel("splitstore", "DEBUG")
 }

 func testSplitStore(t *testing.T, cfg *Config) {
+	ctx := context.Background()
 	chain := &mockChain{t: t}

 	// the myriads of stores
@@ -38,7 +44,7 @@ func testSplitStore(t *testing.T, cfg *Config) {

 	// this is necessary to avoid the garbage mock puts in the blocks
 	garbage := blocks.NewBlock([]byte{1, 2, 3})
-	err := cold.Put(garbage)
+	err := cold.Put(ctx, garbage)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -59,27 +65,29 @@ func testSplitStore(t *testing.T, cfg *Config) {
 		t.Fatal(err)
 	}

-	err = cold.Put(blk)
+	err = cold.Put(ctx, blk)
 	if err != nil {
 		t.Fatal(err)
 	}

 	// create a garbage block that is protected with a rgistered protector
 	protected := blocks.NewBlock([]byte("protected!"))
-	err = hot.Put(protected)
+	err = hot.Put(ctx, protected)
 	if err != nil {
 		t.Fatal(err)
 	}

 	// and another one that is not protected
 	unprotected := blocks.NewBlock([]byte("unprotected!"))
-	err = hot.Put(unprotected)
+	err = hot.Put(ctx, unprotected)
 	if err != nil {
 		t.Fatal(err)
 	}

+	path := t.TempDir()
+
 	// open the splitstore
-	ss, err := Open("", ds, hot, cold, cfg)
+	ss, err := Open(path, ds, hot, cold, cfg)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -90,7 +98,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
 		return protect(protected.Cid())
 	})

-	err = ss.Start(chain)
+	err = ss.Start(chain, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -108,11 +116,11 @@ func testSplitStore(t *testing.T, cfg *Config) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		err = ss.Put(stateRoot)
+		err = ss.Put(ctx, stateRoot)
 		if err != nil {
 			t.Fatal(err)
 		}
-		err = ss.Put(sblk)
+		err = ss.Put(ctx, sblk)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -123,6 +131,10 @@ func testSplitStore(t *testing.T, cfg *Config) {
 	}

 	waitForCompaction := func() {
+		ss.txnSyncMx.Lock()
+		ss.txnSync = true
+		ss.txnSyncCond.Broadcast()
+		ss.txnSyncMx.Unlock()
 		for atomic.LoadInt32(&ss.compacting) == 1 {
 			time.Sleep(100 * time.Millisecond)
 		}
@@ -175,7 +187,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
 	}

 	// ensure our protected block is still there
-	has, err := hot.Has(protected.Cid())
+	has, err := hot.Has(ctx, protected.Cid())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -185,7 +197,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
 	}

 	// ensure our unprotected block is in the coldstore now
-	has, err = hot.Has(unprotected.Cid())
+	has, err = hot.Has(ctx, unprotected.Cid())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -194,7 +206,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
 		t.Fatal("unprotected block is still in hotstore")
 	}

-	has, err = cold.Has(unprotected.Cid())
+	has, err = cold.Has(ctx, unprotected.Cid())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -208,10 +220,16 @@ func testSplitStore(t *testing.T, cfg *Config) {
 }

 func TestSplitStoreCompaction(t *testing.T) {
+	//stm: @SPLITSTORE_SPLITSTORE_OPEN_001, @SPLITSTORE_SPLITSTORE_CLOSE_001
+	//stm: @SPLITSTORE_SPLITSTORE_PUT_001, @SPLITSTORE_SPLITSTORE_ADD_PROTECTOR_001
+	//stm: @SPLITSTORE_SPLITSTORE_CLOSE_001
 	testSplitStore(t, &Config{MarkSetType: "map"})
 }

 func TestSplitStoreCompactionWithBadger(t *testing.T) {
+	//stm: @SPLITSTORE_SPLITSTORE_OPEN_001, @SPLITSTORE_SPLITSTORE_CLOSE_001
+	//stm: @SPLITSTORE_SPLITSTORE_PUT_001, @SPLITSTORE_SPLITSTORE_ADD_PROTECTOR_001
+	//stm: @SPLITSTORE_SPLITSTORE_CLOSE_001
 	bs := badgerMarkSetBatchSize
 	badgerMarkSetBatchSize = 1
 	t.Cleanup(func() {
@@ -220,6 +238,365 @@ func TestSplitStoreCompactionWithBadger(t *testing.T) {
 	testSplitStore(t, &Config{MarkSetType: "badger"})
 }

+func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
+	//stm: @SPLITSTORE_SPLITSTORE_OPEN_001, @SPLITSTORE_SPLITSTORE_CLOSE_001
+	//stm: @SPLITSTORE_SPLITSTORE_PUT_001, @SPLITSTORE_SPLITSTORE_ADD_PROTECTOR_001
+	//stm: @SPLITSTORE_SPLITSTORE_CLOSE_001
+	ctx := context.Background()
+	chain := &mockChain{t: t}
+
+	// the myriads of stores
+	ds := dssync.MutexWrap(datastore.NewMapDatastore())
+	hot := newMockStore()
+	cold := newMockStore()
+
+	// this is necessary to avoid the garbage mock puts in the blocks
+	garbage := blocks.NewBlock([]byte{1, 2, 3})
+	err := cold.Put(ctx, garbage)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// genesis
+	genBlock := mock.MkBlock(nil, 0, 0)
+	genBlock.Messages = garbage.Cid()
+	genBlock.ParentMessageReceipts = garbage.Cid()
+	genBlock.ParentStateRoot = garbage.Cid()
+	genBlock.Timestamp = uint64(time.Now().Unix())
+
+	genTs := mock.TipSet(genBlock)
+	chain.push(genTs)
+
+	// put the genesis block to cold store
+	blk, err := genBlock.ToStorageBlock()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = cold.Put(ctx, blk)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	path := t.TempDir()
+
+	// open the splitstore
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close() //nolint
+
+	// create an upgrade schedule that will suppress compaction during the test
+	upgradeBoundary = 0
+	upgrade := stmgr.Upgrade{
+		Height:        10,
+		PreMigrations: []stmgr.PreMigration{{StartWithin: 10}},
+	}
+
+	err = ss.Start(chain, []stmgr.Upgrade{upgrade})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mkBlock := func(curTs *types.TipSet, i int, stateRoot blocks.Block) *types.TipSet {
+		blk := mock.MkBlock(curTs, uint64(i), uint64(i))
+
+		blk.Messages = garbage.Cid()
+		blk.ParentMessageReceipts = garbage.Cid()
+		blk.ParentStateRoot = stateRoot.Cid()
+		blk.Timestamp = uint64(time.Now().Unix())
+
+		sblk, err := blk.ToStorageBlock()
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = ss.Put(ctx, stateRoot)
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = ss.Put(ctx, sblk)
+		if err != nil {
+			t.Fatal(err)
+		}
+		ts := mock.TipSet(blk)
+		chain.push(ts)
+
+		return ts
+	}
+
+	waitForCompaction := func() {
+		ss.txnSyncMx.Lock()
+		ss.txnSync = true
+		ss.txnSyncCond.Broadcast()
+		ss.txnSyncMx.Unlock()
+		for atomic.LoadInt32(&ss.compacting) == 1 {
+			time.Sleep(100 * time.Millisecond)
+		}
+	}
+
+	curTs := genTs
+	for i := 1; i < 10; i++ {
+		stateRoot := blocks.NewBlock([]byte{byte(i), 3, 3, 7})
+		curTs = mkBlock(curTs, i, stateRoot)
+		waitForCompaction()
+	}
+
+	countBlocks := func(bs blockstore.Blockstore) int {
+		count := 0
+		_ = bs.(blockstore.BlockstoreIterator).ForEachKey(func(_ cid.Cid) error {
+			count++
+			return nil
+		})
+		return count
+	}
+
+	// we should not have compacted due to suppression and everything should still be hot
+	hotCnt := countBlocks(hot)
+	coldCnt := countBlocks(cold)
+
+	if hotCnt != 20 {
+		t.Errorf("expected %d blocks, but got %d", 20, hotCnt)
+	}
+
+	if coldCnt != 2 {
+		t.Errorf("expected %d blocks, but got %d", 2, coldCnt)
+	}
+
+	// put some more blocks, now we should compact
+	for i := 10; i < 20; i++ {
+		stateRoot := blocks.NewBlock([]byte{byte(i), 3, 3, 7})
+		curTs = mkBlock(curTs, i, stateRoot)
+		waitForCompaction()
+	}
+
+	hotCnt = countBlocks(hot)
+	coldCnt = countBlocks(cold)
+
+	if hotCnt != 24 {
+		t.Errorf("expected %d blocks, but got %d", 24, hotCnt)
+	}
+
+	if coldCnt != 18 {
+		t.Errorf("expected %d blocks, but got %d", 18, coldCnt)
+	}
+}
+
+func testSplitStoreReification(t *testing.T, f func(context.Context, blockstore.Blockstore, cid.Cid) error) {
+	ds := dssync.MutexWrap(datastore.NewMapDatastore())
+	hot := newMockStore()
+	cold := newMockStore()
+
+	mkRandomBlock := func() blocks.Block {
+		data := make([]byte, 128)
+		_, err := rand.Read(data)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return blocks.NewBlock(data)
+	}
+
+	block1 := mkRandomBlock()
+	block2 := mkRandomBlock()
+	block3 := mkRandomBlock()
+
+	hdr := mock.MkBlock(nil, 0, 0)
+	hdr.Messages = block1.Cid()
+	hdr.ParentMessageReceipts = block2.Cid()
+	hdr.ParentStateRoot = block3.Cid()
+	block4, err := hdr.ToStorageBlock()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	allBlocks := []blocks.Block{block1, block2, block3, block4}
+	for _, blk := range allBlocks {
+		err := cold.Put(context.Background(), blk)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	path := t.TempDir()
+
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close() //nolint
+
+	ss.warmupEpoch = 1
+	go ss.reifyOrchestrator()
+
+	waitForReification := func() {
+		for {
+			ss.reifyMx.Lock()
+			ready := len(ss.reifyPend) == 0 && len(ss.reifyInProgress) == 0
+			ss.reifyMx.Unlock()
+
+			if ready {
+				return
+			}
+
+			time.Sleep(time.Millisecond)
+		}
+	}
+
+	// first access using the standard view
+	err = f(context.Background(), ss, block4.Cid())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// nothing should be reified
+	waitForReification()
+	for _, blk := range allBlocks {
+		has, err := hot.Has(context.Background(), blk.Cid())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("block unexpectedly reified")
+		}
+	}
+
+	// now make the hot/reifying view and ensure access reifies
+	err = f(blockstore.WithHotView(context.Background()), ss, block4.Cid())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// everything should be reified
+	waitForReification()
+	for i, blk := range allBlocks {
+		has, err := hot.Has(context.Background(), blk.Cid())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !has {
+			t.Fatalf("block%d was not reified", i+1)
+		}
+	}
+}
+
+func testSplitStoreReificationLimit(t *testing.T, f func(context.Context, blockstore.Blockstore, cid.Cid) error) {
+	ds := dssync.MutexWrap(datastore.NewMapDatastore())
+	hot := newMockStore()
+	cold := newMockStore()
+
+	mkRandomBlock := func() blocks.Block {
+		data := make([]byte, 128)
+		_, err := rand.Read(data)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return blocks.NewBlock(data)
+	}
+
+	block1 := mkRandomBlock()
+	block2 := mkRandomBlock()
+	block3 := mkRandomBlock()
+
+	hdr := mock.MkBlock(nil, 0, 0)
+	hdr.Messages = block1.Cid()
+	hdr.ParentMessageReceipts = block2.Cid()
+	hdr.ParentStateRoot = block3.Cid()
+	block4, err := hdr.ToStorageBlock()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	allBlocks := []blocks.Block{block1, block2, block3, block4}
+	for _, blk := range allBlocks {
+		err := cold.Put(context.Background(), blk)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	path := t.TempDir()
+
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close() //nolint
+
+	ss.warmupEpoch = 1
+	go ss.reifyOrchestrator()
+
+	waitForReification := func() {
+		for {
+			ss.reifyMx.Lock()
+			ready := len(ss.reifyPend) == 0 && len(ss.reifyInProgress) == 0
+			ss.reifyMx.Unlock()
+
+			if ready {
+				return
+			}
+
+			time.Sleep(time.Millisecond)
+		}
+	}
+
+	// do a hot access -- nothing should be reified as the limit should be exceeded
+	oldReifyLimit := ReifyLimit
+	ReifyLimit = 2
+	t.Cleanup(func() {
+		ReifyLimit = oldReifyLimit
+	})
+
+	err = f(blockstore.WithHotView(context.Background()), ss, block4.Cid())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	waitForReification()
+
+	for _, blk := range allBlocks {
+		has, err := hot.Has(context.Background(), blk.Cid())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("block unexpectedly reified")
+		}
+	}
+
+}
+
+func TestSplitStoreReification(t *testing.T) {
+	t.Log("test reification with Has")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.Has(ctx, c)
+		return err
+	})
+	t.Log("test reification with Get")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.Get(ctx, c)
+		return err
+	})
+	t.Log("test reification with GetSize")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.GetSize(ctx, c)
+		return err
+	})
+	t.Log("test reification with View")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		return s.View(ctx, c, func(_ []byte) error { return nil })
+	})
+	t.Log("test reification limit")
+	testSplitStoreReificationLimit(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.Has(ctx, c)
+		return err
+	})
+}
+
 type mockChain struct {
 	t testing.TB

@@ -289,35 +666,43 @@ func (c *mockChain) SubscribeHeadChanges(change func(revert []*types.TipSet, app

 type mockStore struct {
 	mx  sync.Mutex
-	set map[cid.Cid]blocks.Block
+	set map[string]blocks.Block
 }

 func newMockStore() *mockStore {
-	return &mockStore{set: make(map[cid.Cid]blocks.Block)}
+	return &mockStore{set: make(map[string]blocks.Block)}
 }

-func (b *mockStore) Has(cid cid.Cid) (bool, error) {
+func (b *mockStore) keyOf(c cid.Cid) string {
+	return string(c.Hash())
+}
+
+func (b *mockStore) cidOf(k string) cid.Cid {
+	return cid.NewCidV1(cid.Raw, mh.Multihash([]byte(k)))
+}
+
+func (b *mockStore) Has(_ context.Context, cid cid.Cid) (bool, error) {
 	b.mx.Lock()
 	defer b.mx.Unlock()
-	_, ok := b.set[cid]
+	_, ok := b.set[b.keyOf(cid)]
 	return ok, nil
 }

 func (b *mockStore) HashOnRead(hor bool) {}

-func (b *mockStore) Get(cid cid.Cid) (blocks.Block, error) {
+func (b *mockStore) Get(_ context.Context, cid cid.Cid) (blocks.Block, error) {
 	b.mx.Lock()
 	defer b.mx.Unlock()

-	blk, ok := b.set[cid]
+	blk, ok := b.set[b.keyOf(cid)]
 	if !ok {
 		return nil, blockstore.ErrNotFound
 	}
 	return blk, nil
 }

-func (b *mockStore) GetSize(cid cid.Cid) (int, error) {
-	blk, err := b.Get(cid)
+func (b *mockStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
+	blk, err := b.Get(ctx, cid)
 	if err != nil {
 		return 0, err
 	}
@@ -325,46 +710,46 @@ func (b *mockStore) GetSize(cid cid.Cid) (int, error) {
 	return len(blk.RawData()), nil
 }

-func (b *mockStore) View(cid cid.Cid, f func([]byte) error) error {
-	blk, err := b.Get(cid)
+func (b *mockStore) View(ctx context.Context, cid cid.Cid, f func([]byte) error) error {
+	blk, err := b.Get(ctx, cid)
 	if err != nil {
 		return err
 	}
 	return f(blk.RawData())
 }

-func (b *mockStore) Put(blk blocks.Block) error {
+func (b *mockStore) Put(_ context.Context, blk blocks.Block) error {
 	b.mx.Lock()
 	defer b.mx.Unlock()

-	b.set[blk.Cid()] = blk
+	b.set[b.keyOf(blk.Cid())] = blk
 	return nil
 }

-func (b *mockStore) PutMany(blks []blocks.Block) error {
+func (b *mockStore) PutMany(_ context.Context, blks []blocks.Block) error {
 	b.mx.Lock()
 	defer b.mx.Unlock()

 	for _, blk := range blks {
-		b.set[blk.Cid()] = blk
+		b.set[b.keyOf(blk.Cid())] = blk
 	}
 	return nil
 }

-func (b *mockStore) DeleteBlock(cid cid.Cid) error {
+func (b *mockStore) DeleteBlock(_ context.Context, cid cid.Cid) error {
 	b.mx.Lock()
 	defer b.mx.Unlock()

-	delete(b.set, cid)
+	delete(b.set, b.keyOf(cid))
 	return nil
 }

-func (b *mockStore) DeleteMany(cids []cid.Cid) error {
+func (b *mockStore) DeleteMany(_ context.Context, cids []cid.Cid) error {
 	b.mx.Lock()
 	defer b.mx.Unlock()

 	for _, c := range cids {
-		delete(b.set, c)
+		delete(b.set, b.keyOf(c))
 	}
 	return nil
 }
@@ -378,7 +763,7 @@ func (b *mockStore) ForEachKey(f func(cid.Cid) error) error {
 	defer b.mx.Unlock()

 	for c := range b.set {
-		err := f(c)
+		err := f(b.cidOf(c))
 		if err != nil {
 			return err
 		}
@@ -1,6 +1,7 @@
 package splitstore

 import (
+	"sync"
 	"sync/atomic"
 	"time"

@@ -55,12 +56,13 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 	if WarmupBoundary < epoch {
 		boundaryEpoch = epoch - WarmupBoundary
 	}
+	var mx sync.Mutex
 	batchHot := make([]blocks.Block, 0, batchSize)
-	count := int64(0)
-	xcount := int64(0)
-	missing := int64(0)
+	count := new(int64)
+	xcount := new(int64)
+	missing := new(int64)

-	visitor, err := s.markSetEnv.CreateVisitor("warmup", 0)
+	visitor, err := s.markSetEnv.New("warmup", 0)
 	if err != nil {
 		return xerrors.Errorf("error creating visitor: %w", err)
 	}
@@ -73,9 +75,9 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 				return errStopWalk
 			}

-			count++
+			atomic.AddInt64(count, 1)

-			has, err := s.hot.Has(c)
+			has, err := s.hot.Has(s.ctx, c)
 			if err != nil {
 				return err
 			}
@@ -84,25 +86,28 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 				return nil
 			}

-			blk, err := s.cold.Get(c)
+			blk, err := s.cold.Get(s.ctx, c)
 			if err != nil {
 				if err == bstore.ErrNotFound {
-					missing++
+					atomic.AddInt64(missing, 1)
 					return errStopWalk
 				}
 				return err
 			}

-			xcount++
+			atomic.AddInt64(xcount, 1)

+			mx.Lock()
 			batchHot = append(batchHot, blk)
 			if len(batchHot) == batchSize {
-				err = s.hot.PutMany(batchHot)
+				err = s.hot.PutMany(s.ctx, batchHot)
 				if err != nil {
+					mx.Unlock()
 					return err
 				}
 				batchHot = batchHot[:0]
 			}
+			mx.Unlock()

 			return nil
 		})
@@ -112,22 +117,22 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 	}

 	if len(batchHot) > 0 {
-		err = s.hot.PutMany(batchHot)
+		err = s.hot.PutMany(s.ctx, batchHot)
 		if err != nil {
 			return err
 		}
 	}

-	log.Infow("warmup stats", "visited", count, "warm", xcount, "missing", missing)
+	log.Infow("warmup stats", "visited", *count, "warm", *xcount, "missing", *missing)

-	s.markSetSize = count + count>>2 // overestimate a bit
-	err = s.ds.Put(markSetSizeKey, int64ToBytes(s.markSetSize))
+	s.markSetSize = *count + *count>>2 // overestimate a bit
+	err = s.ds.Put(s.ctx, markSetSizeKey, int64ToBytes(s.markSetSize))
 	if err != nil {
 		log.Warnf("error saving mark set size: %s", err)
 	}

 	// save the warmup epoch
-	err = s.ds.Put(warmupEpochKey, epochToBytes(epoch))
+	err = s.ds.Put(s.ctx, warmupEpochKey, epochToBytes(epoch))
 	if err != nil {
 		return xerrors.Errorf("error saving warm up epoch: %w", err)
 	}
@@ -136,7 +141,7 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 	s.mx.Unlock()

 	// also save the compactionIndex, as this is used as an indicator of warmup for upgraded nodes
-	err = s.ds.Put(compactionIndexKey, int64ToBytes(s.compactionIndex))
+	err = s.ds.Put(s.ctx, compactionIndexKey, int64ToBytes(s.compactionIndex))
 	if err != nil {
 		return xerrors.Errorf("error saving compaction index: %w", err)
 	}
@@ -1,6 +1,8 @@
 package splitstore

 import (
+	"sync"
+
 	cid "github.com/ipfs/go-cid"
 )

@@ -17,16 +19,42 @@ func (v *noopVisitor) Visit(_ cid.Cid) (bool, error) {
 	return true, nil
 }

-type cidSetVisitor struct {
+type tmpVisitor struct {
 	set *cid.Set
 }

-var _ ObjectVisitor = (*cidSetVisitor)(nil)
+var _ ObjectVisitor = (*tmpVisitor)(nil)
+
+func (v *tmpVisitor) Visit(c cid.Cid) (bool, error) {
+	if isUnitaryObject(c) {
+		return false, nil
+	}

-func (v *cidSetVisitor) Visit(c cid.Cid) (bool, error) {
 	return v.set.Visit(c), nil
 }

-func tmpVisitor() ObjectVisitor {
-	return &cidSetVisitor{set: cid.NewSet()}
+func newTmpVisitor() ObjectVisitor {
+	return &tmpVisitor{set: cid.NewSet()}
+}
+
+type concurrentVisitor struct {
+	mx  sync.Mutex
+	set *cid.Set
+}
+
+var _ ObjectVisitor = (*concurrentVisitor)(nil)
+
+func newConcurrentVisitor() *concurrentVisitor {
+	return &concurrentVisitor{set: cid.NewSet()}
+}
+
+func (v *concurrentVisitor) Visit(c cid.Cid) (bool, error) {
+	if isUnitaryObject(c) {
+		return false, nil
+	}
+
+	v.mx.Lock()
+	defer v.mx.Unlock()
+
+	return v.set.Visit(c), nil
 }
@@ -20,53 +20,53 @@ type SyncBlockstore struct {
 	bs MemBlockstore // specifically use a memStore to save indirection overhead.
 }

-func (m *SyncBlockstore) DeleteBlock(k cid.Cid) error {
+func (m *SyncBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	return m.bs.DeleteBlock(k)
+	return m.bs.DeleteBlock(ctx, k)
 }

-func (m *SyncBlockstore) DeleteMany(ks []cid.Cid) error {
+func (m *SyncBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	return m.bs.DeleteMany(ks)
+	return m.bs.DeleteMany(ctx, ks)
 }

-func (m *SyncBlockstore) Has(k cid.Cid) (bool, error) {
+func (m *SyncBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error) {
 	m.mu.RLock()
 	defer m.mu.RUnlock()
-	return m.bs.Has(k)
+	return m.bs.Has(ctx, k)
 }

-func (m *SyncBlockstore) View(k cid.Cid, callback func([]byte) error) error {
+func (m *SyncBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error {
 	m.mu.RLock()
 	defer m.mu.RUnlock()

-	return m.bs.View(k, callback)
+	return m.bs.View(ctx, k, callback)
 }

-func (m *SyncBlockstore) Get(k cid.Cid) (blocks.Block, error) {
+func (m *SyncBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) {
 	m.mu.RLock()
 	defer m.mu.RUnlock()
-	return m.bs.Get(k)
+	return m.bs.Get(ctx, k)
 }

-func (m *SyncBlockstore) GetSize(k cid.Cid) (int, error) {
+func (m *SyncBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
 	m.mu.RLock()
 	defer m.mu.RUnlock()
-	return m.bs.GetSize(k)
+	return m.bs.GetSize(ctx, k)
 }

-func (m *SyncBlockstore) Put(b blocks.Block) error {
+func (m *SyncBlockstore) Put(ctx context.Context, b blocks.Block) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	return m.bs.Put(b)
+	return m.bs.Put(ctx, b)
 }

-func (m *SyncBlockstore) PutMany(bs []blocks.Block) error {
+func (m *SyncBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	return m.bs.PutMany(bs)
+	return m.bs.PutMany(ctx, bs)
 }

 func (m *SyncBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
@@ -92,28 +92,28 @@ func (t *TimedCacheBlockstore) rotate() {
 	t.mu.Unlock()
 }

-func (t *TimedCacheBlockstore) Put(b blocks.Block) error {
+func (t *TimedCacheBlockstore) Put(ctx context.Context, b blocks.Block) error {
 	// Don't check the inactive set here. We want to keep this block for at
 	// least one interval.
 	t.mu.Lock()
 	defer t.mu.Unlock()
-	return t.active.Put(b)
+	return t.active.Put(ctx, b)
 }

-func (t *TimedCacheBlockstore) PutMany(bs []blocks.Block) error {
+func (t *TimedCacheBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
-	return t.active.PutMany(bs)
+	return t.active.PutMany(ctx, bs)
 }

-func (t *TimedCacheBlockstore) View(k cid.Cid, callback func([]byte) error) error {
+func (t *TimedCacheBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error {
 	// The underlying blockstore is always a "mem" blockstore so there's no difference,
 	// from a performance perspective, between view & get. So we call Get to avoid
 	// calling an arbitrary callback while holding a lock.
 	t.mu.RLock()
-	block, err := t.active.Get(k)
+	block, err := t.active.Get(ctx, k)
 	if err == ErrNotFound {
-		block, err = t.inactive.Get(k)
+		block, err = t.inactive.Get(ctx, k)
 	}
 	t.mu.RUnlock()

@@ -123,51 +123,51 @@ func (t *TimedCacheBlockstore) View(k cid.Cid, callback func([]byte) error) erro
 	return callback(block.RawData())
 }

-func (t *TimedCacheBlockstore) Get(k cid.Cid) (blocks.Block, error) {
+func (t *TimedCacheBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
-	b, err := t.active.Get(k)
+	b, err := t.active.Get(ctx, k)
 	if err == ErrNotFound {
-		b, err = t.inactive.Get(k)
+		b, err = t.inactive.Get(ctx, k)
 	}
 	return b, err
 }

-func (t *TimedCacheBlockstore) GetSize(k cid.Cid) (int, error) {
+func (t *TimedCacheBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
-	size, err := t.active.GetSize(k)
+	size, err := t.active.GetSize(ctx, k)
 	if err == ErrNotFound {
-		size, err = t.inactive.GetSize(k)
+		size, err = t.inactive.GetSize(ctx, k)
 	}
 	return size, err
 }

-func (t *TimedCacheBlockstore) Has(k cid.Cid) (bool, error) {
+func (t *TimedCacheBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error) {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
-	if has, err := t.active.Has(k); err != nil {
+	if has, err := t.active.Has(ctx, k); err != nil {
 		return false, err
 	} else if has {
 		return true, nil
 	}
-	return t.inactive.Has(k)
+	return t.inactive.Has(ctx, k)
 }

 func (t *TimedCacheBlockstore) HashOnRead(_ bool) {
 	// no-op
 }

-func (t *TimedCacheBlockstore) DeleteBlock(k cid.Cid) error {
+func (t *TimedCacheBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
-	return multierr.Combine(t.active.DeleteBlock(k), t.inactive.DeleteBlock(k))
+	return multierr.Combine(t.active.DeleteBlock(ctx, k), t.inactive.DeleteBlock(ctx, k))
 }

-func (t *TimedCacheBlockstore) DeleteMany(ks []cid.Cid) error {
+func (t *TimedCacheBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
-	return multierr.Combine(t.active.DeleteMany(ks), t.inactive.DeleteMany(ks))
+	return multierr.Combine(t.active.DeleteMany(ctx, ks), t.inactive.DeleteMany(ctx, ks))
 }

 func (t *TimedCacheBlockstore) AllKeysChan(_ context.Context) (<-chan cid.Cid, error) {
@@ -1,3 +1,4 @@
+//stm: #unit
 package blockstore

 import (
@@ -13,12 +14,17 @@ import (
 )

 func TestTimedCacheBlockstoreSimple(t *testing.T) {
+	//stm: @SPLITSTORE_TIMED_BLOCKSTORE_START_001
+	//stm: @SPLITSTORE_TIMED_BLOCKSTORE_PUT_001, @SPLITSTORE_TIMED_BLOCKSTORE_HAS_001, @SPLITSTORE_TIMED_BLOCKSTORE_GET_001
+	//stm: @SPLITSTORE_TIMED_BLOCKSTORE_ALL_KEYS_CHAN_001
 	tc := NewTimedCacheBlockstore(10 * time.Millisecond)
 	mClock := clock.NewMock()
 	mClock.Set(time.Now())
 	tc.clock = mClock
 	tc.doneRotatingCh = make(chan struct{})

+	ctx := context.Background()
+
 	_ = tc.Start(context.Background())
 	mClock.Add(1) // IDK why it is needed but it makes it work

@@ -27,18 +33,18 @@ func TestTimedCacheBlockstoreSimple(t *testing.T) {
 	}()

 	b1 := blocks.NewBlock([]byte("foo"))
-	require.NoError(t, tc.Put(b1))
+	require.NoError(t, tc.Put(ctx, b1))

 	b2 := blocks.NewBlock([]byte("bar"))
-	require.NoError(t, tc.Put(b2))
+	require.NoError(t, tc.Put(ctx, b2))

 	b3 := blocks.NewBlock([]byte("baz"))

-	b1out, err := tc.Get(b1.Cid())
+	b1out, err := tc.Get(ctx, b1.Cid())
 	require.NoError(t, err)
 	require.Equal(t, b1.RawData(), b1out.RawData())

-	has, err := tc.Has(b1.Cid())
+	has, err := tc.Has(ctx, b1.Cid())
 	require.NoError(t, err)
 	require.True(t, has)

@@ -46,17 +52,17 @@ func TestTimedCacheBlockstoreSimple(t *testing.T) {
 	<-tc.doneRotatingCh

 	// We should still have everything.
-	has, err = tc.Has(b1.Cid())
+	has, err = tc.Has(ctx, b1.Cid())
 	require.NoError(t, err)
 	require.True(t, has)

-	has, err = tc.Has(b2.Cid())
+	has, err = tc.Has(ctx, b2.Cid())
 	require.NoError(t, err)
 	require.True(t, has)

 	// extend b2, add b3.
-	require.NoError(t, tc.Put(b2))
-	require.NoError(t, tc.Put(b3))
+	require.NoError(t, tc.Put(ctx, b2))
+	require.NoError(t, tc.Put(ctx, b3))

 	// all keys once.
 	allKeys, err := tc.AllKeysChan(context.Background())
@@ -71,15 +77,15 @@ func TestTimedCacheBlockstoreSimple(t *testing.T) {
 	<-tc.doneRotatingCh
 	// should still have b2, and b3, but not b1

-	has, err = tc.Has(b1.Cid())
+	has, err = tc.Has(ctx, b1.Cid())
 	require.NoError(t, err)
 	require.False(t, has)

-	has, err = tc.Has(b2.Cid())
+	has, err = tc.Has(ctx, b2.Cid())
 	require.NoError(t, err)
 	require.True(t, has)

-	has, err = tc.Has(b3.Cid())
+	has, err = tc.Has(ctx, b3.Cid())
 	require.NoError(t, err)
 	require.True(t, has)
 }
@@ -19,72 +19,72 @@ func Union(stores ...Blockstore) Blockstore {
 	return unionBlockstore(stores)
 }

-func (m unionBlockstore) Has(cid cid.Cid) (has bool, err error) {
+func (m unionBlockstore) Has(ctx context.Context, cid cid.Cid) (has bool, err error) {
 	for _, bs := range m {
-		if has, err = bs.Has(cid); has || err != nil {
+		if has, err = bs.Has(ctx, cid); has || err != nil {
 			break
 		}
 	}
 	return has, err
 }

-func (m unionBlockstore) Get(cid cid.Cid) (blk blocks.Block, err error) {
+func (m unionBlockstore) Get(ctx context.Context, cid cid.Cid) (blk blocks.Block, err error) {
 	for _, bs := range m {
-		if blk, err = bs.Get(cid); err == nil || err != ErrNotFound {
+		if blk, err = bs.Get(ctx, cid); err == nil || err != ErrNotFound {
 			break
 		}
 	}
 	return blk, err
 }

-func (m unionBlockstore) View(cid cid.Cid, callback func([]byte) error) (err error) {
+func (m unionBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) (err error) {
 	for _, bs := range m {
-		if err = bs.View(cid, callback); err == nil || err != ErrNotFound {
+		if err = bs.View(ctx, cid, callback); err == nil || err != ErrNotFound {
 			break
 		}
 	}
 	return err
 }

-func (m unionBlockstore) GetSize(cid cid.Cid) (size int, err error) {
+func (m unionBlockstore) GetSize(ctx context.Context, cid cid.Cid) (size int, err error) {
 	for _, bs := range m {
-		if size, err = bs.GetSize(cid); err == nil || err != ErrNotFound {
+		if size, err = bs.GetSize(ctx, cid); err == nil || err != ErrNotFound {
 			break
 		}
 	}
 	return size, err
 }

-func (m unionBlockstore) Put(block blocks.Block) (err error) {
+func (m unionBlockstore) Put(ctx context.Context, block blocks.Block) (err error) {
 	for _, bs := range m {
-		if err = bs.Put(block); err != nil {
+		if err = bs.Put(ctx, block); err != nil {
 			break
 		}
 	}
 	return err
 }

-func (m unionBlockstore) PutMany(blks []blocks.Block) (err error) {
+func (m unionBlockstore) PutMany(ctx context.Context, blks []blocks.Block) (err error) {
 	for _, bs := range m {
-		if err = bs.PutMany(blks); err != nil {
+		if err = bs.PutMany(ctx, blks); err != nil {
 			break
 		}
 	}
 	return err
 }

-func (m unionBlockstore) DeleteBlock(cid cid.Cid) (err error) {
+func (m unionBlockstore) DeleteBlock(ctx context.Context, cid cid.Cid) (err error) {
 	for _, bs := range m {
-		if err = bs.DeleteBlock(cid); err != nil {
+		if err = bs.DeleteBlock(ctx, cid); err != nil {
 			break
 		}
 	}
 	return err
 }

-func (m unionBlockstore) DeleteMany(cids []cid.Cid) (err error) {
+func (m unionBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) (err error) {
 	for _, bs := range m {
-		if err = bs.DeleteMany(cids); err != nil {
+		if err = bs.DeleteMany(ctx, cids); err != nil {
 			break
 		}
 	}
@@ -1,3 +1,4 @@
+//stm: #unit
 package blockstore

 import (
@@ -15,79 +16,85 @@ var (
 )

 func TestUnionBlockstore_Get(t *testing.T) {
+	//stm: @SPLITSTORE_UNION_BLOCKSTORE_GET_001
+	ctx := context.Background()
 	m1 := NewMemory()
 	m2 := NewMemory()

-	_ = m1.Put(b1)
-	_ = m2.Put(b2)
+	_ = m1.Put(ctx, b1)
+	_ = m2.Put(ctx, b2)

 	u := Union(m1, m2)

-	v1, err := u.Get(b1.Cid())
+	v1, err := u.Get(ctx, b1.Cid())
 	require.NoError(t, err)
 	require.Equal(t, b1.RawData(), v1.RawData())

-	v2, err := u.Get(b2.Cid())
+	v2, err := u.Get(ctx, b2.Cid())
 	require.NoError(t, err)
 	require.Equal(t, b2.RawData(), v2.RawData())
 }

 func TestUnionBlockstore_Put_PutMany_Delete_AllKeysChan(t *testing.T) {
+	//stm: @SPLITSTORE_UNION_BLOCKSTORE_PUT_001, @SPLITSTORE_UNION_BLOCKSTORE_HAS_001
+	//stm: @SPLITSTORE_UNION_BLOCKSTORE_PUT_MANY_001, @SPLITSTORE_UNION_BLOCKSTORE_DELETE_001
+	//stm: @SPLITSTORE_UNION_BLOCKSTORE_ALL_KEYS_CHAN_001
+	ctx := context.Background()
 	m1 := NewMemory()
 	m2 := NewMemory()

 	u := Union(m1, m2)

-	err := u.Put(b0)
+	err := u.Put(ctx, b0)
 	require.NoError(t, err)

 	var has bool

 	// write was broadcasted to all stores.
-	has, _ = m1.Has(b0.Cid())
+	has, _ = m1.Has(ctx, b0.Cid())
 	require.True(t, has)

-	has, _ = m2.Has(b0.Cid())
+	has, _ = m2.Has(ctx, b0.Cid())
 	require.True(t, has)

-	has, _ = u.Has(b0.Cid())
+	has, _ = u.Has(ctx, b0.Cid())
 	require.True(t, has)

 	// put many.
-	err = u.PutMany([]blocks.Block{b1, b2})
+	err = u.PutMany(ctx, []blocks.Block{b1, b2})
 	require.NoError(t, err)

 	// write was broadcasted to all stores.
-	has, _ = m1.Has(b1.Cid())
+	has, _ = m1.Has(ctx, b1.Cid())
 	require.True(t, has)

-	has, _ = m1.Has(b2.Cid())
+	has, _ = m1.Has(ctx, b2.Cid())
 	require.True(t, has)

-	has, _ = m2.Has(b1.Cid())
+	has, _ = m2.Has(ctx, b1.Cid())
 	require.True(t, has)

-	has, _ = m2.Has(b2.Cid())
+	has, _ = m2.Has(ctx, b2.Cid())
 	require.True(t, has)

 	// also in the union store.
-	has, _ = u.Has(b1.Cid())
+	has, _ = u.Has(ctx, b1.Cid())
 	require.True(t, has)

-	has, _ = u.Has(b2.Cid())
+	has, _ = u.Has(ctx, b2.Cid())
 	require.True(t, has)

 	// deleted from all stores.
-	err = u.DeleteBlock(b1.Cid())
+	err = u.DeleteBlock(ctx, b1.Cid())
 	require.NoError(t, err)

-	has, _ = u.Has(b1.Cid())
+	has, _ = u.Has(ctx, b1.Cid())
 	require.False(t, has)

-	has, _ = m1.Has(b1.Cid())
+	has, _ = m1.Has(ctx, b1.Cid())
 	require.False(t, has)

-	has, _ = m2.Has(b1.Cid())
+	has, _ = m2.Has(ctx, b1.Cid())
 	require.False(t, has)

 	// check that AllKeysChan returns b0 and b2, twice (once per backing store)
@@ -1,2 +1,2 @@
-/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWBzv5sf4eTyo8cjJGfGnpxo6QkEPkRShG9GqjE2A5QaW5
-/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWBo9TSD4XXRFtu6snv6QNYvXgRaSaVb116YiYEsDWgKtq
+/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWFHDtFx7CVTy4xoCDutVo1cScvSnQjDeaM8UzwVS1qwkh
+/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWKt8cwpkiumkT8x32c3YFxsPRwhV5J8hCYPn9mhUmcAXt
@@ -1,2 +1,2 @@
-/dns4/bootstrap-0.interop.fildev.network/tcp/1347/p2p/12D3KooWLGPq9JL1xwL6gHok7HSNxtK1Q5kyfg4Hk69ifRPghn4i
-/dns4/bootstrap-1.interop.fildev.network/tcp/1347/p2p/12D3KooWFYS1f31zafv8mqqYu8U3hEqYvaZ6avWzYU3BmZdpyH3h
+/dns4/bootstrap-0.interop.fildev.network/tcp/1347/p2p/12D3KooWPfaWdJjpPTQp5kSqsqvauJaM4cFz5NWja7qyjx7vVfjL
+/dns4/bootstrap-1.interop.fildev.network/tcp/1347/p2p/12D3KooWS5tjcL6s4hHZ1HVRTgGKaXpKkiHfbBSJdjbzgejgJpEh
@@ -1,3 +1,4 @@
+//stm: #unit
 package build

 import (
@@ -7,6 +8,7 @@ import (
 )

 func TestOpenRPCDiscoverJSON_Version(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_OPENRPC_VERSION_001
 	// openRPCDocVersion is the current OpenRPC version of the API docs.
 	openRPCDocVersion := "1.2.6"

@@ -17,7 +17,7 @@ import (
 const BootstrappersFile = ""
 const GenesisFile = ""

-const GenesisNetworkVersion = network.Version14
+const GenesisNetworkVersion = network.Version15

 var UpgradeBreezeHeight = abi.ChainEpoch(-1)

@@ -47,6 +47,8 @@ var UpgradeHyperdriveHeight = abi.ChainEpoch(-16)

 var UpgradeChocolateHeight = abi.ChainEpoch(-17)

+var UpgradeOhSnapHeight = abi.ChainEpoch(-18)
+
 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 	0: DrandMainnet,
 }
@@ -88,6 +90,7 @@ func init() {
 	UpgradeTurboHeight = getUpgradeHeight("LOTUS_ACTORSV4_HEIGHT", UpgradeTurboHeight)
 	UpgradeHyperdriveHeight = getUpgradeHeight("LOTUS_HYPERDRIVE_HEIGHT", UpgradeHyperdriveHeight)
 	UpgradeChocolateHeight = getUpgradeHeight("LOTUS_CHOCOLATE_HEIGHT", UpgradeChocolateHeight)
+	UpgradeOhSnapHeight = getUpgradeHeight("LOTUS_OHSNAP_HEIGHT", UpgradeOhSnapHeight)

 	BuildType |= Build2k

@@ -16,7 +16,7 @@ var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 	0: DrandMainnet,
 }

-const GenesisNetworkVersion = network.Version13
+const GenesisNetworkVersion = network.Version14

 const BootstrappersFile = "butterflynet.pi"
 const GenesisFile = "butterflynet.car"
@@ -40,12 +40,16 @@ const UpgradeTrustHeight = -13
 const UpgradeNorwegianHeight = -14
 const UpgradeTurboHeight = -15
 const UpgradeHyperdriveHeight = -16
-const UpgradeChocolateHeight = 6360
+const UpgradeChocolateHeight = -17
+
+const UpgradeOhSnapHeight = 240

 func init() {
 	policy.SetConsensusMinerMinPower(abi.NewStoragePower(2 << 30))
 	policy.SetSupportedProofTypes(
 		abi.RegisteredSealProof_StackedDrg512MiBV1,
+		abi.RegisteredSealProof_StackedDrg32GiBV1,
+		abi.RegisteredSealProof_StackedDrg64GiBV1,
 	)

 	SetAddressNetwork(address.Testnet)
@@ -54,6 +54,9 @@ const UpgradeHyperdriveHeight = 420

 const UpgradeChocolateHeight = 312746

+// 2022-02-10T19:23:00Z
+const UpgradeOhSnapHeight = 682006
+
 func init() {
 	policy.SetConsensusMinerMinPower(abi.NewStoragePower(32 << 30))
 	policy.SetSupportedProofTypes(
@@ -13,13 +13,12 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/network"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
-	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
 )

 const BootstrappersFile = "interopnet.pi"
 const GenesisFile = "interopnet.car"

-const GenesisNetworkVersion = network.Version13
+const GenesisNetworkVersion = network.Version15

 var UpgradeBreezeHeight = abi.ChainEpoch(-1)

@@ -47,21 +46,25 @@ var UpgradeTurboHeight = abi.ChainEpoch(-15)

 var UpgradeHyperdriveHeight = abi.ChainEpoch(-16)
 var UpgradeChocolateHeight = abi.ChainEpoch(-17)
+var UpgradeOhSnapHeight = abi.ChainEpoch(-18)

 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 	0: DrandMainnet,
 }

 func init() {
-	policy.SetSupportedProofTypes(
-		abi.RegisteredSealProof_StackedDrg2KiBV1,
-		abi.RegisteredSealProof_StackedDrg8MiBV1,
-		abi.RegisteredSealProof_StackedDrg512MiBV1,
-	)
-	policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048))
 	policy.SetMinVerifiedDealSize(abi.NewStoragePower(256))
 	policy.SetPreCommitChallengeDelay(abi.ChainEpoch(10))

+	InsecurePoStValidation = true
+
+	policy.SetConsensusMinerMinPower(abi.NewStoragePower(2 << 30))
+	policy.SetSupportedProofTypes(
+		abi.RegisteredSealProof_StackedDrg512MiBV1,
+		abi.RegisteredSealProof_StackedDrg32GiBV1,
+		abi.RegisteredSealProof_StackedDrg64GiBV1,
+	)
+
 	getUpgradeHeight := func(ev string, def abi.ChainEpoch) abi.ChainEpoch {
 		hs, found := os.LookupEnv(ev)
 		if found {
@@ -92,6 +95,8 @@ func init() {
 	UpgradeNorwegianHeight = getUpgradeHeight("LOTUS_NORWEGIAN_HEIGHT", UpgradeNorwegianHeight)
 	UpgradeTurboHeight = getUpgradeHeight("LOTUS_ACTORSV4_HEIGHT", UpgradeTurboHeight)
 	UpgradeHyperdriveHeight = getUpgradeHeight("LOTUS_HYPERDRIVE_HEIGHT", UpgradeHyperdriveHeight)
+	UpgradeChocolateHeight = getUpgradeHeight("LOTUS_CHOCOLATE_HEIGHT", UpgradeChocolateHeight)
+	UpgradeOhSnapHeight = getUpgradeHeight("LOTUS_OHSNAP_HEIGHT", UpgradeOhSnapHeight)

 	BuildType |= BuildInteropnet
 	SetAddressNetwork(address.Testnet)
@@ -99,9 +104,9 @@ func init() {

 }

-const BlockDelaySecs = uint64(builtin2.EpochDurationSeconds)
+const BlockDelaySecs = uint64(5)

-const PropagationDelaySecs = uint64(6)
+const PropagationDelaySecs = uint64(1)

 // BootstrapPeerThreshold is the minimum number peers we need to track for a sync worker to start
 const BootstrapPeerThreshold = 2
@@ -62,18 +62,21 @@ const UpgradeNorwegianHeight = 665280
 const UpgradeTurboHeight = 712320

 // 2021-06-30T22:00:00Z
-var UpgradeHyperdriveHeight = abi.ChainEpoch(892800)
+const UpgradeHyperdriveHeight = 892800

 // 2021-10-26T13:30:00Z
-var UpgradeChocolateHeight = abi.ChainEpoch(1231620)
+const UpgradeChocolateHeight = 1231620
+
+// 2022-03-01T15:00:00Z
+var UpgradeOhSnapHeight = abi.ChainEpoch(1594680)

 func init() {
 	if os.Getenv("LOTUS_USE_TEST_ADDRESSES") != "1" {
 		SetAddressNetwork(address.Mainnet)
 	}

-	if os.Getenv("LOTUS_DISABLE_CHOCOLATE") == "1" {
-		UpgradeChocolateHeight = math.MaxInt64
+	if os.Getenv("LOTUS_DISABLE_SNAPDEALS") == "1" {
+		UpgradeOhSnapHeight = math.MaxInt64
 	}

 	Devnet = false
@@ -13,6 +13,17 @@ import (

 func BlocksTopic(netName dtypes.NetworkName) string   { return "/fil/blocks/" + string(netName) }
 func MessagesTopic(netName dtypes.NetworkName) string { return "/fil/msgs/" + string(netName) }
+func IndexerIngestTopic(netName dtypes.NetworkName) string {
+
+	nn := string(netName)
+	// The network name testnetnet is here for historical reasons.
+	// Going forward we aim to use the name `mainnet` where possible.
+	if nn == "testnetnet" {
+		nn = "mainnet"
+	}
+
+	return "/indexer/ingest/" + nn
+}
 func DhtProtocolName(netName dtypes.NetworkName) protocol.ID {
 	return protocol.ID("/fil/kad/" + string(netName))
 }
@@ -28,7 +28,15 @@ const UnixfsLinksPerLevel = 1024
 const AllowableClockDriftSecs = uint64(1)

 // TODO: This is still terrible...What's the impact of updating this before mainnet actually upgrades
-const NewestNetworkVersion = network.Version14
+/* inline-gen template
+
+const NewestNetworkVersion = network.Version{{.latestNetworkVersion}}
+
+/* inline-gen start */
+
+const NewestNetworkVersion = network.Version15
+
+/* inline-gen end */

 // Epochs
 const ForkLengthThreshold = Finality
@@ -99,6 +99,7 @@ var (
 	UpgradeTurboHeight      abi.ChainEpoch = -14
 	UpgradeHyperdriveHeight abi.ChainEpoch = -15
 	UpgradeChocolateHeight  abi.ChainEpoch = -16
+	UpgradeOhSnapHeight     abi.ChainEpoch = -17

 	DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 		0: DrandMainnet,
@@ -106,8 +107,8 @@ var (

 	GenesisNetworkVersion = network.Version0

-	NewestNetworkVersion       = network.Version14
-	ActorUpgradeNetworkVersion = network.Version4
+	NewestNetworkVersion       = network.Version15
+	ActorUpgradeNetworkVersion = network.Version15

 	Devnet      = true
 	ZeroAddress = MustParseAddress("f3yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaby2smx7a")
@@ -1,4 +1,54 @@
 {
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-61fa69f38b9cc771ba27b670124714b4ea77fbeae05e377fb859c4a43b73a30c.params": {
+    "cid": "Qma5WL6abSqYg9uUQAZ3EHS286bsNsha7oAGsJBD48Bq2q",
+    "digest": "c3ad7bb549470b82ad52ed070aebb4f4",
+    "sector_size": 536870912
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-61fa69f38b9cc771ba27b670124714b4ea77fbeae05e377fb859c4a43b73a30c.vk": {
+    "cid": "QmUa7f9JtJMsqJJ3s3ZXk6WyF4xJLE8FiqYskZGgk8GCDv",
+    "digest": "994c5b7d450ca9da348c910689f2dc7f",
+    "sector_size": 536870912
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-92180959e1918d26350b8e6cfe217bbdd0a2d8de51ebec269078b364b715ad63.params": {
+    "cid": "QmQiT4qBGodrVNEgVTDXxBNDdPbaD8Ag7Sx3ZTq1zHX79S",
+    "digest": "5aedd2cf3e5c0a15623d56a1b43110ad",
+    "sector_size": 8388608
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-92180959e1918d26350b8e6cfe217bbdd0a2d8de51ebec269078b364b715ad63.vk": {
+    "cid": "QmdcpKUQvHM8RFRVKbk1yHfEqMcBzhtFWKRp9SNEmWq37i",
+    "digest": "abd80269054d391a734febdac0d2e687",
+    "sector_size": 8388608
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-fb9e095bebdd77511c0269b967b4d87ba8b8a525edaa0e165de23ba454510194.params": {
+    "cid": "QmYM6Hg7mjmvA3ZHTsqkss1fkdyDju5dDmLiBZGJ5pz9y9",
+    "digest": "311f92a3e75036ced01b1c0025f1fa0c",
+    "sector_size": 2048
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-fb9e095bebdd77511c0269b967b4d87ba8b8a525edaa0e165de23ba454510194.vk": {
+    "cid": "QmaQsTLL3nc5dw6wAvaioJSBfd1jhQrA2o6ucFf7XeV74P",
+    "digest": "eadad9784969890d30f2749708c79771",
+    "sector_size": 2048
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-8-0-3b7f44a9362e3985369454947bc94022e118211e49fd672d52bec1cbfd599d18.params": {
+    "cid": "QmNPc75iEfcahCwNKdqnWLtxnjspUGGR4iscjiz3wP3RtS",
+    "digest": "1b3cfd761a961543f9eb273e435a06a2",
+    "sector_size": 34359738368
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-8-0-3b7f44a9362e3985369454947bc94022e118211e49fd672d52bec1cbfd599d18.vk": {
+    "cid": "QmdFFUe1gcz9MMHc6YW8aoV48w4ckvcERjt7PkydQAMfCN",
+    "digest": "3a6941983754737fde880d29c7094905",
+    "sector_size": 34359738368
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-8-2-102e1444a7e9a97ebf1e3d6855dcc77e66c011ea66f936d9b2c508f87f2f83a7.params": {
+    "cid": "QmUB6xTVjzBQGuDNeyJMrrJ1byk58vhPm8eY2Lv9pgwanp",
+    "digest": "1a392e7b759fb18e036c7559b5ece816",
+    "sector_size": 68719476736
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-8-2-102e1444a7e9a97ebf1e3d6855dcc77e66c011ea66f936d9b2c508f87f2f83a7.vk": {
+    "cid": "Qmd794Jty7k26XJ8Eg4NDEks65Qk8G4GVfGkwqvymv8HAg",
+    "digest": "80e366df2f1011953c2d01c7b7c9ee8e",
+    "sector_size": 68719476736
+  },
  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.params": {
    "cid": "QmVxjFRyhmyQaZEtCh7nk2abc7LhFkzhnRX4rcHqCCpikR",
    "digest": "7610b9f82bfc88405b7a832b651ce2f6",
@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }

 // BuildVersion is the local build version
-const BuildVersion = "1.13.0"
+const BuildVersion = "1.15.2-dev"

 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {
@@ -1,3 +1,4 @@
+//stm: #unit
 package adt

 import (
@@ -44,6 +45,7 @@ func TestDiffAdtArray(t *testing.T) {

 	changes := new(TestDiffArray)

+	//stm: @CHAIN_ADT_ARRAY_DIFF_001
 	assert.NoError(t, DiffAdtArray(arrA, arrB, changes))
 	assert.NotNil(t, changes)

@@ -98,6 +100,7 @@ func TestDiffAdtMap(t *testing.T) {

 	changes := new(TestDiffMap)

+	//stm: @CHAIN_ADT_MAP_DIFF_001
 	assert.NoError(t, DiffAdtMap(mapA, mapB, changes))
 	assert.NotNil(t, changes)

@@ -1,3 +1,4 @@
+//stm: #unit
 package aerrors_test

 import (
@@ -11,6 +12,7 @@ import (
 )

 func TestFatalError(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_ACTOR_ERRORS_001
 	e1 := xerrors.New("out of disk space")
 	e2 := xerrors.Errorf("could not put node: %w", e1)
 	e3 := xerrors.Errorf("could not save head: %w", e2)
@@ -24,6 +26,7 @@ func TestFatalError(t *testing.T) {
 	assert.True(t, IsFatal(aw4), "should be fatal")
 }
 func TestAbsorbeError(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_ACTOR_ERRORS_001
 	e1 := xerrors.New("EOF")
 	e2 := xerrors.Errorf("could not decode: %w", e1)
 	ae := Absorb(e2, 35, "failed to decode CBOR")
@@ -23,6 +23,8 @@ import (
 	builtin5 "github.com/filecoin-project/specs-actors/v5/actors/builtin"

 	builtin6 "github.com/filecoin-project/specs-actors/v6/actors/builtin"
+
+	builtin7 "github.com/filecoin-project/specs-actors/v7/actors/builtin"
 )

 func init() {
@@ -50,6 +52,10 @@ func init() {
 	builtin.RegisterActorState(builtin6.AccountActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) {
 		return load6(store, root)
 	})
+
+	builtin.RegisterActorState(builtin7.AccountActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) {
+		return load7(store, root)
+	})
 }

 var Methods = builtin4.MethodsAccount
@@ -75,6 +81,9 @@ func Load(store adt.Store, act *types.Actor) (State, error) {
 	case builtin6.AccountActorCodeID:
 		return load6(store, act.Head)

+	case builtin7.AccountActorCodeID:
+		return load7(store, act.Head)
+
 	}
 	return nil, xerrors.Errorf("unknown actor code %s", act.Code)
 }
@@ -100,6 +109,9 @@ func MakeState(store adt.Store, av actors.Version, addr address.Address) (State,
 	case actors.Version6:
 		return make6(store, addr)

+	case actors.Version7:
+		return make7(store, addr)
+
 	}
 	return nil, xerrors.Errorf("unknown actor version %d", av)
 }
@@ -125,6 +137,9 @@ func GetActorCodeID(av actors.Version) (cid.Cid, error) {
 	case actors.Version6:
 		return builtin6.AccountActorCodeID, nil

+	case actors.Version7:
+		return builtin7.AccountActorCodeID, nil
+
 	}

 	return cid.Undef, xerrors.Errorf("unknown actor version %d", av)
@@ -0,0 +1,40 @@
+package account
+
+import (
+	"github.com/filecoin-project/go-address"
+	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/lotus/chain/actors/adt"
+
+	account7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/account"
+)
+
+var _ State = (*state7)(nil)
+
+func load7(store adt.Store, root cid.Cid) (State, error) {
+	out := state7{store: store}
+	err := store.Get(store.Context(), root, &out)
+	if err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func make7(store adt.Store, addr address.Address) (State, error) {
+	out := state7{store: store}
+	out.State = account7.State{Address: addr}
+	return &out, nil
+}
+
+type state7 struct {
+	account7.State
+	store adt.Store
+}
+
+func (s *state7) PubkeyAddress() (address.Address, error) {
+	return s.Address, nil
+}
+
+func (s *state7) GetState() interface{} {
+	return &s.State
+}
@@ -23,46 +23,50 @@ import (
 	builtin6 "github.com/filecoin-project/specs-actors/v6/actors/builtin"
 	smoothing6 "github.com/filecoin-project/specs-actors/v6/actors/util/smoothing"

+	builtin7 "github.com/filecoin-project/specs-actors/v7/actors/builtin"
+	smoothing7 "github.com/filecoin-project/specs-actors/v7/actors/util/smoothing"
+
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/cbor"

 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/types"

-	miner6 "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner"
-	proof6 "github.com/filecoin-project/specs-actors/v6/actors/runtime/proof"
+	miner7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner"
+	proof7 "github.com/filecoin-project/specs-actors/v7/actors/runtime/proof"
 )

-var SystemActorAddr = builtin6.SystemActorAddr
-var BurntFundsActorAddr = builtin6.BurntFundsActorAddr
-var CronActorAddr = builtin6.CronActorAddr
+var SystemActorAddr = builtin7.SystemActorAddr
+var BurntFundsActorAddr = builtin7.BurntFundsActorAddr
+var CronActorAddr = builtin7.CronActorAddr
 var SaftAddress = makeAddress("t0122")
 var ReserveAddress = makeAddress("t090")
 var RootVerifierAddress = makeAddress("t080")

 var (
-	ExpectedLeadersPerEpoch = builtin6.ExpectedLeadersPerEpoch
+	ExpectedLeadersPerEpoch = builtin7.ExpectedLeadersPerEpoch
 )

 const (
-	EpochDurationSeconds = builtin6.EpochDurationSeconds
-	EpochsInDay          = builtin6.EpochsInDay
-	SecondsInDay         = builtin6.SecondsInDay
+	EpochDurationSeconds = builtin7.EpochDurationSeconds
+	EpochsInDay          = builtin7.EpochsInDay
+	SecondsInDay         = builtin7.SecondsInDay
 )

 const (
-	MethodSend        = builtin6.MethodSend
-	MethodConstructor = builtin6.MethodConstructor
+	MethodSend        = builtin7.MethodSend
+	MethodConstructor = builtin7.MethodConstructor
 )

 // These are all just type aliases across actor versions. In the future, that might change
 // and we might need to do something fancier.
-type SectorInfo = proof6.SectorInfo
-type PoStProof = proof6.PoStProof
+type SectorInfo = proof7.SectorInfo
+type ExtendedSectorInfo = proof7.ExtendedSectorInfo
+type PoStProof = proof7.PoStProof
 type FilterEstimate = smoothing0.FilterEstimate

 func QAPowerForWeight(size abi.SectorSize, duration abi.ChainEpoch, dealWeight, verifiedWeight abi.DealWeight) abi.StoragePower {
-	return miner6.QAPowerForWeight(size, duration, dealWeight, verifiedWeight)
+	return miner7.QAPowerForWeight(size, duration, dealWeight, verifiedWeight)
 }

 func FromV0FilterEstimate(v0 smoothing0.FilterEstimate) FilterEstimate {
@@ -101,6 +105,12 @@ func FromV6FilterEstimate(v6 smoothing6.FilterEstimate) FilterEstimate {

 }

+func FromV7FilterEstimate(v7 smoothing7.FilterEstimate) FilterEstimate {
+
+	return (FilterEstimate)(v7)
+
+}
+
 type ActorStateLoader func(store adt.Store, root cid.Cid) (cbor.Marshaler, error)

 var ActorStateLoaders = make(map[cid.Cid]ActorStateLoader)
@@ -138,6 +148,9 @@ func ActorNameByCode(c cid.Cid) string {
 	case builtin6.IsBuiltinActor(c):
 		return builtin6.ActorNameByCode(c)

+	case builtin7.IsBuiltinActor(c):
+		return builtin7.ActorNameByCode(c)
+
 	default:
 		return "<unknown>"
 	}
@@ -169,6 +182,10 @@ func IsBuiltinActor(c cid.Cid) bool {
 		return true
 	}

+	if builtin7.IsBuiltinActor(c) {
+		return true
+	}
+
 	return false
 }

@@ -198,6 +215,10 @@ func IsAccountActor(c cid.Cid) bool {
 		return true
 	}

+	if c == builtin7.AccountActorCodeID {
+		return true
+	}
+
 	return false
 }

@@ -227,6 +248,10 @@ func IsStorageMinerActor(c cid.Cid) bool {
 		return true
 	}

+	if c == builtin7.StorageMinerActorCodeID {
+		return true
+	}
+
 	return false
 }

@@ -256,6 +281,10 @@ func IsMultisigActor(c cid.Cid) bool {
 		return true
 	}

+	if c == builtin7.MultisigActorCodeID {
+		return true
+	}
+
 	return false
 }

@@ -285,6 +314,10 @@ func IsPaymentChannelActor(c cid.Cid) bool {
 		return true
 	}

+	if c == builtin7.PaymentChannelActorCodeID {
+		return true
+	}
+
 	return false
 }

@@ -45,6 +45,7 @@ const (
 // These are all just type aliases across actor versions. In the future, that might change
 // and we might need to do something fancier.
 type SectorInfo = proof{{.latestVersion}}.SectorInfo
+type ExtendedSectorInfo = proof{{.latestVersion}}.ExtendedSectorInfo
 type PoStProof = proof{{.latestVersion}}.PoStProof
 type FilterEstimate = smoothing0.FilterEstimate

@@ -17,6 +17,8 @@ import (
 	builtin5 "github.com/filecoin-project/specs-actors/v5/actors/builtin"

 	builtin6 "github.com/filecoin-project/specs-actors/v6/actors/builtin"
+
+	builtin7 "github.com/filecoin-project/specs-actors/v7/actors/builtin"
 )

 func MakeState(store adt.Store, av actors.Version) (State, error) {
@@ -40,6 +42,9 @@ func MakeState(store adt.Store, av actors.Version) (State, error) {
 	case actors.Version6:
 		return make6(store)

+	case actors.Version7:
+		return make7(store)
+
 	}
 	return nil, xerrors.Errorf("unknown actor version %d", av)
 }
@@ -65,14 +70,17 @@ func GetActorCodeID(av actors.Version) (cid.Cid, error) {
 	case actors.Version6:
 		return builtin6.CronActorCodeID, nil

+	case actors.Version7:
+		return builtin7.CronActorCodeID, nil
+
 	}

 	return cid.Undef, xerrors.Errorf("unknown actor version %d", av)
 }

 var (
-	Address = builtin6.CronActorAddr
-	Methods = builtin6.MethodsCron
+	Address = builtin7.CronActorAddr
+	Methods = builtin7.MethodsCron
 )

 type State interface {
@@ -0,0 +1,35 @@
+package cron
+
+import (
+	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/lotus/chain/actors/adt"
+
+	cron7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/cron"
+)
+
+var _ State = (*state7)(nil)
+
+func load7(store adt.Store, root cid.Cid) (State, error) {
+	out := state7{store: store}
+	err := store.Get(store.Context(), root, &out)
+	if err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func make7(store adt.Store) (State, error) {
+	out := state7{store: store}
+	out.State = *cron7.ConstructState(cron7.BuiltInEntries())
+	return &out, nil
+}
+
+type state7 struct {
+	cron7.State
+	store adt.Store
+}
+
+func (s *state7) GetState() interface{} {
+	return &s.State
+}
@@ -25,6 +25,8 @@ import (
 	builtin5 "github.com/filecoin-project/specs-actors/v5/actors/builtin"

 	builtin6 "github.com/filecoin-project/specs-actors/v6/actors/builtin"
+
+	builtin7 "github.com/filecoin-project/specs-actors/v7/actors/builtin"
 )

 func init() {
@@ -52,11 +54,15 @@ func init() {
 	builtin.RegisterActorState(builtin6.InitActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) {
 		return load6(store, root)
 	})
+
+	builtin.RegisterActorState(builtin7.InitActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) {
+		return load7(store, root)
+	})
 }

 var (
-	Address = builtin6.InitActorAddr
-	Methods = builtin6.MethodsInit
+	Address = builtin7.InitActorAddr
+	Methods = builtin7.MethodsInit
 )

 func Load(store adt.Store, act *types.Actor) (State, error) {
@@ -80,6 +86,9 @@ func Load(store adt.Store, act *types.Actor) (State, error) {
 	case builtin6.InitActorCodeID:
 		return load6(store, act.Head)

+	case builtin7.InitActorCodeID:
+		return load7(store, act.Head)
+
 	}
 	return nil, xerrors.Errorf("unknown actor code %s", act.Code)
 }
@@ -105,6 +114,9 @@ func MakeState(store adt.Store, av actors.Version, networkName string) (State, e
 	case actors.Version6:
 		return make6(store, networkName)

+	case actors.Version7:
+		return make7(store, networkName)
+
 	}
 	return nil, xerrors.Errorf("unknown actor version %d", av)
 }
@@ -130,6 +142,9 @@ func GetActorCodeID(av actors.Version) (cid.Cid, error) {
 	case actors.Version6:
 		return builtin6.InitActorCodeID, nil

+	case actors.Version7:
+		return builtin7.InitActorCodeID, nil
+
 	}

 	return cid.Undef, xerrors.Errorf("unknown actor version %d", av)
@@ -0,0 +1,114 @@
+package init
+
+import (
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/ipfs/go-cid"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/chain/actors/adt"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+
+	builtin7 "github.com/filecoin-project/specs-actors/v7/actors/builtin"
+
+	init7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/init"
+	adt7 "github.com/filecoin-project/specs-actors/v7/actors/util/adt"
+)
+
+var _ State = (*state7)(nil)
+
+func load7(store adt.Store, root cid.Cid) (State, error) {
+	out := state7{store: store}
+	err := store.Get(store.Context(), root, &out)
+	if err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func make7(store adt.Store, networkName string) (State, error) {
+	out := state7{store: store}
+
+	s, err := init7.ConstructState(store, networkName)
+	if err != nil {
+		return nil, err
+	}
+
+	out.State = *s
+
+	return &out, nil
+}
+
+type state7 struct {
+	init7.State
+	store adt.Store
+}
+
+func (s *state7) ResolveAddress(address address.Address) (address.Address, bool, error) {
+	return s.State.ResolveAddress(s.store, address)
+}
+
+func (s *state7) MapAddressToNewID(address address.Address) (address.Address, error) {
+	return s.State.MapAddressToNewID(s.store, address)
+}
+
+func (s *state7) ForEachActor(cb func(id abi.ActorID, address address.Address) error) error {
+	addrs, err := adt7.AsMap(s.store, s.State.AddressMap, builtin7.DefaultHamtBitwidth)
+	if err != nil {
+		return err
+	}
+	var actorID cbg.CborInt
+	return addrs.ForEach(&actorID, func(key string) error {
+		addr, err := address.NewFromBytes([]byte(key))
+		if err != nil {
+			return err
+		}
+		return cb(abi.ActorID(actorID), addr)
+	})
+}
+
+func (s *state7) NetworkName() (dtypes.NetworkName, error) {
+	return dtypes.NetworkName(s.State.NetworkName), nil
+}
+
+func (s *state7) SetNetworkName(name string) error {
+	s.State.NetworkName = name
+	return nil
+}
+
+func (s *state7) SetNextID(id abi.ActorID) error {
+	s.State.NextID = id
+	return nil
+}
+
+func (s *state7) Remove(addrs ...address.Address) (err error) {
+	m, err := adt7.AsMap(s.store, s.State.AddressMap, builtin7.DefaultHamtBitwidth)
+	if err != nil {
+		return err
+	}
+	for _, addr := range addrs {
+		if err = m.Delete(abi.AddrKey(addr)); err != nil {
+			return xerrors.Errorf("failed to delete entry for address: %s; err: %w", addr, err)
+		}
+	}
+	amr, err := m.Root()
+	if err != nil {
+		return xerrors.Errorf("failed to get address map root: %w", err)
+	}
+	s.State.AddressMap = amr
+	return nil
+}
+
+func (s *state7) SetAddressMap(mcid cid.Cid) error {
+	s.State.AddressMap = mcid
+	return nil
+}
+
+func (s *state7) AddressMap() (adt.Map, error) {
+	return adt7.AsMap(s.store, s.State.AddressMap, builtin7.DefaultHamtBitwidth)
+}
+
+func (s *state7) GetState() interface{} {
+	return &s.State
+}
@@ -25,6 +25,8 @@ import (

 	builtin6 "github.com/filecoin-project/specs-actors/v6/actors/builtin"

+	builtin7 "github.com/filecoin-project/specs-actors/v7/actors/builtin"
+
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -56,11 +58,15 @@ func init() {
 	builtin.RegisterActorState(builtin6.StorageMarketActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) {
 		return load6(store, root)
 	})
+
+	builtin.RegisterActorState(builtin7.StorageMarketActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) {
+		return load7(store, root)
+	})
 }

 var (
-	Address = builtin6.StorageMarketActorAddr
-	Methods = builtin6.MethodsMarket
+	Address = builtin7.StorageMarketActorAddr
+	Methods = builtin7.MethodsMarket
 )

 func Load(store adt.Store, act *types.Actor) (State, error) {
@@ -84,6 +90,9 @@ func Load(store adt.Store, act *types.Actor) (State, error) {
 	case builtin6.StorageMarketActorCodeID:
 		return load6(store, act.Head)

+	case builtin7.StorageMarketActorCodeID:
+		return load7(store, act.Head)
+
 	}
 	return nil, xerrors.Errorf("unknown actor code %s", act.Code)
 }
@@ -109,6 +118,9 @@ func MakeState(store adt.Store, av actors.Version) (State, error) {
 	case actors.Version6:
 		return make6(store)

+	case actors.Version7:
+		return make7(store)
+
 	}
 	return nil, xerrors.Errorf("unknown actor version %d", av)
 }
@@ -134,6 +146,9 @@ func GetActorCodeID(av actors.Version) (cid.Cid, error) {
 	case actors.Version6:
 		return builtin6.StorageMarketActorCodeID, nil

+	case actors.Version7:
+		return builtin7.StorageMarketActorCodeID, nil
+
 	}

 	return cid.Undef, xerrors.Errorf("unknown actor version %d", av)
@@ -211,6 +226,9 @@ func DecodePublishStorageDealsReturn(b []byte, nv network.Version) (PublishStora
 	case actors.Version6:
 		return decodePublishStorageDealsReturn6(b)

+	case actors.Version7:
+		return decodePublishStorageDealsReturn7(b)
+
 	}
 	return nil, xerrors.Errorf("unknown actor version %d", av)
 }
@@ -0,0 +1,252 @@
+package market
+
+import (
+	"bytes"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/ipfs/go-cid"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/chain/actors/adt"
+	"github.com/filecoin-project/lotus/chain/types"
+
+	market7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/market"
+	adt7 "github.com/filecoin-project/specs-actors/v7/actors/util/adt"
+)
+
+var _ State = (*state7)(nil)
+
+func load7(store adt.Store, root cid.Cid) (State, error) {
+	out := state7{store: store}
+	err := store.Get(store.Context(), root, &out)
+	if err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func make7(store adt.Store) (State, error) {
+	out := state7{store: store}
+
+	s, err := market7.ConstructState(store)
+	if err != nil {
+		return nil, err
+	}
+
+	out.State = *s
+
+	return &out, nil
+}
+
+type state7 struct {
+	market7.State
+	store adt.Store
+}
+
+func (s *state7) TotalLocked() (abi.TokenAmount, error) {
+	fml := types.BigAdd(s.TotalClientLockedCollateral, s.TotalProviderLockedCollateral)
+	fml = types.BigAdd(fml, s.TotalClientStorageFee)
+	return fml, nil
+}
+
+func (s *state7) BalancesChanged(otherState State) (bool, error) {
+	otherState7, ok := otherState.(*state7)
+	if !ok {
+		// there's no way to compare different versions of the state, so let's
+		// just say that means the state of balances has changed
+		return true, nil
+	}
+	return !s.State.EscrowTable.Equals(otherState7.State.EscrowTable) || !s.State.LockedTable.Equals(otherState7.State.LockedTable), nil
+}
+
+func (s *state7) StatesChanged(otherState State) (bool, error) {
+	otherState7, ok := otherState.(*state7)
+	if !ok {
+		// there's no way to compare different versions of the state, so let's
+		// just say that means the state of balances has changed
+		return true, nil
+	}
+	return !s.State.States.Equals(otherState7.State.States), nil
+}
+
+func (s *state7) States() (DealStates, error) {
+	stateArray, err := adt7.AsArray(s.store, s.State.States, market7.StatesAmtBitwidth)
+	if err != nil {
+		return nil, err
+	}
+	return &dealStates7{stateArray}, nil
+}
+
+func (s *state7) ProposalsChanged(otherState State) (bool, error) {
+	otherState7, ok := otherState.(*state7)
+	if !ok {
+		// there's no way to compare different versions of the state, so let's
+		// just say that means the state of balances has changed
+		return true, nil
+	}
+	return !s.State.Proposals.Equals(otherState7.State.Proposals), nil
+}
+
+func (s *state7) Proposals() (DealProposals, error) {
+	proposalArray, err := adt7.AsArray(s.store, s.State.Proposals, market7.ProposalsAmtBitwidth)
+	if err != nil {
+		return nil, err
+	}
+	return &dealProposals7{proposalArray}, nil
+}
+
+func (s *state7) EscrowTable() (BalanceTable, error) {
+	bt, err := adt7.AsBalanceTable(s.store, s.State.EscrowTable)
+	if err != nil {
+		return nil, err
+	}
+	return &balanceTable7{bt}, nil
+}
+
+func (s *state7) LockedTable() (BalanceTable, error) {
+	bt, err := adt7.AsBalanceTable(s.store, s.State.LockedTable)
+	if err != nil {
+		return nil, err
+	}
+	return &balanceTable7{bt}, nil
+}
+
+func (s *state7) VerifyDealsForActivation(
+	minerAddr address.Address, deals []abi.DealID, currEpoch, sectorExpiry abi.ChainEpoch,
+) (weight, verifiedWeight abi.DealWeight, err error) {
+	w, vw, _, err := market7.ValidateDealsForActivation(&s.State, s.store, deals, minerAddr, sectorExpiry, currEpoch)
+	return w, vw, err
+}
+
+func (s *state7) NextID() (abi.DealID, error) {
+	return s.State.NextID, nil
+}
+
+type balanceTable7 struct {
+	*adt7.BalanceTable
+}
+
+func (bt *balanceTable7) ForEach(cb func(address.Address, abi.TokenAmount) error) error {
+	asMap := (*adt7.Map)(bt.BalanceTable)
+	var ta abi.TokenAmount
+	return asMap.ForEach(&ta, func(key string) error {
+		a, err := address.NewFromBytes([]byte(key))
+		if err != nil {
+			return err
+		}
+		return cb(a, ta)
+	})
+}
+
+type dealStates7 struct {
+	adt.Array
+}
+
+func (s *dealStates7) Get(dealID abi.DealID) (*DealState, bool, error) {
+	var deal7 market7.DealState
+	found, err := s.Array.Get(uint64(dealID), &deal7)
+	if err != nil {
+		return nil, false, err
+	}
+	if !found {
+		return nil, false, nil
+	}
+	deal := fromV7DealState(deal7)
+	return &deal, true, nil
+}
+
+func (s *dealStates7) ForEach(cb func(dealID abi.DealID, ds DealState) error) error {
+	var ds7 market7.DealState
+	return s.Array.ForEach(&ds7, func(idx int64) error {
+		return cb(abi.DealID(idx), fromV7DealState(ds7))
+	})
+}
+
+func (s *dealStates7) decode(val *cbg.Deferred) (*DealState, error) {
+	var ds7 market7.DealState
+	if err := ds7.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil {
+		return nil, err
+	}
+	ds := fromV7DealState(ds7)
+	return &ds, nil
+}
+
+func (s *dealStates7) array() adt.Array {
+	return s.Array
+}
+
+func fromV7DealState(v7 market7.DealState) DealState {
+	return (DealState)(v7)
+}
+
+type dealProposals7 struct {
+	adt.Array
+}
+
+func (s *dealProposals7) Get(dealID abi.DealID) (*DealProposal, bool, error) {
+	var proposal7 market7.DealProposal
+	found, err := s.Array.Get(uint64(dealID), &proposal7)
+	if err != nil {
+		return nil, false, err
+	}
+	if !found {
+		return nil, false, nil
+	}
+	proposal := fromV7DealProposal(proposal7)
+	return &proposal, true, nil
+}
+
+func (s *dealProposals7) ForEach(cb func(dealID abi.DealID, dp DealProposal) error) error {
+	var dp7 market7.DealProposal
+	return s.Array.ForEach(&dp7, func(idx int64) error {
+		return cb(abi.DealID(idx), fromV7DealProposal(dp7))
+	})
+}
+
+func (s *dealProposals7) decode(val *cbg.Deferred) (*DealProposal, error) {
+	var dp7 market7.DealProposal
+	if err := dp7.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil {
+		return nil, err
+	}
+	dp := fromV7DealProposal(dp7)
+	return &dp, nil
+}
+
+func (s *dealProposals7) array() adt.Array {
+	return s.Array
+}
+
+func fromV7DealProposal(v7 market7.DealProposal) DealProposal {
+	return (DealProposal)(v7)
+}
+
+func (s *state7) GetState() interface{} {
+	return &s.State
+}
+
+var _ PublishStorageDealsReturn = (*publishStorageDealsReturn7)(nil)
+
+func decodePublishStorageDealsReturn7(b []byte) (PublishStorageDealsReturn, error) {
+	var retval market7.PublishStorageDealsReturn
+	if err := retval.UnmarshalCBOR(bytes.NewReader(b)); err != nil {
+		return nil, xerrors.Errorf("failed to unmarshal PublishStorageDealsReturn: %w", err)
+	}
+
+	return &publishStorageDealsReturn7{retval}, nil
+}
+
+type publishStorageDealsReturn7 struct {
+	market7.PublishStorageDealsReturn
+}
+
+func (r *publishStorageDealsReturn7) IsDealValid(index uint64) (bool, error) {
+
+	return r.ValidDeals.IsSet(index)
+
+}
+
+func (r *publishStorageDealsReturn7) DealIDs() ([]abi.DealID, error) {
+	return r.IDs, nil
+}
@@ -23,6 +23,7 @@ import (
 	miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner"
 	miner3 "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner"
 	miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
+	miner7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner"
 {{range .versions}}
 	builtin{{.}} "github.com/filecoin-project/specs-actors{{import .}}actors/builtin"
 {{end}}
@@ -177,6 +178,7 @@ type SectorOnChainInfo struct {
 	InitialPledge         abi.TokenAmount
 	ExpectedDayReward     abi.TokenAmount
 	ExpectedStoragePledge abi.TokenAmount
+	SectorKeyCID          *cid.Cid
 }

 type SectorPreCommitInfo = miner0.SectorPreCommitInfo
@@ -192,6 +194,7 @@ type SectorPreCommitOnChainInfo struct {
 type PoStPartition = miner0.PoStPartition
 type RecoveryDeclaration = miner0.RecoveryDeclaration
 type FaultDeclaration = miner0.FaultDeclaration
+type ReplicaUpdate = miner7.ReplicaUpdate

 // Params
 type DeclareFaultsParams = miner0.DeclareFaultsParams
@@ -200,6 +203,7 @@ type SubmitWindowedPoStParams = miner0.SubmitWindowedPoStParams
 type ProveCommitSectorParams = miner0.ProveCommitSectorParams
 type DisputeWindowedPoStParams = miner3.DisputeWindowedPoStParams
 type ProveCommitAggregateParams = miner5.ProveCommitAggregateParams
+type ProveReplicaUpdatesParams = miner7.ProveReplicaUpdatesParams

 func PreferredSealProofTypeFromWindowPoStType(nver network.Version, proof abi.RegisteredPoStProof) (abi.RegisteredSealProof, error) {
 	// We added support for the new proofs in network version 7, and removed support for the old
@@ -23,6 +23,7 @@ import (
 	miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner"
 	miner3 "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner"
 	miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
+	miner7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner"

 	builtin0 "github.com/filecoin-project/specs-actors/actors/builtin"

@@ -35,6 +36,8 @@ import (
 	builtin5 "github.com/filecoin-project/specs-actors/v5/actors/builtin"

 	builtin6 "github.com/filecoin-project/specs-actors/v6/actors/builtin"
+
+	builtin7 "github.com/filecoin-project/specs-actors/v7/actors/builtin"
 )

 func init() {
@@ -63,9 +66,13 @@ func init() {
 		return load6(store, root)
 	})

+	builtin.RegisterActorState(builtin7.StorageMinerActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) {
+		return load7(store, root)
+	})
+
 }

-var Methods = builtin6.MethodsMiner
+var Methods = builtin7.MethodsMiner

 // Unchanged between v0, v2, v3, v4, and v5 actors
 var WPoStProvingPeriod = miner0.WPoStProvingPeriod
@@ -102,6 +109,9 @@ func Load(store adt.Store, act *types.Actor) (State, error) {
 	case builtin6.StorageMinerActorCodeID:
 		return load6(store, act.Head)

+	case builtin7.StorageMinerActorCodeID:
+		return load7(store, act.Head)
+
 	}
 	return nil, xerrors.Errorf("unknown actor code %s", act.Code)
 }
@@ -127,6 +137,9 @@ func MakeState(store adt.Store, av actors.Version) (State, error) {
 	case actors.Version6:
 		return make6(store)

+	case actors.Version7:
+		return make7(store)
+
 	}
 	return nil, xerrors.Errorf("unknown actor version %d", av)
 }
@@ -152,6 +165,9 @@ func GetActorCodeID(av actors.Version) (cid.Cid, error) {
 	case actors.Version6:
 		return builtin6.StorageMinerActorCodeID, nil

+	case actors.Version7:
+		return builtin7.StorageMinerActorCodeID, nil
+
 	}

 	return cid.Undef, xerrors.Errorf("unknown actor version %d", av)
@@ -251,6 +267,7 @@ type SectorOnChainInfo struct {
 	InitialPledge         abi.TokenAmount
 	ExpectedDayReward     abi.TokenAmount
 	ExpectedStoragePledge abi.TokenAmount
+	SectorKeyCID          *cid.Cid
 }

 type SectorPreCommitInfo = miner0.SectorPreCommitInfo
@@ -266,6 +283,7 @@ type SectorPreCommitOnChainInfo struct {
 type PoStPartition = miner0.PoStPartition
 type RecoveryDeclaration = miner0.RecoveryDeclaration
 type FaultDeclaration = miner0.FaultDeclaration
+type ReplicaUpdate = miner7.ReplicaUpdate

 // Params
 type DeclareFaultsParams = miner0.DeclareFaultsParams
@@ -274,6 +292,7 @@ type SubmitWindowedPoStParams = miner0.SubmitWindowedPoStParams
 type ProveCommitSectorParams = miner0.ProveCommitSectorParams
 type DisputeWindowedPoStParams = miner3.DisputeWindowedPoStParams
 type ProveCommitAggregateParams = miner5.ProveCommitAggregateParams
+type ProveReplicaUpdatesParams = miner7.ProveReplicaUpdatesParams

 func PreferredSealProofTypeFromWindowPoStType(nver network.Version, proof abi.RegisteredPoStProof) (abi.RegisteredSealProof, error) {
 	// We added support for the new proofs in network version 7, and removed support for the old
@@ -138,11 +138,22 @@ func (s *state{{.v}}) GetSectorExpiration(num abi.SectorNumber) (*SectorExpirati
 		return nil, err
 	}
 	// NOTE: this can be optimized significantly.
-	// 1. If the sector is non-faulty, it will either expire on-time (can be
+{{if (ge .v 7) -}}
+    // 1. If the sector is non-faulty, it will expire on-time (can be
+	// learned from the sector info).
+{{- else -}}
+    // 1. If the sector is non-faulty, it will either expire on-time (can be
 	// learned from the sector info), or in the next quantized expiration
 	// epoch (i.e., the first element in the partition's expiration queue.
+{{- end}}
+{{if (ge .v 6) -}}
+	// 2. If it's faulty, it will expire early within the first 42 entries
+	// of the expiration queue.
+{{- else -}}
 	// 2. If it's faulty, it will expire early within the first 14 entries
 	// of the expiration queue.
+{{- end}}
+
 	stopErr := errors.New("stop")
 	out := SectorExpiration{}
 	err = dls.ForEach(s.store, func(dlIdx uint64, dl *miner{{.v}}.Deadline) error {
@@ -554,8 +565,7 @@ func (p *partition{{.v}}) UnprovenSectors() (bitfield.BitField, error) {
 }

 func fromV{{.v}}SectorOnChainInfo(v{{.v}} miner{{.v}}.SectorOnChainInfo) SectorOnChainInfo {
-{{if (ge .v 2)}}
-	return SectorOnChainInfo{
+	info := SectorOnChainInfo{
 		SectorNumber:          v{{.v}}.SectorNumber,
 		SealProof:             v{{.v}}.SealProof,
 		SealedCID:             v{{.v}}.SealedCID,
@@ -567,10 +577,11 @@ func fromV{{.v}}SectorOnChainInfo(v{{.v}} miner{{.v}}.SectorOnChainInfo) SectorO
 		InitialPledge:         v{{.v}}.InitialPledge,
 		ExpectedDayReward:     v{{.v}}.ExpectedDayReward,
 		ExpectedStoragePledge: v{{.v}}.ExpectedStoragePledge,
+		{{if (ge .v 7)}}
+		SectorKeyCID: v{{.v}}.SectorKeyCID,
+        {{end}}
 	}
-{{else}}
-	return (SectorOnChainInfo)(v0)
-{{end}}
+	return info
 }

 func fromV{{.v}}SectorPreCommitOnChainInfo(v{{.v}} miner{{.v}}.SectorPreCommitOnChainInfo) SectorPreCommitOnChainInfo {
@@ -67,3 +67,22 @@ func SealProofTypeFromSectorSize(ssize abi.SectorSize, nv network.Version) (abi.

 	return 0, xerrors.Errorf("unsupported network version")
 }
+
+// WindowPoStProofTypeFromSectorSize returns preferred post proof type for creating
+// new miner actors and new sectors
+func WindowPoStProofTypeFromSectorSize(ssize abi.SectorSize) (abi.RegisteredPoStProof, error) {
+	switch ssize {
+	case 2 << 10:
+		return abi.RegisteredPoStProof_StackedDrgWindow2KiBV1, nil
+	case 8 << 20:
+		return abi.RegisteredPoStProof_StackedDrgWindow8MiBV1, nil
+	case 512 << 20:
+		return abi.RegisteredPoStProof_StackedDrgWindow512MiBV1, nil
+	case 32 << 30:
+		return abi.RegisteredPoStProof_StackedDrgWindow32GiBV1, nil
+	case 64 << 30:
+		return abi.RegisteredPoStProof_StackedDrgWindow64GiBV1, nil
+	default:
+		return 0, xerrors.Errorf("unsupported sector size for miner: %v", ssize)
+	}
+}
@@ -140,6 +140,7 @@ func (s *state0) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, e
 	// epoch (i.e., the first element in the partition's expiration queue.
 	// 2. If it's faulty, it will expire early within the first 14 entries
 	// of the expiration queue.
+
 	stopErr := errors.New("stop")
 	out := SectorExpiration{}
 	err = dls.ForEach(s.store, func(dlIdx uint64, dl *miner0.Deadline) error {
@@ -505,9 +506,20 @@ func (p *partition0) UnprovenSectors() (bitfield.BitField, error) {
 }

 func fromV0SectorOnChainInfo(v0 miner0.SectorOnChainInfo) SectorOnChainInfo {
-
-	return (SectorOnChainInfo)(v0)
-
+	info := SectorOnChainInfo{
+		SectorNumber:          v0.SectorNumber,
+		SealProof:             v0.SealProof,
+		SealedCID:             v0.SealedCID,
+		DealIDs:               v0.DealIDs,
+		Activation:            v0.Activation,
+		Expiration:            v0.Expiration,
+		DealWeight:            v0.DealWeight,
+		VerifiedDealWeight:    v0.VerifiedDealWeight,
+		InitialPledge:         v0.InitialPledge,
+		ExpectedDayReward:     v0.ExpectedDayReward,
+		ExpectedStoragePledge: v0.ExpectedStoragePledge,
+	}
+	return info
 }

 func fromV0SectorPreCommitOnChainInfo(v0 miner0.SectorPreCommitOnChainInfo) SectorPreCommitOnChainInfo {
@@ -138,6 +138,7 @@ func (s *state2) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, e
 	// epoch (i.e., the first element in the partition's expiration queue.
 	// 2. If it's faulty, it will expire early within the first 14 entries
 	// of the expiration queue.
+
 	stopErr := errors.New("stop")
 	out := SectorExpiration{}
 	err = dls.ForEach(s.store, func(dlIdx uint64, dl *miner2.Deadline) error {
@@ -535,8 +536,7 @@ func (p *partition2) UnprovenSectors() (bitfield.BitField, error) {
 }

 func fromV2SectorOnChainInfo(v2 miner2.SectorOnChainInfo) SectorOnChainInfo {
-
-	return SectorOnChainInfo{
+	info := SectorOnChainInfo{
 		SectorNumber:          v2.SectorNumber,
 		SealProof:             v2.SealProof,
 		SealedCID:             v2.SealedCID,
@@ -549,7 +549,7 @@ func fromV2SectorOnChainInfo(v2 miner2.SectorOnChainInfo) SectorOnChainInfo {
 		ExpectedDayReward:     v2.ExpectedDayReward,
 		ExpectedStoragePledge: v2.ExpectedStoragePledge,
 	}
-
+	return info
 }

 func fromV2SectorPreCommitOnChainInfo(v2 miner2.SectorPreCommitOnChainInfo) SectorPreCommitOnChainInfo {
@@ -140,6 +140,7 @@ func (s *state3) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, e
 	// epoch (i.e., the first element in the partition's expiration queue.
 	// 2. If it's faulty, it will expire early within the first 14 entries
 	// of the expiration queue.
+
 	stopErr := errors.New("stop")
 	out := SectorExpiration{}
 	err = dls.ForEach(s.store, func(dlIdx uint64, dl *miner3.Deadline) error {
@@ -536,8 +537,7 @@ func (p *partition3) UnprovenSectors() (bitfield.BitField, error) {
 }

 func fromV3SectorOnChainInfo(v3 miner3.SectorOnChainInfo) SectorOnChainInfo {
-
-	return SectorOnChainInfo{
+	info := SectorOnChainInfo{
 		SectorNumber:          v3.SectorNumber,
 		SealProof:             v3.SealProof,
 		SealedCID:             v3.SealedCID,
@@ -550,7 +550,7 @@ func fromV3SectorOnChainInfo(v3 miner3.SectorOnChainInfo) SectorOnChainInfo {
 		ExpectedDayReward:     v3.ExpectedDayReward,
 		ExpectedStoragePledge: v3.ExpectedStoragePledge,
 	}
-
+	return info
 }

 func fromV3SectorPreCommitOnChainInfo(v3 miner3.SectorPreCommitOnChainInfo) SectorPreCommitOnChainInfo {
@@ -140,6 +140,7 @@ func (s *state4) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, e
 	// epoch (i.e., the first element in the partition's expiration queue.
 	// 2. If it's faulty, it will expire early within the first 14 entries
 	// of the expiration queue.
+
 	stopErr := errors.New("stop")
 	out := SectorExpiration{}
 	err = dls.ForEach(s.store, func(dlIdx uint64, dl *miner4.Deadline) error {
@@ -536,8 +537,7 @@ func (p *partition4) UnprovenSectors() (bitfield.BitField, error) {
 }

 func fromV4SectorOnChainInfo(v4 miner4.SectorOnChainInfo) SectorOnChainInfo {
-
-	return SectorOnChainInfo{
+	info := SectorOnChainInfo{
 		SectorNumber:          v4.SectorNumber,
 		SealProof:             v4.SealProof,
 		SealedCID:             v4.SealedCID,
@@ -550,7 +550,7 @@ func fromV4SectorOnChainInfo(v4 miner4.SectorOnChainInfo) SectorOnChainInfo {
 		ExpectedDayReward:     v4.ExpectedDayReward,
 		ExpectedStoragePledge: v4.ExpectedStoragePledge,
 	}
-
+	return info
 }

 func fromV4SectorPreCommitOnChainInfo(v4 miner4.SectorPreCommitOnChainInfo) SectorPreCommitOnChainInfo {
@@ -140,6 +140,7 @@ func (s *state5) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, e
 	// epoch (i.e., the first element in the partition's expiration queue.
 	// 2. If it's faulty, it will expire early within the first 14 entries
 	// of the expiration queue.
+
 	stopErr := errors.New("stop")
 	out := SectorExpiration{}
 	err = dls.ForEach(s.store, func(dlIdx uint64, dl *miner5.Deadline) error {
@@ -536,8 +537,7 @@ func (p *partition5) UnprovenSectors() (bitfield.BitField, error) {
 }

 func fromV5SectorOnChainInfo(v5 miner5.SectorOnChainInfo) SectorOnChainInfo {
-
-	return SectorOnChainInfo{
+	info := SectorOnChainInfo{
 		SectorNumber:          v5.SectorNumber,
 		SealProof:             v5.SealProof,
 		SealedCID:             v5.SealedCID,
@@ -550,7 +550,7 @@ func fromV5SectorOnChainInfo(v5 miner5.SectorOnChainInfo) SectorOnChainInfo {
 		ExpectedDayReward:     v5.ExpectedDayReward,
 		ExpectedStoragePledge: v5.ExpectedStoragePledge,
 	}
-
+	return info
 }

 func fromV5SectorPreCommitOnChainInfo(v5 miner5.SectorPreCommitOnChainInfo) SectorPreCommitOnChainInfo {
--- a/Show More
+++ b/Show More