Merge pull request #8482 from filecoin-project/jen/11521

build: release: v1.15.2-rc1
v1.15.2-rc1
2022-04-13 19:29:24 +02:00 · 2022-04-13 19:11:02 +02:00 · 2022-04-12 16:12:03 +02:00 · 2022-04-12 15:41:36 +02:00 · 2022-04-12 15:29:54 +02:00 · 2022-04-12 13:53:03 +02:00
384 changed files with 18985 additions and 3521 deletions
@@ -2,7 +2,6 @@ version: 2.1
 orbs:
  go: gotest/tools@0.0.13
  aws-cli: circleci/aws-cli@1.3.2
-  packer: salaxander/packer@0.0.3

 executors:
  golang:
@@ -12,6 +11,16 @@ executors:
  ubuntu:
    docker:
      - image: ubuntu:20.04
+  packer:
+    description: |
+      The HashiCorp provided Packer container
+    parameters:
+      packer-version:
+        type: string
+        default: "1.8"
+    docker:
+      - image: hashicorp/packer:<< parameters.packer-version >>
+

 commands:
  install-deps:
@@ -44,13 +53,13 @@ commands:
      - restore_cache:
          name: Restore parameters cache
          keys:
-            - 'v25-2k-lotus-params'
+            - 'v26-2k-lotus-params'
          paths:
            - /var/tmp/filecoin-proof-parameters/
      - run:  ./lotus fetch-params 2048
      - save_cache:
          name: Save parameters cache
-          key: 'v25-2k-lotus-params'
+          key: 'v26-2k-lotus-params'
          paths:
            - /var/tmp/filecoin-proof-parameters/
  install_ipfs:
@@ -75,6 +84,26 @@ commands:
          command: |
            git fetch --all

+  packer_build:
+    description: "Run a packer build"
+    parameters:
+      template:
+        description: |
+          The name of the packer template file
+        type: string
+        default: packer.json
+      args:
+        description: |
+          Arguments to pass to the packer build command
+        type: string
+        default: ""
+
+    steps:
+      - run:
+          name: "Run a packer build"
+          command: packer build << parameters.args >> << parameters.template >>
+          no_output_timeout: 30m
+
 jobs:
  mod-tidy-check:
    executor: golang
@@ -390,7 +419,7 @@ jobs:

  build-appimage:
    machine:
-      image: ubuntu-2004:202104-01
+      image: ubuntu-2004:202111-02
    steps:
      - checkout
      - attach_workspace:
@@ -398,6 +427,16 @@ jobs:
      - run:
          name: install appimage-builder
          command: |
+            # appimage-builder requires /dev/snd to exist. It creates containers during the testing phase
+            # that pass sound devices from the host to the testing container. (hard coded!)
+            # https://github.com/AppImageCrafters/appimage-builder/blob/master/appimagebuilder/modules/test/execution_test.py#L54
+            # Circleci doesn't provide a working sound device; this is enough to fake it.
+            if [ ! -e /dev/snd ]
+            then
+              sudo mkdir /dev/snd
+              sudo mknod /dev/snd/ControlC0 c 1 2
+            fi
+
            # docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html
            sudo apt update
            sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace
@@ -618,6 +657,11 @@ jobs:
        default: "latest"
        description: A comma-separated string containing docker image tags to build and push (default = latest)

+      target:
+        type: string
+        default: "lotus-all-in-one"
+        description: Docker target to build
+
    steps:
      - run:
          name: Confirm that environment variables are set
@@ -657,6 +701,7 @@ jobs:

            docker build \
              <<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
+              --target <<parameters.target>> \
              -f <<parameters.path>>/<<parameters.dockerfile>> \
              $docker_tag_args \
              <<parameters.path>>
@@ -669,42 +714,16 @@ jobs:
              docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
            done

-  publish-packer-mainnet:
-    description: build and push AWS IAM and DigitalOcean droplet.
+  publish-packer-snap:
+    description: build packer image with snap. mainnet only.
    executor:
-      name: packer/default
-      packer-version: 1.6.6
+      name: packer
    steps:
      - checkout
      - attach_workspace:
          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
-  publish-packer-calibrationnet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
-  publish-packer-butterflynet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG"
+      - packer_build:
+          template: tools/packer/lotus-snap.pkr.hcl
  publish-dockerhub:
    description: publish to dockerhub
    machine:
@@ -855,6 +874,11 @@ workflows:
          suite: itest-get_messages_in_ts
          target: "./itests/get_messages_in_ts_test.go"
      
+      - test:
+          name: test-itest-mempool
+          suite: itest-mempool
+          target: "./itests/mempool_test.go"
+      
      - test:
          name: test-itest-multisig
          suite: itest-multisig
@@ -885,6 +909,11 @@ workflows:
          suite: itest-sector_finalize_early
          target: "./itests/sector_finalize_early_test.go"
      
+      - test:
+          name: test-itest-sector_make_cc_avail
+          suite: itest-sector_make_cc_avail
+          target: "./itests/sector_make_cc_avail_test.go"
+      
      - test:
          name: test-itest-sector_miner_collateral
          suite: itest-sector_miner_collateral
@@ -895,6 +924,16 @@ workflows:
          suite: itest-sector_pledge
          target: "./itests/sector_pledge_test.go"
      
+      - test:
+          name: test-itest-sector_prefer_no_upgrade
+          suite: itest-sector_prefer_no_upgrade
+          target: "./itests/sector_prefer_no_upgrade_test.go"
+      
+      - test:
+          name: test-itest-sector_revert_available
+          suite: itest-sector_revert_available
+          target: "./itests/sector_revert_available_test.go"
+      
      - test:
          name: test-itest-sector_terminate
          suite: itest-sector_terminate
@@ -925,6 +964,11 @@ workflows:
          suite: itest-wdpost
          target: "./itests/wdpost_test.go"
      
+      - test:
+          name: test-itest-worker
+          suite: itest-worker
+          target: "./itests/worker_test.go"
+      
      - test:
          name: test-unit-cli
          suite: utest-unit-cli
@@ -986,10 +1030,19 @@ workflows:
            tags:
              only:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-appimage:
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
      - publish:
          requires:
            - build-all
            - build-macos
+            - build-appimage
          filters:
            branches:
              ignore:
@@ -1002,36 +1055,13 @@ workflows:
          path: .
          repo: lotus-dev
          tag: '${CIRCLE_SHA1:0:8}'
-      - publish-packer-mainnet:
-          requires:
-            - build-all
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-calibrationnet:
-          requires:
-            - build-ntwk-calibration
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-butterflynet:
-          requires:
-            - build-ntwk-butterfly
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+          target: lotus-all-in-one
+      - build-and-push-image:
+          dockerfile: Dockerfile.lotus
+          path: .
+          repo: lotus-test
+          tag: '${CIRCLE_SHA1:0:8}'
+          target: lotus-test
      - publish-snapcraft:
          name: publish-snapcraft-stable
          channel: stable
@@ -1068,3 +1098,13 @@ workflows:
      - publish-dockerhub:
          name: publish-dockerhub-nightly
          tag: nightly
+  monthly:
+    triggers:
+      - schedule:
+          cron: "0 0 1 * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - publish-packer-snap
@@ -2,7 +2,6 @@ version: 2.1
 orbs:
  go: gotest/tools@0.0.13
  aws-cli: circleci/aws-cli@1.3.2
-  packer: salaxander/packer@0.0.3

 executors:
  golang:
@@ -12,6 +11,16 @@ executors:
  ubuntu:
    docker:
      - image: ubuntu:20.04
+  packer:
+    description: |
+      The HashiCorp provided Packer container
+    parameters:
+      packer-version:
+        type: string
+        default: "1.8"
+    docker:
+      - image: hashicorp/packer:<< parameters.packer-version >>
+

 commands:
  install-deps:
@@ -44,13 +53,13 @@ commands:
      - restore_cache:
          name: Restore parameters cache
          keys:
-            - 'v25-2k-lotus-params'
+            - 'v26-2k-lotus-params'
          paths:
            - /var/tmp/filecoin-proof-parameters/
      - run:  ./lotus fetch-params 2048
      - save_cache:
          name: Save parameters cache
-          key: 'v25-2k-lotus-params'
+          key: 'v26-2k-lotus-params'
          paths:
            - /var/tmp/filecoin-proof-parameters/
  install_ipfs:
@@ -75,6 +84,26 @@ commands:
          command: |
            git fetch --all

+  packer_build:
+    description: "Run a packer build"
+    parameters:
+      template:
+        description: |
+          The name of the packer template file
+        type: string
+        default: packer.json
+      args:
+        description: |
+          Arguments to pass to the packer build command
+        type: string
+        default: ""
+
+    steps:
+      - run:
+          name: "Run a packer build"
+          command: packer build << parameters.args >> << parameters.template >>
+          no_output_timeout: 30m
+
 jobs:
  mod-tidy-check:
    executor: golang
@@ -390,7 +419,7 @@ jobs:

  build-appimage:
    machine:
-      image: ubuntu-2004:202104-01
+      image: ubuntu-2004:202111-02
    steps:
      - checkout
      - attach_workspace:
@@ -398,6 +427,16 @@ jobs:
      - run:
          name: install appimage-builder
          command: |
+            # appimage-builder requires /dev/snd to exist. It creates containers during the testing phase
+            # that pass sound devices from the host to the testing container. (hard coded!)
+            # https://github.com/AppImageCrafters/appimage-builder/blob/master/appimagebuilder/modules/test/execution_test.py#L54
+            # Circleci doesn't provide a working sound device; this is enough to fake it.
+            if [ ! -e /dev/snd ]
+            then
+              sudo mkdir /dev/snd
+              sudo mknod /dev/snd/ControlC0 c 1 2
+            fi
+
            # docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html
            sudo apt update
            sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace
@@ -618,6 +657,11 @@ jobs:
        default: "latest"
        description: A comma-separated string containing docker image tags to build and push (default = latest)

+      target:
+        type: string
+        default: "lotus-all-in-one"
+        description: Docker target to build
+
    steps:
      - run:
          name: Confirm that environment variables are set
@@ -657,6 +701,7 @@ jobs:

            docker build \
              <<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
+              --target <<parameters.target>> \
              -f <<parameters.path>>/<<parameters.dockerfile>> \
              $docker_tag_args \
              <<parameters.path>>
@@ -669,42 +714,16 @@ jobs:
              docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
            done

-  publish-packer-mainnet:
-    description: build and push AWS IAM and DigitalOcean droplet.
+  publish-packer-snap:
+    description: build packer image with snap. mainnet only.
    executor:
-      name: packer/default
-      packer-version: 1.6.6
+      name: packer
    steps:
      - checkout
      - attach_workspace:
          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
-  publish-packer-calibrationnet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
-  publish-packer-butterflynet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG"
+      - packer_build:
+          template: tools/packer/lotus-snap.pkr.hcl
  publish-dockerhub:
    description: publish to dockerhub
    machine:
@@ -816,10 +835,19 @@ workflows:
            tags:
              only:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-appimage:
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
      - publish:
          requires:
            - build-all
            - build-macos
+            - build-appimage
          filters:
            branches:
              ignore:
@@ -832,36 +860,13 @@ workflows:
          path: .
          repo: lotus-dev
          tag: '${CIRCLE_SHA1:0:8}'
-      - publish-packer-mainnet:
-          requires:
-            - build-all
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-calibrationnet:
-          requires:
-            - build-ntwk-calibration
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-butterflynet:
-          requires:
-            - build-ntwk-butterfly
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+          target: lotus-all-in-one
+      - build-and-push-image:
+          dockerfile: Dockerfile.lotus
+          path: .
+          repo: lotus-test
+          tag: '${CIRCLE_SHA1:0:8}'
+          target: lotus-test
      - publish-snapcraft:
          name: publish-snapcraft-stable
          channel: stable
@@ -898,3 +903,13 @@ workflows:
      - publish-dockerhub:
          name: publish-dockerhub-nightly
          tag: nightly
+  monthly:
+    triggers:
+      - schedule:
+          cron: "0 0 1 * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - publish-packer-snap
@@ -40,6 +40,7 @@ build/paramfetch.sh
 /bundle
 /darwin
 /linux
+*.snap

 *-fuzz.zip
 /chain/types/work_msg/
@@ -49,23 +49,23 @@ AppDir:
    fedora:
      image: appimagecrafters/tests-env:fedora-30
      command: ./AppRun
-      use_host_x: true
+      use_host_x: false
    debian:
      image: appimagecrafters/tests-env:debian-stable
      command: ./AppRun
-      use_host_x: true
+      use_host_x: false
    arch:
      image: appimagecrafters/tests-env:archlinux-latest
      command: ./AppRun
-      use_host_x: true
+      use_host_x: false
    centos:
      image: appimagecrafters/tests-env:centos-7
      command: ./AppRun
-      use_host_x: true
+      use_host_x: false
    ubuntu:
      image: appimagecrafters/tests-env:ubuntu-xenial
      command: ./AppRun
-      use_host_x: true
+      use_host_x: false
 AppImage:
  arch: x86_64
  update-information: guess
@@ -28,6 +28,14 @@ WORKDIR /opt/filecoin
 RUN make clean deps


+FROM builder-local AS builder-test
+MAINTAINER Lotus Development Team
+
+WORKDIR /opt/filecoin
+
+RUN make debug
+
+
 FROM builder-local AS builder
 MAINTAINER Lotus Development Team

@@ -53,8 +61,9 @@ COPY --from=builder /usr/lib/x86_64-linux-gnu/libnuma.so.1   /lib/
 COPY --from=builder /usr/lib/x86_64-linux-gnu/libhwloc.so.5  /lib/
 COPY --from=builder /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /lib/

-RUN useradd -r -u 532 -U fc
-
+RUN useradd -r -u 532 -U fc \
+ && mkdir -p /etc/OpenCL/vendors \
+ && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd

 ###
 FROM base AS lotus
@@ -207,3 +216,41 @@ EXPOSE 1234
 EXPOSE 2345
 EXPOSE 3456
 EXPOSE 1777
+
+###
+from base as lotus-test
+
+ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
+ENV LOTUS_MINER_PATH /var/lib/lotus-miner
+ENV LOTUS_PATH /var/lib/lotus
+ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
+ENV WALLET_PATH /var/lib/lotus-wallet
+
+COPY --from=builder-test /opt/filecoin/lotus      /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-miner /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-worker /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-seed /usr/local/bin/
+
+RUN mkdir /var/tmp/filecoin-proof-parameters
+RUN mkdir /var/lib/lotus
+RUN mkdir /var/lib/lotus-miner
+RUN mkdir /var/lib/lotus-worker
+RUN mkdir /var/lib/lotus-wallet
+RUN chown fc: /var/tmp/filecoin-proof-parameters
+RUN chown fc: /var/lib/lotus
+RUN chown fc: /var/lib/lotus-miner
+RUN chown fc: /var/lib/lotus-worker
+RUN chown fc: /var/lib/lotus-wallet
+
+
+VOLUME /var/tmp/filecoin-proof-parameters
+VOLUME /var/lib/lotus
+VOLUME /var/lib/lotus-miner
+VOLUME /var/lib/lotus-worker
+VOLUME /var/lib/lotus-wallet
+
+EXPOSE 1234
+EXPOSE 2345
+EXPOSE 3456
+EXPOSE 1777
+
@@ -97,7 +97,7 @@ BINS+=lotus-miner

 lotus-worker: $(BUILD_DEPS)
 	rm -f lotus-worker
-	$(GOCC) build $(GOFLAGS) -o lotus-worker ./cmd/lotus-seal-worker
+	$(GOCC) build $(GOFLAGS) -o lotus-worker ./cmd/lotus-worker
 .PHONY: lotus-worker
 BINS+=lotus-worker

@@ -345,6 +345,8 @@ gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci
 	@echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO MAKE docsgen-cli"
 .PHONY: gen

+jen: gen
+
 snap: lotus lotus-miner lotus-worker
 	snapcraft
 	# snapcraft upload ./lotus_*.snap
@@ -360,7 +360,7 @@ type FullNode interface {
 	// ClientGetRetrievalUpdates returns status of updated retrieval deals
 	ClientGetRetrievalUpdates(ctx context.Context) (<-chan RetrievalInfo, error) //perm:write
 	// ClientQueryAsk returns a signed StorageAsk from the specified miner.
-	ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error) //perm:read
+	ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*StorageAsk, error) //perm:read
 	// ClientCalcCommP calculates the CommP and data size of the specified CID
 	ClientDealPieceCID(ctx context.Context, root cid.Cid) (DataCIDSize, error) //perm:read
 	// ClientCalcCommP calculates the CommP for a specified file
@@ -689,7 +689,17 @@ type FullNode interface {
 	// MethodGroup: Paych
 	// The Paych methods are for interacting with and managing payment channels

-	PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*ChannelInfo, error)                     //perm:sign
+	// PaychGet gets or creates a payment channel between address pair
+	//  The specified amount will be reserved for use. If there aren't enough non-reserved funds
+	//    available, funds will be added through an on-chain message.
+	//  - When opts.OffChain is true, this call will not cause any messages to be sent to the chain (no automatic
+	//    channel creation/funds adding). If the operation can't be performed without sending a message an error will be
+	//    returned. Note that even when this option is specified, this call can be blocked by previous operations on the
+	//    channel waiting for on-chain operations.
+	PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt, opts PaychGetOpts) (*ChannelInfo, error) //perm:sign
+	// PaychFund gets or creates a payment channel between address pair.
+	// The specified amount will be added to the channel through on-chain send for future use
+	PaychFund(ctx context.Context, from, to address.Address, amt types.BigInt) (*ChannelInfo, error)                    //perm:sign
 	PaychGetWaitReady(context.Context, cid.Cid) (address.Address, error)                                                //perm:sign
 	PaychAvailableFunds(ctx context.Context, ch address.Address) (*ChannelAvailableFunds, error)                        //perm:sign
 	PaychAvailableFundsByFromTo(ctx context.Context, from, to address.Address) (*ChannelAvailableFunds, error)          //perm:sign
@@ -718,6 +728,12 @@ type FullNode interface {
 	CreateBackup(ctx context.Context, fpath string) error //perm:admin
 }

+type StorageAsk struct {
+	Response *storagemarket.StorageAsk
+
+	DealProtocols []string
+}
+
 type FileRef struct {
 	Path  string
 	IsCAR bool
@@ -828,6 +844,10 @@ const (
 	PCHOutbound
 )

+type PaychGetOpts struct {
+	OffChain bool
+}
+
 type PaychStatus struct {
 	ControlAddr address.Address
 	Direction   PCHDir
@@ -845,16 +865,23 @@ type ChannelAvailableFunds struct {
 	From address.Address
 	// To is the to address of the channel
 	To address.Address
-	// ConfirmedAmt is the amount of funds that have been confirmed on-chain
-	// for the channel
+
+	// ConfirmedAmt is the total amount of funds that have been confirmed on-chain for the channel
 	ConfirmedAmt types.BigInt
 	// PendingAmt is the amount of funds that are pending confirmation on-chain
 	PendingAmt types.BigInt
+
+	// NonReservedAmt is part of ConfirmedAmt that is available for use (e.g. when the payment channel was pre-funded)
+	NonReservedAmt types.BigInt
+	// PendingAvailableAmt is the amount of funds that are pending confirmation on-chain that will become available once confirmed
+	PendingAvailableAmt types.BigInt
+
 	// PendingWaitSentinel can be used with PaychGetWaitReady to wait for
 	// confirmation of pending funds
 	PendingWaitSentinel *cid.Cid
 	// QueuedAmt is the amount that is queued up behind a pending request
 	QueuedAmt types.BigInt
+
 	// VoucherRedeemedAmt is the amount that is redeemed by vouchers on-chain
 	// and in the local datastore
 	VoucherReedeemedAmt types.BigInt
@@ -45,8 +45,9 @@ type Gateway interface {
 	GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
 	MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
 	MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
-	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
 	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
+	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
+	MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MsigVesting, error)
 	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
 	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
 	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
@@ -2,6 +2,7 @@ package api

 import (
 	"context"
+	"time"

 	metrics "github.com/libp2p/go-libp2p-core/metrics"
 	"github.com/libp2p/go-libp2p-core/network"
@@ -25,6 +26,7 @@ type Net interface {

 	NetConnectedness(context.Context, peer.ID) (network.Connectedness, error) //perm:read
 	NetPeers(context.Context) ([]peer.AddrInfo, error)                        //perm:read
+	NetPing(context.Context, peer.ID) (time.Duration, error)                  //perm:read
 	NetConnect(context.Context, peer.AddrInfo) error                          //perm:write
 	NetAddrsListen(context.Context) (peer.AddrInfo, error)                    //perm:read
 	NetDisconnect(context.Context, peer.ID) error                             //perm:write
@@ -6,6 +6,7 @@ import (
 	"time"

 	"github.com/filecoin-project/lotus/chain/actors/builtin"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"

 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
@@ -24,7 +25,6 @@ import (

 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 	"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
 )
@@ -52,6 +52,8 @@ type StorageMiner interface {

 	MiningBase(context.Context) (*types.TipSet, error) //perm:read

+	ComputeWindowPoSt(ctx context.Context, dlIdx uint64, tsk types.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) //perm:admin
+
 	// Temp api for testing
 	PledgeSector(context.Context) (abi.SectorID, error) //perm:write

@@ -92,7 +94,8 @@ type StorageMiner interface {
 	SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error   //perm:admin
 	// SectorRemove removes the sector from storage. It doesn't terminate it on-chain, which can
 	// be done with SectorTerminate. Removing and not terminating live sectors will cause additional penalties.
-	SectorRemove(context.Context, abi.SectorNumber) error //perm:admin
+	SectorRemove(context.Context, abi.SectorNumber) error                           //perm:admin
+	SectorMarkForUpgrade(ctx context.Context, id abi.SectorNumber, snap bool) error //perm:admin
 	// SectorTerminate terminates the sector on-chain (adding it to a termination batch first), then
 	// automatically removes it from storage
 	SectorTerminate(context.Context, abi.SectorNumber) error //perm:admin
@@ -100,8 +103,7 @@ type StorageMiner interface {
 	// Returns null if message wasn't sent
 	SectorTerminateFlush(ctx context.Context) (*cid.Cid, error) //perm:admin
 	// SectorTerminatePending returns a list of pending sector terminations to be sent in the next batch message
-	SectorTerminatePending(ctx context.Context) ([]abi.SectorID, error)             //perm:admin
-	SectorMarkForUpgrade(ctx context.Context, id abi.SectorNumber, snap bool) error //perm:admin
+	SectorTerminatePending(ctx context.Context) ([]abi.SectorID, error) //perm:admin
 	// SectorPreCommitFlush immediately sends a PreCommit message with sectors batched for PreCommit.
 	// Returns null if message wasn't sent
 	SectorPreCommitFlush(ctx context.Context) ([]sealiface.PreCommitBatchRes, error) //perm:admin
@@ -113,6 +115,8 @@ type StorageMiner interface {
 	// SectorCommitPending returns a list of pending Commit sectors to be sent in the next aggregate message
 	SectorCommitPending(ctx context.Context) ([]abi.SectorID, error) //perm:admin
 	SectorMatchPendingPiecesToOpenSectors(ctx context.Context) error //perm:admin
+	// SectorAbortUpgrade can be called on sectors that are in the process of being upgraded to abort it
+	SectorAbortUpgrade(context.Context, abi.SectorNumber) error //perm:admin

 	// WorkerConnect tells the node to connect to workers RPC
 	WorkerConnect(context.Context, string) error                              //perm:admin retry:true
@@ -130,6 +134,7 @@ type StorageMiner interface {
 	ReturnProveReplicaUpdate1(ctx context.Context, callID storiface.CallID, vanillaProofs storage.ReplicaVanillaProofs, err *storiface.CallError) error //perm:admin retry:true
 	ReturnProveReplicaUpdate2(ctx context.Context, callID storiface.CallID, proof storage.ReplicaUpdateProof, err *storiface.CallError) error           //perm:admin retry:true
 	ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                       //perm:admin retry:true
+	ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                           //perm:admin retry:true
 	ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                 //perm:admin retry:true
 	ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                     //perm:admin retry:true
 	ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                     //perm:admin retry:true
@@ -140,21 +145,21 @@ type StorageMiner interface {
 	SealingSchedDiag(ctx context.Context, doSched bool) (interface{}, error) //perm:admin
 	SealingAbort(ctx context.Context, call storiface.CallID) error           //perm:admin

-	//stores.SectorIndex
-	StorageAttach(context.Context, stores.StorageInfo, fsutil.FsStat) error                                                                                             //perm:admin
-	StorageInfo(context.Context, stores.ID) (stores.StorageInfo, error)                                                                                                 //perm:admin
-	StorageReportHealth(context.Context, stores.ID, stores.HealthReport) error                                                                                          //perm:admin
-	StorageDeclareSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error                                     //perm:admin
-	StorageDropSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType) error                                                      //perm:admin
-	StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]stores.SectorStorageInfo, error) //perm:admin
-	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]stores.StorageInfo, error)           //perm:admin
-	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                                          //perm:admin
-	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                               //perm:admin
-	StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error)                                                                                               //perm:admin
-	StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error)                                                                                                 //perm:admin
+	// SectorIndex
+	StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error                                                                                             //perm:admin
+	StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error)                                                                                              //perm:admin
+	StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error                                                                                       //perm:admin
+	StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error                                     //perm:admin
+	StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error                                                      //perm:admin
+	StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error) //perm:admin
+	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error)           //perm:admin
+	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                                             //perm:admin
+	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                                  //perm:admin
+	StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error)                                                                                            //perm:admin
+	StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error)                                                                                                    //perm:admin

-	StorageLocal(ctx context.Context) (map[stores.ID]string, error)       //perm:admin
-	StorageStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) //perm:admin
+	StorageLocal(ctx context.Context) (map[storiface.ID]string, error)       //perm:admin
+	StorageStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error) //perm:admin

 	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error                                                                                                        //perm:write
 	MarketListDeals(ctx context.Context) ([]MarketDeal, error)                                                                                                                           //perm:read
@@ -269,7 +274,6 @@ type StorageMiner interface {
 }

 var _ storiface.WorkerReturn = *new(StorageMiner)
-var _ stores.SectorIndex = *new(StorageMiner)

 type SealRes struct {
 	Err   string
@@ -293,19 +297,20 @@ type SectorPiece struct {
 }

 type SectorInfo struct {
-	SectorID     abi.SectorNumber
-	State        SectorState
-	CommD        *cid.Cid
-	CommR        *cid.Cid
-	Proof        []byte
-	Deals        []abi.DealID
-	Pieces       []SectorPiece
-	Ticket       SealTicket
-	Seed         SealSeed
-	PreCommitMsg *cid.Cid
-	CommitMsg    *cid.Cid
-	Retries      uint64
-	ToUpgrade    bool
+	SectorID             abi.SectorNumber
+	State                SectorState
+	CommD                *cid.Cid
+	CommR                *cid.Cid
+	Proof                []byte
+	Deals                []abi.DealID
+	Pieces               []SectorPiece
+	Ticket               SealTicket
+	Seed                 SealSeed
+	PreCommitMsg         *cid.Cid
+	CommitMsg            *cid.Cid
+	Retries              uint64
+	ToUpgrade            bool
+	ReplicaUpdateMessage *cid.Cid

 	LastErr string

@@ -1,3 +1,4 @@
+//stm: #unit
 package api

 import (
@@ -26,6 +27,7 @@ func goCmd() string {
 }

 func TestDoesntDependOnFFI(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_FFI_DEPENDENCE_001
 	deps, err := exec.Command(goCmd(), "list", "-deps", "github.com/filecoin-project/lotus/api").Output()
 	if err != nil {
 		t.Fatal(err)
@@ -38,6 +40,7 @@ func TestDoesntDependOnFFI(t *testing.T) {
 }

 func TestDoesntDependOnBuild(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_FFI_DEPENDENCE_002
 	deps, err := exec.Command(goCmd(), "list", "-deps", "github.com/filecoin-project/lotus/api").Output()
 	if err != nil {
 		t.Fatal(err)
@@ -50,6 +53,7 @@ func TestDoesntDependOnBuild(t *testing.T) {
 }

 func TestReturnTypes(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_001
 	errType := reflect.TypeOf(new(error)).Elem()
 	bareIface := reflect.TypeOf(new(interface{})).Elem()
 	jmarsh := reflect.TypeOf(new(json.Marshaler)).Elem()
@@ -115,6 +119,7 @@ func TestReturnTypes(t *testing.T) {
 }

 func TestPermTags(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_PERM_TAGS_001
 	_ = PermissionedFullAPI(&FullNodeStruct{})
 	_ = PermissionedStorMinerAPI(&StorageMinerStruct{})
 	_ = PermissionedWorkerAPI(&WorkerStruct{})
@@ -7,8 +7,9 @@ import (
 	"github.com/ipfs/go-cid"

 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-actors/v5/actors/runtime/proof"
+
 	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 	"github.com/filecoin-project/specs-storage/storage"
 )
@@ -29,7 +30,7 @@ type Worker interface {

 	// TaskType -> Weight
 	TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) //perm:admin
-	Paths(context.Context) ([]stores.StoragePath, error)                //perm:admin
+	Paths(context.Context) ([]storiface.StoragePath, error)             //perm:admin
 	Info(context.Context) (storiface.WorkerInfo, error)                 //perm:admin

 	// storiface.WorkerCalls
@@ -39,6 +40,7 @@ type Worker interface {
 	SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error) //perm:admin
 	SealCommit2(ctx context.Context, sector storage.SectorRef, c1o storage.Commit1Out) (storiface.CallID, error)                                                                                         //perm:admin
 	FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                                //perm:admin
+	FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                         //perm:admin
 	ReplicaUpdate(ctx context.Context, sector storage.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error)                                                                                       //perm:admin
 	ProveReplicaUpdate1(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error)                                                              //perm:admin
 	ProveReplicaUpdate2(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storage.ReplicaVanillaProofs) (storiface.CallID, error)                  //perm:admin
@@ -48,6 +50,9 @@ type Worker interface {
 	UnsealPiece(context.Context, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error)                                           //perm:admin
 	Fetch(context.Context, storage.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error)                                                             //perm:admin

+	GenerateWinningPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, randomness abi.PoStRandomness) ([]proof.PoStProof, error)                           //perm:admin
+	GenerateWindowPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, partitionIdx int, randomness abi.PoStRandomness) (storiface.WindowPoStResult, error) //perm:admin
+
 	TaskDisable(ctx context.Context, tt sealtasks.TaskType) error //perm:admin
 	TaskEnable(ctx context.Context, tt sealtasks.TaskType) error  //perm:admin

@@ -40,7 +40,6 @@ import (
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
@@ -122,7 +121,7 @@ func init() {
 	addExample(api.FullAPIVersion1)
 	addExample(api.PCHInbound)
 	addExample(time.Minute)
-	addExample(graphsync.RequestID(4))
+	addExample(graphsync.NewRequestID())
 	addExample(datatransfer.TransferID(3))
 	addExample(datatransfer.Ongoing)
 	addExample(storeIDExample)
@@ -199,10 +198,10 @@ func init() {
 		},
 	})
 	addExample(api.SectorState(sealing.Proving))
-	addExample(stores.ID("76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8"))
+	addExample(storiface.ID("76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8"))
 	addExample(storiface.FTUnsealed)
 	addExample(storiface.PathSealing)
-	addExample(map[stores.ID][]stores.Decl{
+	addExample(map[storiface.ID][]storiface.Decl{
 		"76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8": {
 			{
 				SectorID:       abi.SectorID{Miner: 1000, Number: 100},
@@ -210,7 +209,7 @@ func init() {
 			},
 		},
 	})
-	addExample(map[stores.ID]string{
+	addExample(map[storiface.ID]string{
 		"76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8": "/data/path",
 	})
 	addExample(map[uuid.UUID][]storiface.WorkerJob{
@@ -8,12 +8,12 @@ import (
 	context "context"
 	json "encoding/json"
 	reflect "reflect"
+	time "time"

 	address "github.com/filecoin-project/go-address"
 	bitfield "github.com/filecoin-project/go-bitfield"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
 	retrievalmarket "github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket"
 	auth "github.com/filecoin-project/go-jsonrpc/auth"
 	abi "github.com/filecoin-project/go-state-types/abi"
 	big "github.com/filecoin-project/go-state-types/big"
@@ -745,10 +745,10 @@ func (mr *MockFullNodeMockRecorder) ClientMinerQueryOffer(arg0, arg1, arg2, arg3
 }

 // ClientQueryAsk mocks base method.
-func (m *MockFullNode) ClientQueryAsk(arg0 context.Context, arg1 peer.ID, arg2 address.Address) (*storagemarket.StorageAsk, error) {
+func (m *MockFullNode) ClientQueryAsk(arg0 context.Context, arg1 peer.ID, arg2 address.Address) (*api.StorageAsk, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "ClientQueryAsk", arg0, arg1, arg2)
-	ret0, _ := ret[0].(*storagemarket.StorageAsk)
+	ret0, _ := ret[0].(*api.StorageAsk)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
@@ -1856,6 +1856,21 @@ func (mr *MockFullNodeMockRecorder) NetPeers(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPeers", reflect.TypeOf((*MockFullNode)(nil).NetPeers), arg0)
 }

+// NetPing mocks base method.
+func (m *MockFullNode) NetPing(arg0 context.Context, arg1 peer.ID) (time.Duration, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetPing", arg0, arg1)
+	ret0, _ := ret[0].(time.Duration)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetPing indicates an expected call of NetPing.
+func (mr *MockFullNodeMockRecorder) NetPing(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPing", reflect.TypeOf((*MockFullNode)(nil).NetPing), arg0, arg1)
+}
+
 // NetProtectAdd mocks base method.
 func (m *MockFullNode) NetProtectAdd(arg0 context.Context, arg1 []peer.ID) error {
 	m.ctrl.T.Helper()
@@ -2018,19 +2033,34 @@ func (mr *MockFullNodeMockRecorder) PaychCollect(arg0, arg1 interface{}) *gomock
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychCollect", reflect.TypeOf((*MockFullNode)(nil).PaychCollect), arg0, arg1)
 }

-// PaychGet mocks base method.
-func (m *MockFullNode) PaychGet(arg0 context.Context, arg1, arg2 address.Address, arg3 big.Int) (*api.ChannelInfo, error) {
+// PaychFund mocks base method.
+func (m *MockFullNode) PaychFund(arg0 context.Context, arg1, arg2 address.Address, arg3 big.Int) (*api.ChannelInfo, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "PaychGet", arg0, arg1, arg2, arg3)
+	ret := m.ctrl.Call(m, "PaychFund", arg0, arg1, arg2, arg3)
+	ret0, _ := ret[0].(*api.ChannelInfo)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// PaychFund indicates an expected call of PaychFund.
+func (mr *MockFullNodeMockRecorder) PaychFund(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychFund", reflect.TypeOf((*MockFullNode)(nil).PaychFund), arg0, arg1, arg2, arg3)
+}
+
+// PaychGet mocks base method.
+func (m *MockFullNode) PaychGet(arg0 context.Context, arg1, arg2 address.Address, arg3 big.Int, arg4 api.PaychGetOpts) (*api.ChannelInfo, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "PaychGet", arg0, arg1, arg2, arg3, arg4)
 	ret0, _ := ret[0].(*api.ChannelInfo)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }

 // PaychGet indicates an expected call of PaychGet.
-func (mr *MockFullNodeMockRecorder) PaychGet(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+func (mr *MockFullNodeMockRecorder) PaychGet(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychGet", reflect.TypeOf((*MockFullNode)(nil).PaychGet), arg0, arg1, arg2, arg3)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychGet", reflect.TypeOf((*MockFullNode)(nil).PaychGet), arg0, arg1, arg2, arg3, arg4)
 }

 // PaychGetWaitReady mocks base method.
@@ -25,12 +25,12 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
 	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 	"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
 	"github.com/filecoin-project/lotus/journal/alerting"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/repo/imports"
+	"github.com/filecoin-project/specs-actors/v5/actors/runtime/proof"
 	"github.com/filecoin-project/specs-storage/storage"
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
@@ -190,7 +190,7 @@ type FullNodeStruct struct {

 		ClientMinerQueryOffer func(p0 context.Context, p1 address.Address, p2 cid.Cid, p3 *cid.Cid) (QueryOffer, error) `perm:"read"`

-		ClientQueryAsk func(p0 context.Context, p1 peer.ID, p2 address.Address) (*storagemarket.StorageAsk, error) `perm:"read"`
+		ClientQueryAsk func(p0 context.Context, p1 peer.ID, p2 address.Address) (*StorageAsk, error) `perm:"read"`

 		ClientRemoveImport func(p0 context.Context, p1 imports.ID) error `perm:"admin"`

@@ -306,7 +306,9 @@ type FullNodeStruct struct {

 		PaychCollect func(p0 context.Context, p1 address.Address) (cid.Cid, error) `perm:"sign"`

-		PaychGet func(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt) (*ChannelInfo, error) `perm:"sign"`
+		PaychFund func(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt) (*ChannelInfo, error) `perm:"sign"`
+
+		PaychGet func(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt, p4 PaychGetOpts) (*ChannelInfo, error) `perm:"sign"`

 		PaychGetWaitReady func(p0 context.Context, p1 cid.Cid) (address.Address, error) `perm:"sign"`

@@ -516,6 +518,8 @@ type GatewayStruct struct {

 		MsigGetVested func(p0 context.Context, p1 address.Address, p2 types.TipSetKey, p3 types.TipSetKey) (types.BigInt, error) ``

+		MsigGetVestingSchedule func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MsigVesting, error) ``
+
 		StateAccountKey func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) ``

 		StateDealProviderCollateralBounds func(p0 context.Context, p1 abi.PaddedPieceSize, p2 bool, p3 types.TipSetKey) (DealCollateralBounds, error) ``
@@ -593,6 +597,8 @@ type NetStruct struct {

 		NetPeers func(p0 context.Context) ([]peer.AddrInfo, error) `perm:"read"`

+		NetPing func(p0 context.Context, p1 peer.ID) (time.Duration, error) `perm:"read"`
+
 		NetProtectAdd func(p0 context.Context, p1 []peer.ID) error `perm:"admin"`

 		NetProtectList func(p0 context.Context) ([]peer.ID, error) `perm:"read"`
@@ -635,6 +641,8 @@ type StorageMinerStruct struct {

 		ComputeProof func(p0 context.Context, p1 []builtin.ExtendedSectorInfo, p2 abi.PoStRandomness, p3 abi.ChainEpoch, p4 abinetwork.Version) ([]builtin.PoStProof, error) `perm:"read"`

+		ComputeWindowPoSt func(p0 context.Context, p1 uint64, p2 types.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) `perm:"admin"`
+
 		CreateBackup func(p0 context.Context, p1 string) error `perm:"admin"`

 		DagstoreGC func(p0 context.Context) ([]DagstoreShardResult, error) `perm:"admin"`
@@ -735,6 +743,8 @@ type StorageMinerStruct struct {

 		ReturnFetch func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"`

+		ReturnFinalizeReplicaUpdate func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"`
+
 		ReturnFinalizeSector func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"`

 		ReturnGenerateSectorKeyFromData func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"`
@@ -767,6 +777,8 @@ type StorageMinerStruct struct {

 		SealingSchedDiag func(p0 context.Context, p1 bool) (interface{}, error) `perm:"admin"`

+		SectorAbortUpgrade func(p0 context.Context, p1 abi.SectorNumber) error `perm:"admin"`
+
 		SectorAddPieceToAny func(p0 context.Context, p1 abi.UnpaddedPieceSize, p2 storage.Data, p3 PieceDealInfo) (SectorOffset, error) `perm:"admin"`

 		SectorCommitFlush func(p0 context.Context) ([]sealiface.CommitBatchRes, error) `perm:"admin"`
@@ -815,29 +827,29 @@ type StorageMinerStruct struct {

 		StorageAddLocal func(p0 context.Context, p1 string) error `perm:"admin"`

-		StorageAttach func(p0 context.Context, p1 stores.StorageInfo, p2 fsutil.FsStat) error `perm:"admin"`
+		StorageAttach func(p0 context.Context, p1 storiface.StorageInfo, p2 fsutil.FsStat) error `perm:"admin"`

-		StorageBestAlloc func(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]stores.StorageInfo, error) `perm:"admin"`
+		StorageBestAlloc func(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]storiface.StorageInfo, error) `perm:"admin"`

-		StorageDeclareSector func(p0 context.Context, p1 stores.ID, p2 abi.SectorID, p3 storiface.SectorFileType, p4 bool) error `perm:"admin"`
+		StorageDeclareSector func(p0 context.Context, p1 storiface.ID, p2 abi.SectorID, p3 storiface.SectorFileType, p4 bool) error `perm:"admin"`

-		StorageDropSector func(p0 context.Context, p1 stores.ID, p2 abi.SectorID, p3 storiface.SectorFileType) error `perm:"admin"`
+		StorageDropSector func(p0 context.Context, p1 storiface.ID, p2 abi.SectorID, p3 storiface.SectorFileType) error `perm:"admin"`

-		StorageFindSector func(p0 context.Context, p1 abi.SectorID, p2 storiface.SectorFileType, p3 abi.SectorSize, p4 bool) ([]stores.SectorStorageInfo, error) `perm:"admin"`
+		StorageFindSector func(p0 context.Context, p1 abi.SectorID, p2 storiface.SectorFileType, p3 abi.SectorSize, p4 bool) ([]storiface.SectorStorageInfo, error) `perm:"admin"`

 		StorageGetLocks func(p0 context.Context) (storiface.SectorLocks, error) `perm:"admin"`

-		StorageInfo func(p0 context.Context, p1 stores.ID) (stores.StorageInfo, error) `perm:"admin"`
+		StorageInfo func(p0 context.Context, p1 storiface.ID) (storiface.StorageInfo, error) `perm:"admin"`

-		StorageList func(p0 context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"`
+		StorageList func(p0 context.Context) (map[storiface.ID][]storiface.Decl, error) `perm:"admin"`

-		StorageLocal func(p0 context.Context) (map[stores.ID]string, error) `perm:"admin"`
+		StorageLocal func(p0 context.Context) (map[storiface.ID]string, error) `perm:"admin"`

 		StorageLock func(p0 context.Context, p1 abi.SectorID, p2 storiface.SectorFileType, p3 storiface.SectorFileType) error `perm:"admin"`

-		StorageReportHealth func(p0 context.Context, p1 stores.ID, p2 stores.HealthReport) error `perm:"admin"`
+		StorageReportHealth func(p0 context.Context, p1 storiface.ID, p2 storiface.HealthReport) error `perm:"admin"`

-		StorageStat func(p0 context.Context, p1 stores.ID) (fsutil.FsStat, error) `perm:"admin"`
+		StorageStat func(p0 context.Context, p1 storiface.ID) (fsutil.FsStat, error) `perm:"admin"`

 		StorageTryLock func(p0 context.Context, p1 abi.SectorID, p2 storiface.SectorFileType, p3 storiface.SectorFileType) (bool, error) `perm:"admin"`

@@ -884,15 +896,21 @@ type WorkerStruct struct {

 		Fetch func(p0 context.Context, p1 storage.SectorRef, p2 storiface.SectorFileType, p3 storiface.PathType, p4 storiface.AcquireMode) (storiface.CallID, error) `perm:"admin"`

+		FinalizeReplicaUpdate func(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) `perm:"admin"`
+
 		FinalizeSector func(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) `perm:"admin"`

 		GenerateSectorKeyFromData func(p0 context.Context, p1 storage.SectorRef, p2 cid.Cid) (storiface.CallID, error) `perm:"admin"`

+		GenerateWindowPoSt func(p0 context.Context, p1 abi.RegisteredPoStProof, p2 abi.ActorID, p3 []storiface.PostSectorChallenge, p4 int, p5 abi.PoStRandomness) (storiface.WindowPoStResult, error) `perm:"admin"`
+
+		GenerateWinningPoSt func(p0 context.Context, p1 abi.RegisteredPoStProof, p2 abi.ActorID, p3 []storiface.PostSectorChallenge, p4 abi.PoStRandomness) ([]proof.PoStProof, error) `perm:"admin"`
+
 		Info func(p0 context.Context) (storiface.WorkerInfo, error) `perm:"admin"`

 		MoveStorage func(p0 context.Context, p1 storage.SectorRef, p2 storiface.SectorFileType) (storiface.CallID, error) `perm:"admin"`

-		Paths func(p0 context.Context) ([]stores.StoragePath, error) `perm:"admin"`
+		Paths func(p0 context.Context) ([]storiface.StoragePath, error) `perm:"admin"`

 		ProcessSession func(p0 context.Context) (uuid.UUID, error) `perm:"admin"`

@@ -1553,14 +1571,14 @@ func (s *FullNodeStub) ClientMinerQueryOffer(p0 context.Context, p1 address.Addr
 	return *new(QueryOffer), ErrNotSupported
 }

-func (s *FullNodeStruct) ClientQueryAsk(p0 context.Context, p1 peer.ID, p2 address.Address) (*storagemarket.StorageAsk, error) {
+func (s *FullNodeStruct) ClientQueryAsk(p0 context.Context, p1 peer.ID, p2 address.Address) (*StorageAsk, error) {
 	if s.Internal.ClientQueryAsk == nil {
 		return nil, ErrNotSupported
 	}
 	return s.Internal.ClientQueryAsk(p0, p1, p2)
 }

-func (s *FullNodeStub) ClientQueryAsk(p0 context.Context, p1 peer.ID, p2 address.Address) (*storagemarket.StorageAsk, error) {
+func (s *FullNodeStub) ClientQueryAsk(p0 context.Context, p1 peer.ID, p2 address.Address) (*StorageAsk, error) {
 	return nil, ErrNotSupported
 }

@@ -2191,14 +2209,25 @@ func (s *FullNodeStub) PaychCollect(p0 context.Context, p1 address.Address) (cid
 	return *new(cid.Cid), ErrNotSupported
 }

-func (s *FullNodeStruct) PaychGet(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt) (*ChannelInfo, error) {
+func (s *FullNodeStruct) PaychFund(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt) (*ChannelInfo, error) {
+	if s.Internal.PaychFund == nil {
+		return nil, ErrNotSupported
+	}
+	return s.Internal.PaychFund(p0, p1, p2, p3)
+}
+
+func (s *FullNodeStub) PaychFund(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt) (*ChannelInfo, error) {
+	return nil, ErrNotSupported
+}
+
+func (s *FullNodeStruct) PaychGet(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt, p4 PaychGetOpts) (*ChannelInfo, error) {
 	if s.Internal.PaychGet == nil {
 		return nil, ErrNotSupported
 	}
-	return s.Internal.PaychGet(p0, p1, p2, p3)
+	return s.Internal.PaychGet(p0, p1, p2, p3, p4)
 }

-func (s *FullNodeStub) PaychGet(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt) (*ChannelInfo, error) {
+func (s *FullNodeStub) PaychGet(p0 context.Context, p1 address.Address, p2 address.Address, p3 types.BigInt, p4 PaychGetOpts) (*ChannelInfo, error) {
 	return nil, ErrNotSupported
 }

@@ -3291,6 +3320,17 @@ func (s *GatewayStub) MsigGetVested(p0 context.Context, p1 address.Address, p2 t
 	return *new(types.BigInt), ErrNotSupported
 }

+func (s *GatewayStruct) MsigGetVestingSchedule(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MsigVesting, error) {
+	if s.Internal.MsigGetVestingSchedule == nil {
+		return *new(MsigVesting), ErrNotSupported
+	}
+	return s.Internal.MsigGetVestingSchedule(p0, p1, p2)
+}
+
+func (s *GatewayStub) MsigGetVestingSchedule(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MsigVesting, error) {
+	return *new(MsigVesting), ErrNotSupported
+}
+
 func (s *GatewayStruct) StateAccountKey(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) {
 	if s.Internal.StateAccountKey == nil {
 		return *new(address.Address), ErrNotSupported
@@ -3676,6 +3716,17 @@ func (s *NetStub) NetPeers(p0 context.Context) ([]peer.AddrInfo, error) {
 	return *new([]peer.AddrInfo), ErrNotSupported
 }

+func (s *NetStruct) NetPing(p0 context.Context, p1 peer.ID) (time.Duration, error) {
+	if s.Internal.NetPing == nil {
+		return *new(time.Duration), ErrNotSupported
+	}
+	return s.Internal.NetPing(p0, p1)
+}
+
+func (s *NetStub) NetPing(p0 context.Context, p1 peer.ID) (time.Duration, error) {
+	return *new(time.Duration), ErrNotSupported
+}
+
 func (s *NetStruct) NetProtectAdd(p0 context.Context, p1 []peer.ID) error {
 	if s.Internal.NetProtectAdd == nil {
 		return ErrNotSupported
@@ -3808,6 +3859,17 @@ func (s *StorageMinerStub) ComputeProof(p0 context.Context, p1 []builtin.Extende
 	return *new([]builtin.PoStProof), ErrNotSupported
 }

+func (s *StorageMinerStruct) ComputeWindowPoSt(p0 context.Context, p1 uint64, p2 types.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) {
+	if s.Internal.ComputeWindowPoSt == nil {
+		return *new([]miner.SubmitWindowedPoStParams), ErrNotSupported
+	}
+	return s.Internal.ComputeWindowPoSt(p0, p1, p2)
+}
+
+func (s *StorageMinerStub) ComputeWindowPoSt(p0 context.Context, p1 uint64, p2 types.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) {
+	return *new([]miner.SubmitWindowedPoStParams), ErrNotSupported
+}
+
 func (s *StorageMinerStruct) CreateBackup(p0 context.Context, p1 string) error {
 	if s.Internal.CreateBackup == nil {
 		return ErrNotSupported
@@ -4358,6 +4420,17 @@ func (s *StorageMinerStub) ReturnFetch(p0 context.Context, p1 storiface.CallID,
 	return ErrNotSupported
 }

+func (s *StorageMinerStruct) ReturnFinalizeReplicaUpdate(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error {
+	if s.Internal.ReturnFinalizeReplicaUpdate == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.ReturnFinalizeReplicaUpdate(p0, p1, p2)
+}
+
+func (s *StorageMinerStub) ReturnFinalizeReplicaUpdate(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error {
+	return ErrNotSupported
+}
+
 func (s *StorageMinerStruct) ReturnFinalizeSector(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error {
 	if s.Internal.ReturnFinalizeSector == nil {
 		return ErrNotSupported
@@ -4534,6 +4607,17 @@ func (s *StorageMinerStub) SealingSchedDiag(p0 context.Context, p1 bool) (interf
 	return nil, ErrNotSupported
 }

+func (s *StorageMinerStruct) SectorAbortUpgrade(p0 context.Context, p1 abi.SectorNumber) error {
+	if s.Internal.SectorAbortUpgrade == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.SectorAbortUpgrade(p0, p1)
+}
+
+func (s *StorageMinerStub) SectorAbortUpgrade(p0 context.Context, p1 abi.SectorNumber) error {
+	return ErrNotSupported
+}
+
 func (s *StorageMinerStruct) SectorAddPieceToAny(p0 context.Context, p1 abi.UnpaddedPieceSize, p2 storage.Data, p3 PieceDealInfo) (SectorOffset, error) {
 	if s.Internal.SectorAddPieceToAny == nil {
 		return *new(SectorOffset), ErrNotSupported
@@ -4798,59 +4882,59 @@ func (s *StorageMinerStub) StorageAddLocal(p0 context.Context, p1 string) error
 	return ErrNotSupported
 }

-func (s *StorageMinerStruct) StorageAttach(p0 context.Context, p1 stores.StorageInfo, p2 fsutil.FsStat) error {
+func (s *StorageMinerStruct) StorageAttach(p0 context.Context, p1 storiface.StorageInfo, p2 fsutil.FsStat) error {
 	if s.Internal.StorageAttach == nil {
 		return ErrNotSupported
 	}
 	return s.Internal.StorageAttach(p0, p1, p2)
 }

-func (s *StorageMinerStub) StorageAttach(p0 context.Context, p1 stores.StorageInfo, p2 fsutil.FsStat) error {
+func (s *StorageMinerStub) StorageAttach(p0 context.Context, p1 storiface.StorageInfo, p2 fsutil.FsStat) error {
 	return ErrNotSupported
 }

-func (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]stores.StorageInfo, error) {
+func (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]storiface.StorageInfo, error) {
 	if s.Internal.StorageBestAlloc == nil {
-		return *new([]stores.StorageInfo), ErrNotSupported
+		return *new([]storiface.StorageInfo), ErrNotSupported
 	}
 	return s.Internal.StorageBestAlloc(p0, p1, p2, p3)
 }

-func (s *StorageMinerStub) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]stores.StorageInfo, error) {
-	return *new([]stores.StorageInfo), ErrNotSupported
+func (s *StorageMinerStub) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]storiface.StorageInfo, error) {
+	return *new([]storiface.StorageInfo), ErrNotSupported
 }

-func (s *StorageMinerStruct) StorageDeclareSector(p0 context.Context, p1 stores.ID, p2 abi.SectorID, p3 storiface.SectorFileType, p4 bool) error {
+func (s *StorageMinerStruct) StorageDeclareSector(p0 context.Context, p1 storiface.ID, p2 abi.SectorID, p3 storiface.SectorFileType, p4 bool) error {
 	if s.Internal.StorageDeclareSector == nil {
 		return ErrNotSupported
 	}
 	return s.Internal.StorageDeclareSector(p0, p1, p2, p3, p4)
 }

-func (s *StorageMinerStub) StorageDeclareSector(p0 context.Context, p1 stores.ID, p2 abi.SectorID, p3 storiface.SectorFileType, p4 bool) error {
+func (s *StorageMinerStub) StorageDeclareSector(p0 context.Context, p1 storiface.ID, p2 abi.SectorID, p3 storiface.SectorFileType, p4 bool) error {
 	return ErrNotSupported
 }

-func (s *StorageMinerStruct) StorageDropSector(p0 context.Context, p1 stores.ID, p2 abi.SectorID, p3 storiface.SectorFileType) error {
+func (s *StorageMinerStruct) StorageDropSector(p0 context.Context, p1 storiface.ID, p2 abi.SectorID, p3 storiface.SectorFileType) error {
 	if s.Internal.StorageDropSector == nil {
 		return ErrNotSupported
 	}
 	return s.Internal.StorageDropSector(p0, p1, p2, p3)
 }

-func (s *StorageMinerStub) StorageDropSector(p0 context.Context, p1 stores.ID, p2 abi.SectorID, p3 storiface.SectorFileType) error {
+func (s *StorageMinerStub) StorageDropSector(p0 context.Context, p1 storiface.ID, p2 abi.SectorID, p3 storiface.SectorFileType) error {
 	return ErrNotSupported
 }

-func (s *StorageMinerStruct) StorageFindSector(p0 context.Context, p1 abi.SectorID, p2 storiface.SectorFileType, p3 abi.SectorSize, p4 bool) ([]stores.SectorStorageInfo, error) {
+func (s *StorageMinerStruct) StorageFindSector(p0 context.Context, p1 abi.SectorID, p2 storiface.SectorFileType, p3 abi.SectorSize, p4 bool) ([]storiface.SectorStorageInfo, error) {
 	if s.Internal.StorageFindSector == nil {
-		return *new([]stores.SectorStorageInfo), ErrNotSupported
+		return *new([]storiface.SectorStorageInfo), ErrNotSupported
 	}
 	return s.Internal.StorageFindSector(p0, p1, p2, p3, p4)
 }

-func (s *StorageMinerStub) StorageFindSector(p0 context.Context, p1 abi.SectorID, p2 storiface.SectorFileType, p3 abi.SectorSize, p4 bool) ([]stores.SectorStorageInfo, error) {
-	return *new([]stores.SectorStorageInfo), ErrNotSupported
+func (s *StorageMinerStub) StorageFindSector(p0 context.Context, p1 abi.SectorID, p2 storiface.SectorFileType, p3 abi.SectorSize, p4 bool) ([]storiface.SectorStorageInfo, error) {
+	return *new([]storiface.SectorStorageInfo), ErrNotSupported
 }

 func (s *StorageMinerStruct) StorageGetLocks(p0 context.Context) (storiface.SectorLocks, error) {
@@ -4864,37 +4948,37 @@ func (s *StorageMinerStub) StorageGetLocks(p0 context.Context) (storiface.Sector
 	return *new(storiface.SectorLocks), ErrNotSupported
 }

-func (s *StorageMinerStruct) StorageInfo(p0 context.Context, p1 stores.ID) (stores.StorageInfo, error) {
+func (s *StorageMinerStruct) StorageInfo(p0 context.Context, p1 storiface.ID) (storiface.StorageInfo, error) {
 	if s.Internal.StorageInfo == nil {
-		return *new(stores.StorageInfo), ErrNotSupported
+		return *new(storiface.StorageInfo), ErrNotSupported
 	}
 	return s.Internal.StorageInfo(p0, p1)
 }

-func (s *StorageMinerStub) StorageInfo(p0 context.Context, p1 stores.ID) (stores.StorageInfo, error) {
-	return *new(stores.StorageInfo), ErrNotSupported
+func (s *StorageMinerStub) StorageInfo(p0 context.Context, p1 storiface.ID) (storiface.StorageInfo, error) {
+	return *new(storiface.StorageInfo), ErrNotSupported
 }

-func (s *StorageMinerStruct) StorageList(p0 context.Context) (map[stores.ID][]stores.Decl, error) {
+func (s *StorageMinerStruct) StorageList(p0 context.Context) (map[storiface.ID][]storiface.Decl, error) {
 	if s.Internal.StorageList == nil {
-		return *new(map[stores.ID][]stores.Decl), ErrNotSupported
+		return *new(map[storiface.ID][]storiface.Decl), ErrNotSupported
 	}
 	return s.Internal.StorageList(p0)
 }

-func (s *StorageMinerStub) StorageList(p0 context.Context) (map[stores.ID][]stores.Decl, error) {
-	return *new(map[stores.ID][]stores.Decl), ErrNotSupported
+func (s *StorageMinerStub) StorageList(p0 context.Context) (map[storiface.ID][]storiface.Decl, error) {
+	return *new(map[storiface.ID][]storiface.Decl), ErrNotSupported
 }

-func (s *StorageMinerStruct) StorageLocal(p0 context.Context) (map[stores.ID]string, error) {
+func (s *StorageMinerStruct) StorageLocal(p0 context.Context) (map[storiface.ID]string, error) {
 	if s.Internal.StorageLocal == nil {
-		return *new(map[stores.ID]string), ErrNotSupported
+		return *new(map[storiface.ID]string), ErrNotSupported
 	}
 	return s.Internal.StorageLocal(p0)
 }

-func (s *StorageMinerStub) StorageLocal(p0 context.Context) (map[stores.ID]string, error) {
-	return *new(map[stores.ID]string), ErrNotSupported
+func (s *StorageMinerStub) StorageLocal(p0 context.Context) (map[storiface.ID]string, error) {
+	return *new(map[storiface.ID]string), ErrNotSupported
 }

 func (s *StorageMinerStruct) StorageLock(p0 context.Context, p1 abi.SectorID, p2 storiface.SectorFileType, p3 storiface.SectorFileType) error {
@@ -4908,25 +4992,25 @@ func (s *StorageMinerStub) StorageLock(p0 context.Context, p1 abi.SectorID, p2 s
 	return ErrNotSupported
 }

-func (s *StorageMinerStruct) StorageReportHealth(p0 context.Context, p1 stores.ID, p2 stores.HealthReport) error {
+func (s *StorageMinerStruct) StorageReportHealth(p0 context.Context, p1 storiface.ID, p2 storiface.HealthReport) error {
 	if s.Internal.StorageReportHealth == nil {
 		return ErrNotSupported
 	}
 	return s.Internal.StorageReportHealth(p0, p1, p2)
 }

-func (s *StorageMinerStub) StorageReportHealth(p0 context.Context, p1 stores.ID, p2 stores.HealthReport) error {
+func (s *StorageMinerStub) StorageReportHealth(p0 context.Context, p1 storiface.ID, p2 storiface.HealthReport) error {
 	return ErrNotSupported
 }

-func (s *StorageMinerStruct) StorageStat(p0 context.Context, p1 stores.ID) (fsutil.FsStat, error) {
+func (s *StorageMinerStruct) StorageStat(p0 context.Context, p1 storiface.ID) (fsutil.FsStat, error) {
 	if s.Internal.StorageStat == nil {
 		return *new(fsutil.FsStat), ErrNotSupported
 	}
 	return s.Internal.StorageStat(p0, p1)
 }

-func (s *StorageMinerStub) StorageStat(p0 context.Context, p1 stores.ID) (fsutil.FsStat, error) {
+func (s *StorageMinerStub) StorageStat(p0 context.Context, p1 storiface.ID) (fsutil.FsStat, error) {
 	return *new(fsutil.FsStat), ErrNotSupported
 }

@@ -5084,6 +5168,17 @@ func (s *WorkerStub) Fetch(p0 context.Context, p1 storage.SectorRef, p2 storifac
 	return *new(storiface.CallID), ErrNotSupported
 }

+func (s *WorkerStruct) FinalizeReplicaUpdate(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) {
+	if s.Internal.FinalizeReplicaUpdate == nil {
+		return *new(storiface.CallID), ErrNotSupported
+	}
+	return s.Internal.FinalizeReplicaUpdate(p0, p1, p2)
+}
+
+func (s *WorkerStub) FinalizeReplicaUpdate(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) {
+	return *new(storiface.CallID), ErrNotSupported
+}
+
 func (s *WorkerStruct) FinalizeSector(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) {
 	if s.Internal.FinalizeSector == nil {
 		return *new(storiface.CallID), ErrNotSupported
@@ -5106,6 +5201,28 @@ func (s *WorkerStub) GenerateSectorKeyFromData(p0 context.Context, p1 storage.Se
 	return *new(storiface.CallID), ErrNotSupported
 }

+func (s *WorkerStruct) GenerateWindowPoSt(p0 context.Context, p1 abi.RegisteredPoStProof, p2 abi.ActorID, p3 []storiface.PostSectorChallenge, p4 int, p5 abi.PoStRandomness) (storiface.WindowPoStResult, error) {
+	if s.Internal.GenerateWindowPoSt == nil {
+		return *new(storiface.WindowPoStResult), ErrNotSupported
+	}
+	return s.Internal.GenerateWindowPoSt(p0, p1, p2, p3, p4, p5)
+}
+
+func (s *WorkerStub) GenerateWindowPoSt(p0 context.Context, p1 abi.RegisteredPoStProof, p2 abi.ActorID, p3 []storiface.PostSectorChallenge, p4 int, p5 abi.PoStRandomness) (storiface.WindowPoStResult, error) {
+	return *new(storiface.WindowPoStResult), ErrNotSupported
+}
+
+func (s *WorkerStruct) GenerateWinningPoSt(p0 context.Context, p1 abi.RegisteredPoStProof, p2 abi.ActorID, p3 []storiface.PostSectorChallenge, p4 abi.PoStRandomness) ([]proof.PoStProof, error) {
+	if s.Internal.GenerateWinningPoSt == nil {
+		return *new([]proof.PoStProof), ErrNotSupported
+	}
+	return s.Internal.GenerateWinningPoSt(p0, p1, p2, p3, p4)
+}
+
+func (s *WorkerStub) GenerateWinningPoSt(p0 context.Context, p1 abi.RegisteredPoStProof, p2 abi.ActorID, p3 []storiface.PostSectorChallenge, p4 abi.PoStRandomness) ([]proof.PoStProof, error) {
+	return *new([]proof.PoStProof), ErrNotSupported
+}
+
 func (s *WorkerStruct) Info(p0 context.Context) (storiface.WorkerInfo, error) {
 	if s.Internal.Info == nil {
 		return *new(storiface.WorkerInfo), ErrNotSupported
@@ -5128,15 +5245,15 @@ func (s *WorkerStub) MoveStorage(p0 context.Context, p1 storage.SectorRef, p2 st
 	return *new(storiface.CallID), ErrNotSupported
 }

-func (s *WorkerStruct) Paths(p0 context.Context) ([]stores.StoragePath, error) {
+func (s *WorkerStruct) Paths(p0 context.Context) ([]storiface.StoragePath, error) {
 	if s.Internal.Paths == nil {
-		return *new([]stores.StoragePath), ErrNotSupported
+		return *new([]storiface.StoragePath), ErrNotSupported
 	}
 	return s.Internal.Paths(p0)
 }

-func (s *WorkerStub) Paths(p0 context.Context) ([]stores.StoragePath, error) {
-	return *new([]stores.StoragePath), ErrNotSupported
+func (s *WorkerStub) Paths(p0 context.Context) ([]storiface.StoragePath, error) {
+	return *new([]storiface.StoragePath), ErrNotSupported
 }

 func (s *WorkerStruct) ProcessSession(p0 context.Context) (uuid.UUID, error) {
@@ -1,3 +1,4 @@
+//stm: #unit
 package api

 import (
@@ -29,6 +30,7 @@ type StrC struct {
 }

 func TestGetInternalStructs(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_API_STRUCTS_001
 	var proxy StrA

 	sts := GetInternalStructs(&proxy)
@@ -44,6 +46,7 @@ func TestGetInternalStructs(t *testing.T) {
 }

 func TestNestedInternalStructs(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_API_STRUCTS_001
 	var proxy StrC

 	// check that only the top-level internal struct gets picked up
@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"time"

+	"github.com/libp2p/go-libp2p-core/network"
+
 	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-state-types/abi"
@@ -12,7 +14,6 @@ import (
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-graphsync"

-	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	ma "github.com/multiformats/go-multiaddr"
@@ -58,7 +59,7 @@ type MessageSendSpec struct {
 // GraphSyncDataTransfer provides diagnostics on a data transfer happening over graphsync
 type GraphSyncDataTransfer struct {
 	// GraphSync request id for this transfer
-	RequestID graphsync.RequestID
+	RequestID *graphsync.RequestID
 	// Graphsync state for this transfer
 	RequestState string
 	// If a channel ID is present, indicates whether this is the current graphsync request for this channel
@@ -124,12 +125,6 @@ func NewDataTransferChannel(hostID peer.ID, channelState datatransfer.ChannelSta
 	return channel
 }

-type NetBlockList struct {
-	Peers     []peer.ID
-	IPAddrs   []string
-	IPSubnets []string
-}
-
 type NetStat struct {
 	System    *network.ScopeStat           `json:",omitempty"`
 	Transient *network.ScopeStat           `json:",omitempty"`
@@ -152,6 +147,12 @@ type NetLimit struct {
 	FD                                       int
 }

+type NetBlockList struct {
+	Peers     []peer.ID
+	IPAddrs   []string
+	IPSubnets []string
+}
+
 type ExtendedPeerInfo struct {
 	ID          peer.ID
 	Agent       string
@@ -7,6 +7,7 @@ package v0mocks
 import (
 	context "context"
 	reflect "reflect"
+	time "time"

 	address "github.com/filecoin-project/go-address"
 	bitfield "github.com/filecoin-project/go-bitfield"
@@ -1769,6 +1770,21 @@ func (mr *MockFullNodeMockRecorder) NetPeers(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPeers", reflect.TypeOf((*MockFullNode)(nil).NetPeers), arg0)
 }

+// NetPing mocks base method.
+func (m *MockFullNode) NetPing(arg0 context.Context, arg1 peer.ID) (time.Duration, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetPing", arg0, arg1)
+	ret0, _ := ret[0].(time.Duration)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetPing indicates an expected call of NetPing.
+func (mr *MockFullNodeMockRecorder) NetPing(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPing", reflect.TypeOf((*MockFullNode)(nil).NetPing), arg0, arg1)
+}
+
 // NetProtectAdd mocks base method.
 func (m *MockFullNode) NetProtectAdd(arg0 context.Context, arg1 []peer.ID) error {
 	m.ctrl.T.Helper()
@@ -3,6 +3,9 @@ package v0api
 import (
 	"context"

+	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/libp2p/go-libp2p-core/peer"
+
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/crypto"
@@ -337,4 +340,16 @@ func (w *WrapperV1Full) clientRetrieve(ctx context.Context, order RetrievalOrder
 	finish(w.ClientExport(ctx, eref, *ref))
 }

+func (w *WrapperV1Full) PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*api.ChannelInfo, error) {
+	return w.FullNode.PaychFund(ctx, from, to, amt)
+}
+
+func (w *WrapperV1Full) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error) {
+	a, err := w.FullNode.ClientQueryAsk(ctx, p, miner)
+	if err != nil {
+		return nil, err
+	}
+	return a.Response, nil
+}
+
 var _ FullNode = &WrapperV1Full{}
@@ -57,8 +57,8 @@ var (
 	FullAPIVersion0 = newVer(1, 5, 0)
 	FullAPIVersion1 = newVer(2, 2, 0)

-	MinerAPIVersion0  = newVer(1, 3, 0)
-	WorkerAPIVersion0 = newVer(1, 5, 0)
+	MinerAPIVersion0  = newVer(1, 5, 0)
+	WorkerAPIVersion0 = newVer(1, 6, 0)
 )

 //nolint:varcheck,deadcode
@@ -1,10 +1,10 @@
+//stm: #unit
 package badgerbs

 import (
 	"bytes"
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -20,6 +20,8 @@ import (
 )

 func TestBadgerBlockstore(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
 	(&Suite{
 		NewBlockstore:  newBlockstore(DefaultOptions),
 		OpenBlockstore: openBlockstore(DefaultOptions),
@@ -38,6 +40,8 @@ func TestBadgerBlockstore(t *testing.T) {
 }

 func TestStorageKey(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_STORAGE_KEY_001
 	bs, _ := newBlockstore(DefaultOptions)(t)
 	bbs := bs.(*Blockstore)
 	defer bbs.Close() //nolint:errcheck
@@ -73,20 +77,13 @@ func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (
 	return func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
 		tb.Helper()

-		path, err := ioutil.TempDir("", "")
-		if err != nil {
-			tb.Fatal(err)
-		}
+		path = tb.TempDir()

 		db, err := Open(optsSupplier(path))
 		if err != nil {
 			tb.Fatal(err)
 		}

-		tb.Cleanup(func() {
-			_ = os.RemoveAll(path)
-		})
-
 		return db, path
 	}
 }
@@ -100,17 +97,10 @@ func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB,

 func testMove(t *testing.T, optsF func(string) Options) {
 	ctx := context.Background()
-	basePath, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
+	basePath := t.TempDir()

 	dbPath := filepath.Join(basePath, "db")

-	t.Cleanup(func() {
-		_ = os.RemoveAll(basePath)
-	})
-
 	db, err := Open(optsF(dbPath))
 	if err != nil {
 		t.Fatal(err)
@@ -265,10 +255,16 @@ func testMove(t *testing.T, optsF func(string) Options) {
 }

 func TestMoveNoPrefix(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_COLLECT_GARBAGE_001
 	testMove(t, DefaultOptions)
 }

 func TestMoveWithPrefix(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_COLLECT_GARBAGE_001
 	testMove(t, func(path string) Options {
 		opts := DefaultOptions(path)
 		opts.Prefix = "/prefixed/"
@@ -1,3 +1,4 @@
+//stm: #unit
 package badgerbs

 import (
@@ -44,6 +45,8 @@ func (s *Suite) RunTests(t *testing.T, prefix string) {
 }

 func (s *Suite) TestGetWhenKeyNotPresent(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_GET_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
 	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
@@ -57,6 +60,8 @@ func (s *Suite) TestGetWhenKeyNotPresent(t *testing.T) {
 }

 func (s *Suite) TestGetWhenKeyIsNil(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_GET_001
 	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
@@ -68,6 +73,9 @@ func (s *Suite) TestGetWhenKeyIsNil(t *testing.T) {
 }

 func (s *Suite) TestPutThenGetBlock(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_GET_001
 	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
@@ -85,6 +93,8 @@ func (s *Suite) TestPutThenGetBlock(t *testing.T) {
 }

 func (s *Suite) TestHas(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_HAS_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
 	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
@@ -106,6 +116,9 @@ func (s *Suite) TestHas(t *testing.T) {
 }

 func (s *Suite) TestCidv0v1(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_GET_001
 	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
@@ -123,6 +136,9 @@ func (s *Suite) TestCidv0v1(t *testing.T) {
 }

 func (s *Suite) TestPutThenGetSizeBlock(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_GET_SIZE_001
 	ctx := context.Background()

 	bs, _ := s.NewBlockstore(t)
@@ -154,6 +170,8 @@ func (s *Suite) TestPutThenGetSizeBlock(t *testing.T) {
 }

 func (s *Suite) TestAllKeysSimple(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
@@ -170,6 +188,9 @@ func (s *Suite) TestAllKeysSimple(t *testing.T) {
 }

 func (s *Suite) TestAllKeysRespectsContext(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_ALL_KEYS_CHAN_001
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
 		defer func() { require.NoError(t, c.Close()) }()
@@ -200,6 +221,7 @@ func (s *Suite) TestAllKeysRespectsContext(t *testing.T) {
 }

 func (s *Suite) TestDoubleClose(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
 	bs, _ := s.NewBlockstore(t)
 	c, ok := bs.(io.Closer)
 	if !ok {
@@ -210,6 +232,9 @@ func (s *Suite) TestDoubleClose(t *testing.T) {
 }

 func (s *Suite) TestReopenPutGet(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_GET_001
 	ctx := context.Background()
 	bs, path := s.NewBlockstore(t)
 	c, ok := bs.(io.Closer)
@@ -236,6 +261,10 @@ func (s *Suite) TestReopenPutGet(t *testing.T) {
 }

 func (s *Suite) TestPutMany(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
+	//stm: @SPLITSTORE_BADGER_HAS_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_GET_001, @SPLITSTORE_BADGER_PUT_MANY_001
+	//stm: @SPLITSTORE_BADGER_ALL_KEYS_CHAN_001
 	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
@@ -268,6 +297,11 @@ func (s *Suite) TestPutMany(t *testing.T) {
 }

 func (s *Suite) TestDelete(t *testing.T) {
+	//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
+	//stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_POOLED_STORAGE_HAS_001
+	//stm: @SPLITSTORE_BADGER_ALL_KEYS_CHAN_001, @SPLITSTORE_BADGER_HAS_001
+	//stm: @SPLITSTORE_BADGER_PUT_MANY_001
+
 	ctx := context.Background()
 	bs, _ := s.NewBlockstore(t)
 	if c, ok := bs.(io.Closer); ok {
@@ -0,0 +1,21 @@
+package blockstore
+
+import (
+	"context"
+)
+
+type hotViewKey struct{}
+
+var hotView = hotViewKey{}
+
+// WithHotView constructs a new context with an option that provides a hint to the blockstore
+// (e.g. the splitstore) that the object (and its ipld references) should be kept hot.
+func WithHotView(ctx context.Context) context.Context {
+	return context.WithValue(ctx, hotView, struct{}{})
+}
+
+// IsHotView returns true if the hot view option is set in the context
+func IsHotView(ctx context.Context) bool {
+	v := ctx.Value(hotView)
+	return v != nil
+}
@@ -49,10 +49,11 @@ These are options in the `[Chainstore.Splitstore]` section of the configuration:
  blockstore and discards writes; this is necessary to support syncing from a snapshot.
 - `MarkSetType` -- specifies the type of markset to use during compaction.
  The markset is the data structure used by compaction/gc to track live objects.
-  The default value is `"map"`, which will use an in-memory map; if you are limited
-  in memory (or indeed see compaction run out of memory), you can also specify
-  `"badger"` which will use an disk backed markset, using badger. This will use
-  much less memory, but will also make compaction slower.
+  The default value is "badger", which will use a disk backed markset using badger.
+  If you have a lot of memory (48G or more) you can also use "map", which will use
+  an in memory markset, speeding up compaction at the cost of higher memory usage.
+  Note: If you are using a VPS with a network volume, you need to provision at least
+  3000 IOPs with the badger markset.
 - `HotStoreMessageRetention` -- specifies how many finalities, beyond the 4
  finalities maintained by default, to maintain messages and message receipts in the
  hotstore. This is useful for assistive nodes that want to support syncing for other
@@ -105,6 +106,12 @@ Compaction works transactionally with the following algorithm:
  - We delete in small batches taking a lock; each batch is checked again for marks, from the concurrent transactional mark, so as to never delete anything live
 - We then end the transaction and compact/gc the hotstore.

+As of [#8008](https://github.com/filecoin-project/lotus/pull/8008) the compaction algorithm has been
+modified to eliminate sorting and maintain the cold object set on disk. This drastically reduces
+memory usage; in fact, when using badger as the markset compaction uses very little memory, and
+it should be now possible to run splitstore with 32GB of RAM or less without danger of running out of
+memory during compaction.
+
 ## Garbage Collection

 TBD -- see [#6577](https://github.com/filecoin-project/lotus/issues/6577)
@@ -0,0 +1,118 @@
+package splitstore
+
+import (
+	"bufio"
+	"io"
+	"os"
+
+	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
+	mh "github.com/multiformats/go-multihash"
+)
+
+type Checkpoint struct {
+	file *os.File
+	buf  *bufio.Writer
+}
+
+func NewCheckpoint(path string) (*Checkpoint, error) {
+	file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_SYNC, 0644)
+	if err != nil {
+		return nil, xerrors.Errorf("error creating checkpoint: %w", err)
+	}
+	buf := bufio.NewWriter(file)
+
+	return &Checkpoint{
+		file: file,
+		buf:  buf,
+	}, nil
+}
+
+func OpenCheckpoint(path string) (*Checkpoint, cid.Cid, error) {
+	filein, err := os.Open(path)
+	if err != nil {
+		return nil, cid.Undef, xerrors.Errorf("error opening checkpoint for reading: %w", err)
+	}
+	defer filein.Close() //nolint:errcheck
+
+	bufin := bufio.NewReader(filein)
+	start, err := readRawCid(bufin, nil)
+	if err != nil && err != io.EOF {
+		return nil, cid.Undef, xerrors.Errorf("error reading cid from checkpoint: %w", err)
+	}
+
+	fileout, err := os.OpenFile(path, os.O_WRONLY|os.O_SYNC, 0644)
+	if err != nil {
+		return nil, cid.Undef, xerrors.Errorf("error opening checkpoint for writing: %w", err)
+	}
+	bufout := bufio.NewWriter(fileout)
+
+	return &Checkpoint{
+		file: fileout,
+		buf:  bufout,
+	}, start, nil
+}
+
+func (cp *Checkpoint) Set(c cid.Cid) error {
+	if _, err := cp.file.Seek(0, io.SeekStart); err != nil {
+		return xerrors.Errorf("error seeking beginning of checkpoint: %w", err)
+	}
+
+	if err := writeRawCid(cp.buf, c, true); err != nil {
+		return xerrors.Errorf("error writing cid to checkpoint: %w", err)
+	}
+
+	return nil
+}
+
+func (cp *Checkpoint) Close() error {
+	if cp.file == nil {
+		return nil
+	}
+
+	err := cp.file.Close()
+	cp.file = nil
+	cp.buf = nil
+
+	return err
+}
+
+func readRawCid(buf *bufio.Reader, hbuf []byte) (cid.Cid, error) {
+	sz, err := buf.ReadByte()
+	if err != nil {
+		return cid.Undef, err // don't wrap EOF as it is not an error here
+	}
+
+	if hbuf == nil {
+		hbuf = make([]byte, int(sz))
+	} else {
+		hbuf = hbuf[:int(sz)]
+	}
+
+	if _, err := io.ReadFull(buf, hbuf); err != nil {
+		return cid.Undef, xerrors.Errorf("error reading hash: %w", err) // wrap EOF, it's corrupt
+	}
+
+	hash, err := mh.Cast(hbuf)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("error casting multihash: %w", err)
+	}
+
+	return cid.NewCidV1(cid.Raw, hash), nil
+}
+
+func writeRawCid(buf *bufio.Writer, c cid.Cid, flush bool) error {
+	hash := c.Hash()
+	if err := buf.WriteByte(byte(len(hash))); err != nil {
+		return err
+	}
+	if _, err := buf.Write(hash); err != nil {
+		return err
+	}
+	if flush {
+		return buf.Flush()
+	}
+
+	return nil
+}
@@ -0,0 +1,138 @@
+package splitstore
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+)
+
+func TestCheckpoint(t *testing.T) {
+	dir := t.TempDir()
+
+	path := filepath.Join(dir, "checkpoint")
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	cp, err := NewCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Set(k1); err != nil {
+		t.Fatal(err)
+	}
+	if err := cp.Set(k2); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err := OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !start.Equals(k2) {
+		t.Fatalf("expected start to be %s; got %s", k2, start)
+	}
+
+	if err := cp.Set(k3); err != nil {
+		t.Fatal(err)
+	}
+	if err := cp.Set(k4); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err = OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !start.Equals(k4) {
+		t.Fatalf("expected start to be %s; got %s", k4, start)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// also test correct operation with an empty checkpoint
+	cp, err = NewCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err = OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if start.Defined() {
+		t.Fatal("expected start to be undefined")
+	}
+
+	if err := cp.Set(k1); err != nil {
+		t.Fatal(err)
+	}
+	if err := cp.Set(k2); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err = OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !start.Equals(k2) {
+		t.Fatalf("expected start to be %s; got %s", k2, start)
+	}
+
+	if err := cp.Set(k3); err != nil {
+		t.Fatal(err)
+	}
+	if err := cp.Set(k4); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err = OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !start.Equals(k4) {
+		t.Fatalf("expected start to be %s; got %s", k4, start)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+}
@@ -0,0 +1,102 @@
+package splitstore
+
+import (
+	"bufio"
+	"io"
+	"os"
+
+	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
+)
+
+type ColdSetWriter struct {
+	file *os.File
+	buf  *bufio.Writer
+}
+
+type ColdSetReader struct {
+	file *os.File
+	buf  *bufio.Reader
+}
+
+func NewColdSetWriter(path string) (*ColdSetWriter, error) {
+	file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return nil, xerrors.Errorf("error creating coldset: %w", err)
+	}
+	buf := bufio.NewWriter(file)
+
+	return &ColdSetWriter{
+		file: file,
+		buf:  buf,
+	}, nil
+}
+
+func NewColdSetReader(path string) (*ColdSetReader, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, xerrors.Errorf("error opening coldset: %w", err)
+	}
+	buf := bufio.NewReader(file)
+
+	return &ColdSetReader{
+		file: file,
+		buf:  buf,
+	}, nil
+}
+
+func (s *ColdSetWriter) Write(c cid.Cid) error {
+	return writeRawCid(s.buf, c, false)
+}
+
+func (s *ColdSetWriter) Close() error {
+	if s.file == nil {
+		return nil
+	}
+
+	err1 := s.buf.Flush()
+	err2 := s.file.Close()
+	s.buf = nil
+	s.file = nil
+
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
+
+func (s *ColdSetReader) ForEach(f func(cid.Cid) error) error {
+	hbuf := make([]byte, 256)
+	for {
+		next, err := readRawCid(s.buf, hbuf)
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+
+			return xerrors.Errorf("error reading coldset: %w", err)
+		}
+
+		if err := f(next); err != nil {
+			return err
+		}
+	}
+}
+
+func (s *ColdSetReader) Reset() error {
+	_, err := s.file.Seek(0, io.SeekStart)
+	return err
+}
+
+func (s *ColdSetReader) Close() error {
+	if s.file == nil {
+		return nil
+	}
+
+	err := s.file.Close()
+	s.file = nil
+	s.buf = nil
+
+	return err
+}
@@ -0,0 +1,90 @@
+package splitstore
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+)
+
+func TestColdSet(t *testing.T) {
+	dir := t.TempDir()
+
+	path := filepath.Join(dir, "coldset")
+
+	makeCid := func(i int) cid.Cid {
+		h, err := multihash.Sum([]byte(fmt.Sprintf("cid.%d", i)), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	const count = 1000
+	cids := make([]cid.Cid, 0, count)
+	for i := 0; i < count; i++ {
+		cids = append(cids, makeCid(i))
+	}
+
+	cw, err := NewColdSetWriter(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, c := range cids {
+		if err := cw.Write(c); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if err := cw.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cr, err := NewColdSetReader(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	index := 0
+	err = cr.ForEach(func(c cid.Cid) error {
+		if index >= count {
+			t.Fatal("too many cids")
+		}
+
+		if !c.Equals(cids[index]) {
+			t.Fatalf("wrong cid %d; expected %s but got %s", index, cids[index], c)
+		}
+
+		index++
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cr.Reset(); err != nil {
+		t.Fatal(err)
+	}
+
+	index = 0
+	err = cr.ForEach(func(c cid.Cid) error {
+		if index >= count {
+			t.Fatal("too many cids")
+		}
+
+		if !c.Equals(cids[index]) {
+			t.Fatalf("wrong cid; expected %s but got %s", cids[index], c)
+		}
+
+		index++
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+}
@@ -14,15 +14,24 @@ var errMarkSetClosed = errors.New("markset closed")
 type MarkSet interface {
 	ObjectVisitor
 	Mark(cid.Cid) error
+	MarkMany([]cid.Cid) error
 	Has(cid.Cid) (bool, error)
 	Close() error
+
+	// BeginCriticalSection ensures that the markset is persisted to disk for recovery in case
+	// of abnormal termination during the critical section span.
+	BeginCriticalSection() error
+	// EndCriticalSection ends the critical section span.
+	EndCriticalSection()
 }

 type MarkSetEnv interface {
-	// Create creates a new markset within the environment.
-	// name is a unique name for this markset, mapped to the filesystem in disk-backed environments
+	// New creates a new markset within the environment.
+	// name is a unique name for this markset, mapped to the filesystem for on-disk persistence.
 	// sizeHint is a hint about the expected size of the markset
-	Create(name string, sizeHint int64) (MarkSet, error)
+	New(name string, sizeHint int64) (MarkSet, error)
+	// Recover recovers an existing markset persisted on-disk.
+	Recover(name string) (MarkSet, error)
 	// Close closes the markset
 	Close() error
 }
@@ -30,7 +39,7 @@ type MarkSetEnv interface {
 func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) {
 	switch mtype {
 	case "map":
-		return NewMapMarkSetEnv()
+		return NewMapMarkSetEnv(path)
 	case "badger":
 		return NewBadgerMarkSetEnv(path)
 	default:
@@ -3,6 +3,7 @@ package splitstore
 import (
 	"os"
 	"path/filepath"
+	"runtime"
 	"sync"

 	"golang.org/x/xerrors"
@@ -28,6 +29,7 @@ type BadgerMarkSet struct {
 	writers int
 	seqno   int
 	version int
+	persist bool

 	db   *badger.DB
 	path string
@@ -47,11 +49,10 @@ func NewBadgerMarkSetEnv(path string) (MarkSetEnv, error) {
 	return &BadgerMarkSetEnv{path: msPath}, nil
 }

-func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
-	name += ".tmp"
+func (e *BadgerMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
 	path := filepath.Join(e.path, name)

-	db, err := openTransientBadgerDB(path)
+	db, err := openBadgerDB(path, false)
 	if err != nil {
 		return nil, xerrors.Errorf("error creating badger db: %w", err)
 	}
@@ -67,8 +68,72 @@ func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error)
 	return ms, nil
 }

+func (e *BadgerMarkSetEnv) Recover(name string) (MarkSet, error) {
+	path := filepath.Join(e.path, name)
+
+	if _, err := os.Stat(path); err != nil {
+		return nil, xerrors.Errorf("error stating badger db path: %w", err)
+	}
+
+	db, err := openBadgerDB(path, true)
+	if err != nil {
+		return nil, xerrors.Errorf("error creating badger db: %w", err)
+	}
+
+	ms := &BadgerMarkSet{
+		pend:    make(map[string]struct{}),
+		writing: make(map[int]map[string]struct{}),
+		db:      db,
+		path:    path,
+		persist: true,
+	}
+	ms.cond.L = &ms.mx
+
+	return ms, nil
+}
+
 func (e *BadgerMarkSetEnv) Close() error {
-	return os.RemoveAll(e.path)
+	return nil
+}
+
+func (s *BadgerMarkSet) BeginCriticalSection() error {
+	s.mx.Lock()
+
+	if s.persist {
+		s.mx.Unlock()
+		return nil
+	}
+
+	var write bool
+	var seqno int
+	if len(s.pend) > 0 {
+		write = true
+		seqno = s.nextBatch()
+	}
+
+	s.persist = true
+	s.mx.Unlock()
+
+	if write {
+		// all writes sync once perist is true
+		return s.write(seqno)
+	}
+
+	// wait for any pending writes and sync
+	s.mx.Lock()
+	for s.writers > 0 {
+		s.cond.Wait()
+	}
+	s.mx.Unlock()
+
+	return s.db.Sync()
+}
+
+func (s *BadgerMarkSet) EndCriticalSection() {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	s.persist = false
 }

 func (s *BadgerMarkSet) Mark(c cid.Cid) error {
@@ -88,6 +153,23 @@ func (s *BadgerMarkSet) Mark(c cid.Cid) error {
 	return nil
 }

+func (s *BadgerMarkSet) MarkMany(batch []cid.Cid) error {
+	s.mx.Lock()
+	if s.pend == nil {
+		s.mx.Unlock()
+		return errMarkSetClosed
+	}
+
+	write, seqno := s.putMany(batch)
+	s.mx.Unlock()
+
+	if write {
+		return s.write(seqno)
+	}
+
+	return nil
+}
+
 func (s *BadgerMarkSet) Has(c cid.Cid) (bool, error) {
 	s.mx.RLock()
 	defer s.mx.RUnlock()
@@ -193,16 +275,34 @@ func (s *BadgerMarkSet) tryDB(key []byte) (has bool, err error) {
 // writer holds the exclusive lock
 func (s *BadgerMarkSet) put(key string) (write bool, seqno int) {
 	s.pend[key] = struct{}{}
-	if len(s.pend) < badgerMarkSetBatchSize {
+	if !s.persist && len(s.pend) < badgerMarkSetBatchSize {
 		return false, 0
 	}

-	seqno = s.seqno
+	seqno = s.nextBatch()
+	return true, seqno
+}
+
+func (s *BadgerMarkSet) putMany(batch []cid.Cid) (write bool, seqno int) {
+	for _, c := range batch {
+		key := string(c.Hash())
+		s.pend[key] = struct{}{}
+	}
+
+	if !s.persist && len(s.pend) < badgerMarkSetBatchSize {
+		return false, 0
+	}
+
+	seqno = s.nextBatch()
+	return true, seqno
+}
+
+func (s *BadgerMarkSet) nextBatch() int {
+	seqno := s.seqno
 	s.seqno++
 	s.writing[seqno] = s.pend
 	s.pend = make(map[string]struct{})
-
-	return true, seqno
+	return seqno
 }

 func (s *BadgerMarkSet) write(seqno int) (err error) {
@@ -247,6 +347,14 @@ func (s *BadgerMarkSet) write(seqno int) (err error) {
 		return xerrors.Errorf("error flushing batch to badger markset: %w", err)
 	}

+	s.mx.RLock()
+	persist := s.persist
+	s.mx.RUnlock()
+
+	if persist {
+		return s.db.Sync()
+	}
+
 	return nil
 }

@@ -266,26 +374,29 @@ func (s *BadgerMarkSet) Close() error {
 	db := s.db
 	s.db = nil

-	return closeTransientBadgerDB(db, s.path)
+	return closeBadgerDB(db, s.path, s.persist)
 }

-func (s *BadgerMarkSet) SetConcurrent() {}
+func openBadgerDB(path string, recover bool) (*badger.DB, error) {
+	// if it is not a recovery, clean up first
+	if !recover {
+		err := os.RemoveAll(path)
+		if err != nil {
+			return nil, xerrors.Errorf("error clearing markset directory: %w", err)
+		}

-func openTransientBadgerDB(path string) (*badger.DB, error) {
-	// clean up first
-	err := os.RemoveAll(path)
-	if err != nil {
-		return nil, xerrors.Errorf("error clearing markset directory: %w", err)
-	}
-
-	err = os.MkdirAll(path, 0755) //nolint:gosec
-	if err != nil {
-		return nil, xerrors.Errorf("error creating markset directory: %w", err)
+		err = os.MkdirAll(path, 0755) //nolint:gosec
+		if err != nil {
+			return nil, xerrors.Errorf("error creating markset directory: %w", err)
+		}
 	}

 	opts := badger.DefaultOptions(path)
+	// we manually sync when we are in critical section
 	opts.SyncWrites = false
+	// no need to do that
 	opts.CompactL0OnClose = false
+	// we store hashes, not much to gain by compression
 	opts.Compression = options.None
 	// Note: We use FileIO for loading modes to avoid memory thrashing and interference
 	//       between the system blockstore and the markset.
@@ -294,6 +405,15 @@ func openTransientBadgerDB(path string) (*badger.DB, error) {
 	//       exceeded 1GB in size.
 	opts.TableLoadingMode = options.FileIO
 	opts.ValueLogLoadingMode = options.FileIO
+	// We increase the number of L0 tables before compaction to make it unlikely to
+	// be necessary.
+	opts.NumLevelZeroTables = 20      // default is 5
+	opts.NumLevelZeroTablesStall = 30 // default is 10
+	// increase the number of compactors from default 2 so that if we ever have to
+	// compact, it is fast
+	if runtime.NumCPU()/2 > opts.NumCompactors {
+		opts.NumCompactors = runtime.NumCPU() / 2
+	}
 	opts.Logger = &badgerLogger{
 		SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(),
 		skip2:         log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(),
@@ -302,12 +422,16 @@ func openTransientBadgerDB(path string) (*badger.DB, error) {
 	return badger.Open(opts)
 }

-func closeTransientBadgerDB(db *badger.DB, path string) error {
+func closeBadgerDB(db *badger.DB, path string, persist bool) error {
 	err := db.Close()
 	if err != nil {
 		return xerrors.Errorf("error closing badger markset: %w", err)
 	}

+	if persist {
+		return nil
+	}
+
 	err = os.RemoveAll(path)
 	if err != nil {
 		return xerrors.Errorf("error deleting badger markset: %w", err)
@@ -1,37 +1,104 @@
 package splitstore

 import (
+	"bufio"
+	"io"
+	"os"
+	"path/filepath"
 	"sync"

+	"golang.org/x/xerrors"
+
 	cid "github.com/ipfs/go-cid"
 )

-type MapMarkSetEnv struct{}
+type MapMarkSetEnv struct {
+	path string
+}

 var _ MarkSetEnv = (*MapMarkSetEnv)(nil)

 type MapMarkSet struct {
 	mx  sync.RWMutex
 	set map[string]struct{}
+
+	persist bool
+	file    *os.File
+	buf     *bufio.Writer
+
+	path string
 }

 var _ MarkSet = (*MapMarkSet)(nil)

-func NewMapMarkSetEnv() (*MapMarkSetEnv, error) {
-	return &MapMarkSetEnv{}, nil
+func NewMapMarkSetEnv(path string) (*MapMarkSetEnv, error) {
+	msPath := filepath.Join(path, "markset.map")
+	err := os.MkdirAll(msPath, 0755) //nolint:gosec
+	if err != nil {
+		return nil, xerrors.Errorf("error creating markset directory: %w", err)
+	}
+
+	return &MapMarkSetEnv{path: msPath}, nil
 }

-func (e *MapMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
+func (e *MapMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
+	path := filepath.Join(e.path, name)
 	return &MapMarkSet{
-		set: make(map[string]struct{}, sizeHint),
+		set:  make(map[string]struct{}, sizeHint),
+		path: path,
 	}, nil
 }

+func (e *MapMarkSetEnv) Recover(name string) (MarkSet, error) {
+	path := filepath.Join(e.path, name)
+	s := &MapMarkSet{
+		set:  make(map[string]struct{}),
+		path: path,
+	}
+
+	in, err := os.Open(path)
+	if err != nil {
+		return nil, xerrors.Errorf("error opening markset file for read: %w", err)
+	}
+	defer in.Close() //nolint:errcheck
+
+	// wrap a buffered reader to make this faster
+	buf := bufio.NewReader(in)
+	for {
+		var sz byte
+		if sz, err = buf.ReadByte(); err != nil {
+			break
+		}
+
+		key := make([]byte, int(sz))
+		if _, err = io.ReadFull(buf, key); err != nil {
+			break
+		}
+
+		s.set[string(key)] = struct{}{}
+	}
+
+	if err != io.EOF {
+		return nil, xerrors.Errorf("error reading markset file: %w", err)
+	}
+
+	file, err := os.OpenFile(s.path, os.O_WRONLY|os.O_APPEND, 0)
+	if err != nil {
+		return nil, xerrors.Errorf("error opening markset file for write: %w", err)
+	}
+
+	s.persist = true
+	s.file = file
+	s.buf = bufio.NewWriter(file)
+
+	return s, nil
+}
+
 func (e *MapMarkSetEnv) Close() error {
 	return nil
 }

-func (s *MapMarkSet) Mark(cid cid.Cid) error {
+func (s *MapMarkSet) BeginCriticalSection() error {
 	s.mx.Lock()
 	defer s.mx.Unlock()

@@ -39,7 +106,104 @@ func (s *MapMarkSet) Mark(cid cid.Cid) error {
 		return errMarkSetClosed
 	}

-	s.set[string(cid.Hash())] = struct{}{}
+	if s.persist {
+		return nil
+	}
+
+	file, err := os.OpenFile(s.path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return xerrors.Errorf("error opening markset file: %w", err)
+	}
+
+	// wrap a buffered writer to make this faster
+	s.buf = bufio.NewWriter(file)
+	for key := range s.set {
+		if err := s.writeKey([]byte(key), false); err != nil {
+			_ = file.Close()
+			s.buf = nil
+			return err
+		}
+	}
+	if err := s.buf.Flush(); err != nil {
+		_ = file.Close()
+		s.buf = nil
+		return xerrors.Errorf("error flushing markset file buffer: %w", err)
+	}
+
+	s.file = file
+	s.persist = true
+
+	return nil
+}
+
+func (s *MapMarkSet) EndCriticalSection() {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	if !s.persist {
+		return
+	}
+
+	_ = s.file.Close()
+	_ = os.Remove(s.path)
+	s.file = nil
+	s.buf = nil
+	s.persist = false
+}
+
+func (s *MapMarkSet) Mark(c cid.Cid) error {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	if s.set == nil {
+		return errMarkSetClosed
+	}
+
+	hash := c.Hash()
+	s.set[string(hash)] = struct{}{}
+
+	if s.persist {
+		if err := s.writeKey(hash, true); err != nil {
+			return err
+		}
+
+		if err := s.file.Sync(); err != nil {
+			return xerrors.Errorf("error syncing markset: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (s *MapMarkSet) MarkMany(batch []cid.Cid) error {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	if s.set == nil {
+		return errMarkSetClosed
+	}
+
+	for _, c := range batch {
+		hash := c.Hash()
+		s.set[string(hash)] = struct{}{}
+
+		if s.persist {
+			if err := s.writeKey(hash, false); err != nil {
+				return err
+			}
+		}
+	}
+
+	if s.persist {
+		if err := s.buf.Flush(); err != nil {
+			return xerrors.Errorf("error flushing markset buffer to disk: %w", err)
+		}
+
+		if err := s.file.Sync(); err != nil {
+			return xerrors.Errorf("error syncing markset: %w", err)
+		}
+	}
+
 	return nil
 }

@@ -63,12 +227,23 @@ func (s *MapMarkSet) Visit(c cid.Cid) (bool, error) {
 		return false, errMarkSetClosed
 	}

-	key := string(c.Hash())
+	hash := c.Hash()
+	key := string(hash)
 	if _, ok := s.set[key]; ok {
 		return false, nil
 	}

 	s.set[key] = struct{}{}
+
+	if s.persist {
+		if err := s.writeKey(hash, true); err != nil {
+			return false, err
+		}
+		if err := s.file.Sync(); err != nil {
+			return false, xerrors.Errorf("error syncing markset: %w", err)
+		}
+	}
+
 	return true, nil
 }

@@ -76,6 +251,39 @@ func (s *MapMarkSet) Close() error {
 	s.mx.Lock()
 	defer s.mx.Unlock()

+	if s.set == nil {
+		return nil
+	}
+
 	s.set = nil
+
+	if s.file != nil {
+		if err := s.file.Close(); err != nil {
+			log.Warnf("error closing markset file: %s", err)
+		}
+
+		if !s.persist {
+			if err := os.Remove(s.path); err != nil {
+				log.Warnf("error removing markset file: %s", err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (s *MapMarkSet) writeKey(k []byte, flush bool) error {
+	if err := s.buf.WriteByte(byte(len(k))); err != nil {
+		return xerrors.Errorf("error writing markset key length to disk: %w", err)
+	}
+	if _, err := s.buf.Write(k); err != nil {
+		return xerrors.Errorf("error writing markset key to disk: %w", err)
+	}
+	if flush {
+		if err := s.buf.Flush(); err != nil {
+			return xerrors.Errorf("error flushing markset buffer to disk: %w", err)
+		}
+	}
+
 	return nil
 }
@@ -1,8 +1,7 @@
+//stm: #unit
 package splitstore

 import (
-	"io/ioutil"
-	"os"
 	"testing"

 	cid "github.com/ipfs/go-cid"
@@ -10,31 +9,32 @@ import (
 )

 func TestMapMarkSet(t *testing.T) {
+	//stm: @SPLITSTORE_MARKSET_CREATE_001, @SPLITSTORE_MARKSET_HAS_001, @@SPLITSTORE_MARKSET_MARK_001
+	//stm: @SPLITSTORE_MARKSET_CLOSE_001, @SPLITSTORE_MARKSET_CREATE_VISITOR_001
 	testMarkSet(t, "map")
+	testMarkSetRecovery(t, "map")
+	testMarkSetMarkMany(t, "map")
 	testMarkSetVisitor(t, "map")
+	testMarkSetVisitorRecovery(t, "map")
 }

 func TestBadgerMarkSet(t *testing.T) {
+	//stm: @SPLITSTORE_MARKSET_CREATE_001, @SPLITSTORE_MARKSET_HAS_001, @@SPLITSTORE_MARKSET_MARK_001
+	//stm: @SPLITSTORE_MARKSET_CLOSE_001, @SPLITSTORE_MARKSET_CREATE_VISITOR_001
 	bs := badgerMarkSetBatchSize
 	badgerMarkSetBatchSize = 1
 	t.Cleanup(func() {
 		badgerMarkSetBatchSize = bs
 	})
 	testMarkSet(t, "badger")
+	testMarkSetRecovery(t, "badger")
+	testMarkSetMarkMany(t, "badger")
 	testMarkSetVisitor(t, "badger")
+	testMarkSetVisitorRecovery(t, "badger")
 }

 func testMarkSet(t *testing.T, lsType string) {
-	t.Helper()
-
-	path, err := ioutil.TempDir("", "markset.*")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() {
-		_ = os.RemoveAll(path)
-	})
+	path := t.TempDir()

 	env, err := OpenMarkSetEnv(path, lsType)
 	if err != nil {
@@ -42,12 +42,13 @@ func testMarkSet(t *testing.T, lsType string) {
 	}
 	defer env.Close() //nolint:errcheck

-	hotSet, err := env.Create("hot", 0)
+	// stm: @SPLITSTORE_MARKSET_CREATE_001
+	hotSet, err := env.New("hot", 0)
 	if err != nil {
 		t.Fatal(err)
 	}

-	coldSet, err := env.Create("cold", 0)
+	coldSet, err := env.New("cold", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -61,7 +62,9 @@ func testMarkSet(t *testing.T, lsType string) {
 		return cid.NewCidV1(cid.Raw, h)
 	}

+	// stm: @SPLITSTORE_MARKSET_HAS_001
 	mustHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
 		has, err := s.Has(cid)
 		if err != nil {
 			t.Fatal(err)
@@ -73,6 +76,7 @@ func testMarkSet(t *testing.T, lsType string) {
 	}

 	mustNotHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
 		has, err := s.Has(cid)
 		if err != nil {
 			t.Fatal(err)
@@ -88,6 +92,7 @@ func testMarkSet(t *testing.T, lsType string) {
 	k3 := makeCid("c")
 	k4 := makeCid("d")

+	// stm: @SPLITSTORE_MARKSET_MARK_001
 	hotSet.Mark(k1)  //nolint
 	hotSet.Mark(k2)  //nolint
 	coldSet.Mark(k3) //nolint
@@ -114,12 +119,12 @@ func testMarkSet(t *testing.T, lsType string) {
 		t.Fatal(err)
 	}

-	hotSet, err = env.Create("hot", 0)
+	hotSet, err = env.New("hot", 0)
 	if err != nil {
 		t.Fatal(err)
 	}

-	coldSet, err = env.Create("cold", 0)
+	coldSet, err = env.New("cold", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -138,6 +143,7 @@ func testMarkSet(t *testing.T, lsType string) {
 	mustNotHave(coldSet, k3)
 	mustNotHave(coldSet, k4)

+	//stm: @SPLITSTORE_MARKSET_CLOSE_001
 	err = hotSet.Close()
 	if err != nil {
 		t.Fatal(err)
@@ -150,16 +156,7 @@ func testMarkSet(t *testing.T, lsType string) {
 }

 func testMarkSetVisitor(t *testing.T, lsType string) {
-	t.Helper()
-
-	path, err := ioutil.TempDir("", "markset.*")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() {
-		_ = os.RemoveAll(path)
-	})
+	path := t.TempDir()

 	env, err := OpenMarkSetEnv(path, lsType)
 	if err != nil {
@@ -167,7 +164,8 @@ func testMarkSetVisitor(t *testing.T, lsType string) {
 	}
 	defer env.Close() //nolint:errcheck

-	visitor, err := env.Create("test", 0)
+	//stm: @SPLITSTORE_MARKSET_CREATE_VISITOR_001
+	visitor, err := env.New("test", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -219,3 +217,301 @@ func testMarkSetVisitor(t *testing.T, lsType string) {
 	mustNotVisit(visitor, k3)
 	mustNotVisit(visitor, k4)
 }
+
+func testMarkSetVisitorRecovery(t *testing.T, lsType string) {
+	path := t.TempDir()
+
+	env, err := OpenMarkSetEnv(path, lsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer env.Close() //nolint:errcheck
+
+	visitor, err := env.New("test", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer visitor.Close() //nolint:errcheck
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	mustVisit := func(v ObjectVisitor, cid cid.Cid) {
+		visit, err := v.Visit(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !visit {
+			t.Fatal("object should be visited")
+		}
+	}
+
+	mustNotVisit := func(v ObjectVisitor, cid cid.Cid) {
+		visit, err := v.Visit(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if visit {
+			t.Fatal("unexpected visit")
+		}
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	mustVisit(visitor, k1)
+	mustVisit(visitor, k2)
+
+	if err := visitor.BeginCriticalSection(); err != nil {
+		t.Fatal(err)
+	}
+
+	mustVisit(visitor, k3)
+	mustVisit(visitor, k4)
+
+	mustNotVisit(visitor, k1)
+	mustNotVisit(visitor, k2)
+	mustNotVisit(visitor, k3)
+	mustNotVisit(visitor, k4)
+
+	if err := visitor.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	visitor, err = env.Recover("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mustNotVisit(visitor, k1)
+	mustNotVisit(visitor, k2)
+	mustNotVisit(visitor, k3)
+	mustNotVisit(visitor, k4)
+
+	visitor.EndCriticalSection()
+
+	if err := visitor.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = env.Recover("test")
+	if err == nil {
+		t.Fatal("expected recovery to fail")
+	}
+}
+
+func testMarkSetRecovery(t *testing.T, lsType string) {
+	path := t.TempDir()
+
+	env, err := OpenMarkSetEnv(path, lsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer env.Close() //nolint:errcheck
+
+	markSet, err := env.New("test", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	mustHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !has {
+			t.Fatal("mark not found")
+		}
+	}
+
+	mustNotHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("unexpected mark")
+		}
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	if err := markSet.Mark(k1); err != nil {
+		t.Fatal(err)
+	}
+	if err := markSet.Mark(k2); err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustNotHave(markSet, k3)
+	mustNotHave(markSet, k4)
+
+	if err := markSet.BeginCriticalSection(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := markSet.Mark(k3); err != nil {
+		t.Fatal(err)
+	}
+	if err := markSet.Mark(k4); err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustHave(markSet, k3)
+	mustHave(markSet, k4)
+
+	if err := markSet.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	markSet, err = env.Recover("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustHave(markSet, k3)
+	mustHave(markSet, k4)
+
+	markSet.EndCriticalSection()
+
+	if err := markSet.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = env.Recover("test")
+	if err == nil {
+		t.Fatal("expected recovery to fail")
+	}
+}
+
+func testMarkSetMarkMany(t *testing.T, lsType string) {
+	path := t.TempDir()
+
+	env, err := OpenMarkSetEnv(path, lsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer env.Close() //nolint:errcheck
+
+	markSet, err := env.New("test", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	mustHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !has {
+			t.Fatal("mark not found")
+		}
+	}
+
+	mustNotHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("unexpected mark")
+		}
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	if err := markSet.MarkMany([]cid.Cid{k1, k2}); err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustNotHave(markSet, k3)
+	mustNotHave(markSet, k4)
+
+	if err := markSet.BeginCriticalSection(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := markSet.MarkMany([]cid.Cid{k3, k4}); err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustHave(markSet, k3)
+	mustHave(markSet, k4)
+
+	if err := markSet.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	markSet, err = env.Recover("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustHave(markSet, k3)
+	mustHave(markSet, k4)
+
+	markSet.EndCriticalSection()
+
+	if err := markSet.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = env.Recover("test")
+	if err == nil {
+		t.Fatal("expected recovery to fail")
+	}
+}
@@ -129,8 +129,6 @@ type SplitStore struct {

 	headChangeMx sync.Mutex

-	coldPurgeSize int
-
 	chain ChainAccessor
 	ds    dstore.Datastore
 	cold  bstore.Blockstore
@@ -158,6 +156,17 @@ type SplitStore struct {
 	txnRefsMx       sync.Mutex
 	txnRefs         map[cid.Cid]struct{}
 	txnMissing      map[cid.Cid]struct{}
+	txnMarkSet      MarkSet
+	txnSyncMx       sync.Mutex
+	txnSyncCond     sync.Cond
+	txnSync         bool
+
+	// background cold object reification
+	reifyWorkers    sync.WaitGroup
+	reifyMx         sync.Mutex
+	reifyCond       sync.Cond
+	reifyPend       map[cid.Cid]struct{}
+	reifyInProgress map[cid.Cid]struct{}

 	// registered protectors
 	protectors []func(func(cid.Cid) error) error
@@ -194,13 +203,16 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 		cold:       cold,
 		hot:        hots,
 		markSetEnv: markSetEnv,
-
-		coldPurgeSize: defaultColdPurgeSize,
 	}

 	ss.txnViewsCond.L = &ss.txnViewsMx
+	ss.txnSyncCond.L = &ss.txnSyncMx
 	ss.ctx, ss.cancel = context.WithCancel(context.Background())

+	ss.reifyCond.L = &ss.reifyMx
+	ss.reifyPend = make(map[cid.Cid]struct{})
+	ss.reifyInProgress = make(map[cid.Cid]struct{})
+
 	if enableDebugLog {
 		ss.debug, err = openDebugLog(path)
 		if err != nil {
@@ -208,6 +220,14 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 		}
 	}

+	if ss.checkpointExists() {
+		log.Info("found compaction checkpoint; resuming compaction")
+		if err := ss.completeCompaction(); err != nil {
+			markSetEnv.Close() //nolint:errcheck
+			return nil, xerrors.Errorf("error resuming compaction: %w", err)
+		}
+	}
+
 	return ss, nil
 }

@@ -230,6 +250,20 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()

+	// critical section
+	if s.txnMarkSet != nil {
+		has, err := s.txnMarkSet.Has(cid)
+		if err != nil {
+			return false, err
+		}
+
+		if has {
+			return s.has(cid)
+		}
+
+		return s.cold.Has(ctx, cid)
+	}
+
 	has, err := s.hot.Has(ctx, cid)

 	if err != nil {
@@ -241,7 +275,13 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 		return true, nil
 	}

-	return s.cold.Has(ctx, cid)
+	has, err = s.cold.Has(ctx, cid)
+	if has && bstore.IsHotView(ctx) {
+		s.reifyColdObject(cid)
+	}
+
+	return has, err
+
 }

 func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
@@ -257,6 +297,20 @@ func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()

+	// critical section
+	if s.txnMarkSet != nil {
+		has, err := s.txnMarkSet.Has(cid)
+		if err != nil {
+			return nil, err
+		}
+
+		if has {
+			return s.get(cid)
+		}
+
+		return s.cold.Get(ctx, cid)
+	}
+
 	blk, err := s.hot.Get(ctx, cid)

 	switch err {
@@ -271,8 +325,11 @@ func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)

 		blk, err = s.cold.Get(ctx, cid)
 		if err == nil {
-			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
+			if bstore.IsHotView(ctx) {
+				s.reifyColdObject(cid)
+			}

+			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return blk, err

@@ -294,6 +351,20 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()

+	// critical section
+	if s.txnMarkSet != nil {
+		has, err := s.txnMarkSet.Has(cid)
+		if err != nil {
+			return 0, err
+		}
+
+		if has {
+			return s.getSize(cid)
+		}
+
+		return s.cold.GetSize(ctx, cid)
+	}
+
 	size, err := s.hot.GetSize(ctx, cid)

 	switch err {
@@ -308,6 +379,10 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {

 		size, err = s.cold.GetSize(ctx, cid)
 		if err == nil {
+			if bstore.IsHotView(ctx) {
+				s.reifyColdObject(cid)
+			}
+
 			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return size, err
@@ -332,6 +407,12 @@ func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error {

 	s.debug.LogWrite(blk)

+	// critical section
+	if s.txnMarkSet != nil {
+		s.markLiveRefs([]cid.Cid{blk.Cid()})
+		return nil
+	}
+
 	s.trackTxnRef(blk.Cid())
 	return nil
 }
@@ -377,6 +458,12 @@ func (s *SplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {

 	s.debug.LogWriteMany(blks)

+	// critical section
+	if s.txnMarkSet != nil {
+		s.markLiveRefs(batch)
+		return nil
+	}
+
 	s.trackTxnRefMany(batch)
 	return nil
 }
@@ -436,6 +523,23 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
 		return cb(data)
 	}

+	// critical section
+	s.txnLk.RLock() // the lock is released in protectView if we are not in critical section
+	if s.txnMarkSet != nil {
+		has, err := s.txnMarkSet.Has(cid)
+		s.txnLk.RUnlock()
+
+		if err != nil {
+			return err
+		}
+
+		if has {
+			return s.view(cid, cb)
+		}
+
+		return s.cold.View(ctx, cid, cb)
+	}
+
 	// views are (optimistically) protected two-fold:
 	// - if there is an active transaction, then the reference is protected.
 	// - if there is no active transaction, active views are tracked in a
@@ -456,6 +560,10 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro

 		err = s.cold.View(ctx, cid, cb)
 		if err == nil {
+			if bstore.IsHotView(ctx) {
+				s.reifyColdObject(cid)
+			}
+
 			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return err
@@ -565,6 +673,9 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
 		}
 	}

+	// spawn the reifier
+	go s.reifyOrchestrator()
+
 	// watch the chain
 	chain.SubscribeHeadChanges(s.HeadChange)

@@ -585,12 +696,19 @@ func (s *SplitStore) Close() error {
 	}

 	if atomic.LoadInt32(&s.compacting) == 1 {
+		s.txnSyncMx.Lock()
+		s.txnSync = true
+		s.txnSyncCond.Broadcast()
+		s.txnSyncMx.Unlock()
+
 		log.Warn("close with ongoing compaction in progress; waiting for it to finish...")
 		for atomic.LoadInt32(&s.compacting) == 1 {
 			time.Sleep(time.Second)
 		}
 	}

+	s.reifyCond.Broadcast()
+	s.reifyWorkers.Wait()
 	s.cancel()
 	return multierr.Combine(s.markSetEnv.Close(), s.debug.Close())
 }
@@ -89,7 +89,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 	coldCnt := new(int64)
 	missingCnt := new(int64)

-	visitor, err := s.markSetEnv.Create("check", 0)
+	visitor, err := s.markSetEnv.New("check", 0)
 	if err != nil {
 		return xerrors.Errorf("error creating visitor: %w", err)
 	}
@@ -3,8 +3,9 @@ package splitstore
 import (
 	"bytes"
 	"errors"
+	"os"
+	"path/filepath"
 	"runtime"
-	"sort"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -48,6 +49,10 @@ var (
 	// SyncGapTime is the time delay from a tipset's min timestamp before we decide
 	// there is a sync gap
 	SyncGapTime = time.Minute
+
+	// SyncWaitTime is the time delay from a tipset's min timestamp before we decide
+	// we have synced.
+	SyncWaitTime = 30 * time.Second
 )

 var (
@@ -57,8 +62,6 @@ var (

 const (
 	batchSize = 16384
-
-	defaultColdPurgeSize = 7_000_000
 )

 func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
@@ -141,9 +144,9 @@ func (s *SplitStore) isNearUpgrade(epoch abi.ChainEpoch) bool {
 // transactionally protect incoming tipsets
 func (s *SplitStore) protectTipSets(apply []*types.TipSet) {
 	s.txnLk.RLock()
-	defer s.txnLk.RUnlock()

 	if !s.txnActive {
+		s.txnLk.RUnlock()
 		return
 	}

@@ -152,12 +155,115 @@ func (s *SplitStore) protectTipSets(apply []*types.TipSet) {
 		cids = append(cids, ts.Cids()...)
 	}

+	if len(cids) == 0 {
+		s.txnLk.RUnlock()
+		return
+	}
+
+	// critical section
+	if s.txnMarkSet != nil {
+		curTs := apply[len(apply)-1]
+		timestamp := time.Unix(int64(curTs.MinTimestamp()), 0)
+		doSync := time.Since(timestamp) < SyncWaitTime
+		go func() {
+			if doSync {
+				defer func() {
+					s.txnSyncMx.Lock()
+					defer s.txnSyncMx.Unlock()
+					s.txnSync = true
+					s.txnSyncCond.Broadcast()
+				}()
+			}
+			defer s.txnLk.RUnlock()
+			s.markLiveRefs(cids)
+
+		}()
+		return
+	}
+
 	s.trackTxnRefMany(cids)
+	s.txnLk.RUnlock()
+}
+
+func (s *SplitStore) markLiveRefs(cids []cid.Cid) {
+	log.Debugf("marking %d live refs", len(cids))
+	startMark := time.Now()
+
+	count := new(int32)
+	visitor := newConcurrentVisitor()
+	walkObject := func(c cid.Cid) error {
+		return s.walkObjectIncomplete(c, visitor,
+			func(c cid.Cid) error {
+				if isUnitaryObject(c) {
+					return errStopWalk
+				}
+
+				visit, err := s.txnMarkSet.Visit(c)
+				if err != nil {
+					return xerrors.Errorf("error visiting object: %w", err)
+				}
+
+				if !visit {
+					return errStopWalk
+				}
+
+				atomic.AddInt32(count, 1)
+				return nil
+			},
+			func(missing cid.Cid) error {
+				log.Warnf("missing object reference %s in %s", missing, c)
+				return errStopWalk
+			})
+	}
+
+	// optimize the common case of single put
+	if len(cids) == 1 {
+		if err := walkObject(cids[0]); err != nil {
+			log.Errorf("error marking tipset refs: %s", err)
+		}
+		log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count)
+		return
+	}
+
+	workch := make(chan cid.Cid, len(cids))
+	for _, c := range cids {
+		workch <- c
+	}
+	close(workch)
+
+	worker := func() error {
+		for c := range workch {
+			if err := walkObject(c); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	workers := runtime.NumCPU() / 2
+	if workers < 2 {
+		workers = 2
+	}
+	if workers > len(cids) {
+		workers = len(cids)
+	}
+
+	g := new(errgroup.Group)
+	for i := 0; i < workers; i++ {
+		g.Go(worker)
+	}
+
+	if err := g.Wait(); err != nil {
+		log.Errorf("error marking tipset refs: %s", err)
+	}
+
+	log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count)
 }

 // transactionally protect a view
 func (s *SplitStore) protectView(c cid.Cid) {
-	s.txnLk.RLock()
+	//  the txnLk is held for read
 	defer s.txnLk.RUnlock()

 	if s.txnActive {
@@ -387,6 +493,12 @@ func (s *SplitStore) compact(curTs *types.TipSet) {
 }

 func (s *SplitStore) doCompact(curTs *types.TipSet) error {
+	if s.checkpointExists() {
+		// this really shouldn't happen, but if it somehow does, it means that the hotstore
+		// might be potentially inconsistent; abort compaction and notify the user to intervene.
+		return xerrors.Errorf("checkpoint exists; aborting compaction")
+	}
+
 	currentEpoch := curTs.Height()
 	boundaryEpoch := currentEpoch - CompactionBoundary

@@ -398,7 +510,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {

 	log.Infow("running compaction", "currentEpoch", currentEpoch, "baseEpoch", s.baseEpoch, "boundaryEpoch", boundaryEpoch, "inclMsgsEpoch", inclMsgsEpoch, "compactionIndex", s.compactionIndex)

-	markSet, err := s.markSetEnv.Create("live", s.markSetSize)
+	markSet, err := s.markSetEnv.New("live", s.markSetSize)
 	if err != nil {
 		return xerrors.Errorf("error creating mark set: %w", err)
 	}
@@ -409,9 +521,6 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		return err
 	}

-	// we are ready for concurrent marking
-	s.beginTxnMarking(markSet)
-
 	// 0. track all protected references at beginning of compaction; anything added later should
 	//    be transactionally protected by the write
 	log.Info("protecting references with registered protectors")
@@ -425,7 +534,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 	log.Info("marking reachable objects")
 	startMark := time.Now()

-	var count int64
+	count := new(int64)
 	err = s.walkChain(curTs, boundaryEpoch, inclMsgsEpoch, &noopVisitor{},
 		func(c cid.Cid) error {
 			if isUnitaryObject(c) {
@@ -441,7 +550,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 				return errStopWalk
 			}

-			count++
+			atomic.AddInt64(count, 1)
 			return nil
 		})

@@ -449,9 +558,9 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		return xerrors.Errorf("error marking: %w", err)
 	}

-	s.markSetSize = count + count>>2 // overestimate a bit
+	s.markSetSize = *count + *count>>2 // overestimate a bit

-	log.Infow("marking done", "took", time.Since(startMark), "marked", count)
+	log.Infow("marking done", "took", time.Since(startMark), "marked", *count)

 	if err := s.checkClosing(); err != nil {
 		return err
@@ -471,10 +580,15 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 	log.Info("collecting cold objects")
 	startCollect := time.Now()

+	coldw, err := NewColdSetWriter(s.coldSetPath())
+	if err != nil {
+		return xerrors.Errorf("error creating coldset: %w", err)
+	}
+	defer coldw.Close() //nolint:errcheck
+
 	// some stats for logging
 	var hotCnt, coldCnt int

-	cold := make([]cid.Cid, 0, s.coldPurgeSize)
 	err = s.hot.ForEachKey(func(c cid.Cid) error {
 		// was it marked?
 		mark, err := markSet.Has(c)
@@ -488,7 +602,9 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		}

 		// it's cold, mark it as candidate for move
-		cold = append(cold, c)
+		if err := coldw.Write(c); err != nil {
+			return xerrors.Errorf("error writing cid to coldstore: %w", err)
+		}
 		coldCnt++

 		return nil
@@ -498,12 +614,12 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		return xerrors.Errorf("error collecting cold objects: %w", err)
 	}

-	log.Infow("cold collection done", "took", time.Since(startCollect))
-
-	if coldCnt > 0 {
-		s.coldPurgeSize = coldCnt + coldCnt>>2 // overestimate a bit
+	if err := coldw.Close(); err != nil {
+		return xerrors.Errorf("error closing coldset: %w", err)
 	}

+	log.Infow("cold collection done", "took", time.Since(startCollect))
+
 	log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt)
 	stats.Record(s.ctx, metrics.SplitstoreCompactionHot.M(int64(hotCnt)))
 	stats.Record(s.ctx, metrics.SplitstoreCompactionCold.M(int64(coldCnt)))
@@ -521,11 +637,17 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		return err
 	}

+	coldr, err := NewColdSetReader(s.coldSetPath())
+	if err != nil {
+		return xerrors.Errorf("error opening coldset: %w", err)
+	}
+	defer coldr.Close() //nolint:errcheck
+
 	// 3. copy the cold objects to the coldstore -- if we have one
 	if !s.cfg.DiscardColdBlocks {
 		log.Info("moving cold objects to the coldstore")
 		startMove := time.Now()
-		err = s.moveColdBlocks(cold)
+		err = s.moveColdBlocks(coldr)
 		if err != nil {
 			return xerrors.Errorf("error moving cold objects: %w", err)
 		}
@@ -534,41 +656,64 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		if err := s.checkClosing(); err != nil {
 			return err
 		}
+
+		if err := coldr.Reset(); err != nil {
+			return xerrors.Errorf("error resetting coldset: %w", err)
+		}
 	}

-	// 4. sort cold objects so that the dags with most references are deleted first
-	//    this ensures that we can't refer to a dag with its consituents already deleted, ie
-	//    we lave no dangling references.
-	log.Info("sorting cold objects")
-	startSort := time.Now()
-	err = s.sortObjects(cold)
-	if err != nil {
-		return xerrors.Errorf("error sorting objects: %w", err)
-	}
-	log.Infow("sorting done", "took", time.Since(startSort))
-
-	// 4.1 protect transactional refs once more
-	//     strictly speaking, this is not necessary as purge will do it before deleting each
-	//     batch.  however, there is likely a largish number of references accumulated during
-	//     ths sort and this protects before entering pruge context.
-	err = s.protectTxnRefs(markSet)
-	if err != nil {
-		return xerrors.Errorf("error protecting transactional refs: %w", err)
+	// 4. Purge cold objects with checkpointing for recovery.
+	// This is the critical section of compaction, whereby any cold object not in the markSet is
+	// considered already deleted.
+	// We delete cold objects in batches, holding the transaction lock, where we check the markSet
+	// again for new references created by the VM.
+	// After each batch, we write a checkpoint to disk; if the process is interrupted before completion,
+	// the process will continue from the checkpoint in the next recovery.
+	if err := s.beginCriticalSection(markSet); err != nil {
+		return xerrors.Errorf("error beginning critical section: %w", err)
 	}

 	if err := s.checkClosing(); err != nil {
 		return err
 	}

+	// wait for the head to catch up so that the current tipset is marked
+	s.waitForSync()
+
+	if err := s.checkClosing(); err != nil {
+		return err
+	}
+
+	checkpoint, err := NewCheckpoint(s.checkpointPath())
+	if err != nil {
+		return xerrors.Errorf("error creating checkpoint: %w", err)
+	}
+	defer checkpoint.Close() //nolint:errcheck
+
 	// 5. purge cold objects from the hotstore, taking protected references into account
 	log.Info("purging cold objects from the hotstore")
 	startPurge := time.Now()
-	err = s.purge(cold, markSet)
+	err = s.purge(coldr, checkpoint, markSet)
 	if err != nil {
-		return xerrors.Errorf("error purging cold blocks: %w", err)
+		return xerrors.Errorf("error purging cold objects: %w", err)
 	}
 	log.Infow("purging cold objects from hotstore done", "took", time.Since(startPurge))

+	s.endCriticalSection()
+
+	if err := checkpoint.Close(); err != nil {
+		log.Warnf("error closing checkpoint: %s", err)
+	}
+	if err := os.Remove(s.checkpointPath()); err != nil {
+		log.Warnf("error removing checkpoint: %s", err)
+	}
+	if err := coldr.Close(); err != nil {
+		log.Warnf("error closing coldset: %s", err)
+	}
+	if err := os.Remove(s.coldSetPath()); err != nil {
+		log.Warnf("error removing coldset: %s", err)
+	}
+
 	// we are done; do some housekeeping
 	s.endTxnProtect()
 	s.gcHotstore()
@@ -599,12 +744,51 @@ func (s *SplitStore) beginTxnProtect() {
 	defer s.txnLk.Unlock()

 	s.txnActive = true
+	s.txnSync = false
 	s.txnRefs = make(map[cid.Cid]struct{})
 	s.txnMissing = make(map[cid.Cid]struct{})
 }

-func (s *SplitStore) beginTxnMarking(markSet MarkSet) {
-	log.Info("beginning transactional marking")
+func (s *SplitStore) beginCriticalSection(markSet MarkSet) error {
+	log.Info("beginning critical section")
+
+	// do that once first to get the bulk before the markset is in critical section
+	if err := s.protectTxnRefs(markSet); err != nil {
+		return xerrors.Errorf("error protecting transactional references: %w", err)
+	}
+
+	if err := markSet.BeginCriticalSection(); err != nil {
+		return xerrors.Errorf("error beginning critical section for markset: %w", err)
+	}
+
+	s.txnLk.Lock()
+	defer s.txnLk.Unlock()
+
+	s.txnMarkSet = markSet
+
+	// and do it again while holding the lock to mark references that might have been created
+	// in the meantime and avoid races of the type Has->txnRef->enterCS->Get fails because
+	// it's not in the markset
+	if err := s.protectTxnRefs(markSet); err != nil {
+		return xerrors.Errorf("error protecting transactional references: %w", err)
+	}
+
+	return nil
+}
+
+func (s *SplitStore) waitForSync() {
+	log.Info("waiting for sync")
+	startWait := time.Now()
+	defer func() {
+		log.Infow("waiting for sync done", "took", time.Since(startWait))
+	}()
+
+	s.txnSyncMx.Lock()
+	defer s.txnSyncMx.Unlock()
+
+	for !s.txnSync {
+		s.txnSyncCond.Wait()
+	}
 }

 func (s *SplitStore) endTxnProtect() {
@@ -616,8 +800,20 @@ func (s *SplitStore) endTxnProtect() {
 	}

 	s.txnActive = false
+	s.txnSync = false
 	s.txnRefs = nil
 	s.txnMissing = nil
+	s.txnMarkSet = nil
+}
+
+func (s *SplitStore) endCriticalSection() {
+	log.Info("ending critical section")
+
+	s.txnLk.Lock()
+	defer s.txnLk.Unlock()
+
+	s.txnMarkSet.EndCriticalSection()
+	s.txnMarkSet = nil
 }

 func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEpoch,
@@ -857,7 +1053,7 @@ func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, m
 	return nil
 }

-// internal version used by walk
+// internal version used during compaction and related operations
 func (s *SplitStore) view(c cid.Cid, cb func([]byte) error) error {
 	if isIdentiyCid(c) {
 		data, err := decodeIdentityCid(c)
@@ -892,10 +1088,34 @@ func (s *SplitStore) has(c cid.Cid) (bool, error) {
 	return s.cold.Has(s.ctx, c)
 }

-func (s *SplitStore) moveColdBlocks(cold []cid.Cid) error {
+func (s *SplitStore) get(c cid.Cid) (blocks.Block, error) {
+	blk, err := s.hot.Get(s.ctx, c)
+	switch err {
+	case nil:
+		return blk, nil
+	case bstore.ErrNotFound:
+		return s.cold.Get(s.ctx, c)
+	default:
+		return nil, err
+	}
+}
+
+func (s *SplitStore) getSize(c cid.Cid) (int, error) {
+	sz, err := s.hot.GetSize(s.ctx, c)
+	switch err {
+	case nil:
+		return sz, nil
+	case bstore.ErrNotFound:
+		return s.cold.GetSize(s.ctx, c)
+	default:
+		return 0, err
+	}
+}
+
+func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
 	batch := make([]blocks.Block, 0, batchSize)

-	for _, c := range cold {
+	err := coldr.ForEach(func(c cid.Cid) error {
 		if err := s.checkClosing(); err != nil {
 			return err
 		}
@@ -904,7 +1124,7 @@ func (s *SplitStore) moveColdBlocks(cold []cid.Cid) error {
 		if err != nil {
 			if err == bstore.ErrNotFound {
 				log.Warnf("hotstore missing block %s", c)
-				continue
+				return nil
 			}

 			return xerrors.Errorf("error retrieving block %s from hotstore: %w", c, err)
@@ -918,6 +1138,12 @@ func (s *SplitStore) moveColdBlocks(cold []cid.Cid) error {
 			}
 			batch = batch[:0]
 		}
+
+		return nil
+	})
+
+	if err != nil {
+		return xerrors.Errorf("error iterating coldset: %w", err)
 	}

 	if len(batch) > 0 {
@@ -930,177 +1156,202 @@ func (s *SplitStore) moveColdBlocks(cold []cid.Cid) error {
 	return nil
 }

-// sorts a slice of objects heaviest first -- it's a little expensive but worth the
-// guarantee that we don't leave dangling references behind, e.g. if we die in the middle
-// of a purge.
-func (s *SplitStore) sortObjects(cids []cid.Cid) error {
-	// we cache the keys to avoid making a gazillion of strings
-	keys := make(map[cid.Cid]string)
-	key := func(c cid.Cid) string {
-		s, ok := keys[c]
-		if !ok {
-			s = string(c.Hash())
-			keys[c] = s
-		}
-		return s
-	}
-
-	// compute sorting weights as the cumulative number of DAG links
-	weights := make(map[string]int)
-	for _, c := range cids {
-		// this can take quite a while, so check for shutdown with every opportunity
-		if err := s.checkClosing(); err != nil {
-			return err
-		}
-
-		w := s.getObjectWeight(c, weights, key)
-		weights[key(c)] = w
-	}
-
-	// sort!
-	sort.Slice(cids, func(i, j int) bool {
-		wi := weights[key(cids[i])]
-		wj := weights[key(cids[j])]
-		if wi == wj {
-			return bytes.Compare(cids[i].Hash(), cids[j].Hash()) > 0
-		}
-
-		return wi > wj
-	})
-
-	return nil
-}
-
-func (s *SplitStore) getObjectWeight(c cid.Cid, weights map[string]int, key func(cid.Cid) string) int {
-	w, ok := weights[key(c)]
-	if ok {
-		return w
-	}
-
-	// we treat block headers specially to avoid walking the entire chain
-	var hdr types.BlockHeader
-	err := s.view(c, func(data []byte) error {
-		return hdr.UnmarshalCBOR(bytes.NewBuffer(data))
-	})
-	if err == nil {
-		w1 := s.getObjectWeight(hdr.ParentStateRoot, weights, key)
-		weights[key(hdr.ParentStateRoot)] = w1
-
-		w2 := s.getObjectWeight(hdr.Messages, weights, key)
-		weights[key(hdr.Messages)] = w2
-
-		return 1 + w1 + w2
-	}
-
-	var links []cid.Cid
-	err = s.view(c, func(data []byte) error {
-		return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
-			links = append(links, c)
-		})
-	})
-	if err != nil {
-		return 1
-	}
-
-	w = 1
-	for _, c := range links {
-		// these are internal refs, so dags will be dags
-		if c.Prefix().Codec != cid.DagCBOR {
-			w++
-			continue
-		}
-
-		wc := s.getObjectWeight(c, weights, key)
-		weights[key(c)] = wc
-
-		w += wc
-	}
-
-	return w
-}
-
-func (s *SplitStore) purgeBatch(cids []cid.Cid, deleteBatch func([]cid.Cid) error) error {
-	if len(cids) == 0 {
-		return nil
-	}
-
-	// we don't delete one giant batch of millions of objects, but rather do smaller batches
-	// so that we don't stop the world for an extended period of time
-	done := false
-	for i := 0; !done; i++ {
-		start := i * batchSize
-		end := start + batchSize
-		if end >= len(cids) {
-			end = len(cids)
-			done = true
-		}
-
-		err := deleteBatch(cids[start:end])
-		if err != nil {
-			return xerrors.Errorf("error deleting batch: %w", err)
-		}
-	}
-
-	return nil
-}
-
-func (s *SplitStore) purge(cids []cid.Cid, markSet MarkSet) error {
+func (s *SplitStore) purge(coldr *ColdSetReader, checkpoint *Checkpoint, markSet MarkSet) error {
+	batch := make([]cid.Cid, 0, batchSize)
 	deadCids := make([]cid.Cid, 0, batchSize)
+
 	var purgeCnt, liveCnt int
 	defer func() {
 		log.Infow("purged cold objects", "purged", purgeCnt, "live", liveCnt)
 	}()

-	return s.purgeBatch(cids,
-		func(cids []cid.Cid) error {
-			deadCids := deadCids[:0]
+	deleteBatch := func() error {
+		pc, lc, err := s.purgeBatch(batch, deadCids, checkpoint, markSet)

-			for {
-				if err := s.checkClosing(); err != nil {
-					return err
-				}
+		purgeCnt += pc
+		liveCnt += lc
+		batch = batch[:0]

-				s.txnLk.Lock()
-				if len(s.txnRefs) == 0 {
-					// keep the lock!
-					break
-				}
+		return err
+	}

-				// unlock and protect
-				s.txnLk.Unlock()
+	err := coldr.ForEach(func(c cid.Cid) error {
+		batch = append(batch, c)
+		if len(batch) == batchSize {
+			return deleteBatch()
+		}

-				err := s.protectTxnRefs(markSet)
-				if err != nil {
-					return xerrors.Errorf("error protecting transactional refs: %w", err)
-				}
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	if len(batch) > 0 {
+		return deleteBatch()
+	}
+
+	return nil
+}
+
+func (s *SplitStore) purgeBatch(batch, deadCids []cid.Cid, checkpoint *Checkpoint, markSet MarkSet) (purgeCnt int, liveCnt int, err error) {
+	if err := s.checkClosing(); err != nil {
+		return 0, 0, err
+	}
+
+	s.txnLk.Lock()
+	defer s.txnLk.Unlock()
+
+	for _, c := range batch {
+		has, err := markSet.Has(c)
+		if err != nil {
+			return 0, 0, xerrors.Errorf("error checking markset for liveness: %w", err)
+		}
+
+		if has {
+			liveCnt++
+			continue
+		}
+
+		deadCids = append(deadCids, c)
+	}
+
+	if len(deadCids) == 0 {
+		if err := checkpoint.Set(batch[len(batch)-1]); err != nil {
+			return 0, 0, xerrors.Errorf("error setting checkpoint: %w", err)
+		}
+
+		return 0, liveCnt, nil
+	}
+
+	if err := s.hot.DeleteMany(s.ctx, deadCids); err != nil {
+		return 0, liveCnt, xerrors.Errorf("error purging cold objects: %w", err)
+	}
+
+	s.debug.LogDelete(deadCids)
+	purgeCnt = len(deadCids)
+
+	if err := checkpoint.Set(batch[len(batch)-1]); err != nil {
+		return purgeCnt, liveCnt, xerrors.Errorf("error setting checkpoint: %w", err)
+	}
+
+	return purgeCnt, liveCnt, nil
+}
+
+func (s *SplitStore) coldSetPath() string {
+	return filepath.Join(s.path, "coldset")
+}
+
+func (s *SplitStore) checkpointPath() string {
+	return filepath.Join(s.path, "checkpoint")
+}
+
+func (s *SplitStore) checkpointExists() bool {
+	_, err := os.Stat(s.checkpointPath())
+	return err == nil
+}
+
+func (s *SplitStore) completeCompaction() error {
+	checkpoint, last, err := OpenCheckpoint(s.checkpointPath())
+	if err != nil {
+		return xerrors.Errorf("error opening checkpoint: %w", err)
+	}
+	defer checkpoint.Close() //nolint:errcheck
+
+	coldr, err := NewColdSetReader(s.coldSetPath())
+	if err != nil {
+		return xerrors.Errorf("error opening coldset: %w", err)
+	}
+	defer coldr.Close() //nolint:errcheck
+
+	markSet, err := s.markSetEnv.Recover("live")
+	if err != nil {
+		return xerrors.Errorf("error recovering markset: %w", err)
+	}
+	defer markSet.Close() //nolint:errcheck
+
+	// PURGE
+	log.Info("purging cold objects from the hotstore")
+	startPurge := time.Now()
+	err = s.completePurge(coldr, checkpoint, last, markSet)
+	if err != nil {
+		return xerrors.Errorf("error purging cold objects: %w", err)
+	}
+	log.Infow("purging cold objects from hotstore done", "took", time.Since(startPurge))
+
+	markSet.EndCriticalSection()
+
+	if err := checkpoint.Close(); err != nil {
+		log.Warnf("error closing checkpoint: %s", err)
+	}
+	if err := os.Remove(s.checkpointPath()); err != nil {
+		log.Warnf("error removing checkpoint: %s", err)
+	}
+	if err := coldr.Close(); err != nil {
+		log.Warnf("error closing coldset: %s", err)
+	}
+	if err := os.Remove(s.coldSetPath()); err != nil {
+		log.Warnf("error removing coldset: %s", err)
+	}
+
+	// Note: at this point we can start the splitstore; a compaction should run on
+	//       the first head change, which will trigger gc on the hotstore.
+	//       We don't mind the second (back-to-back) compaction as the head will
+	//       have advanced during marking and coldset accumulation.
+	return nil
+}
+
+func (s *SplitStore) completePurge(coldr *ColdSetReader, checkpoint *Checkpoint, start cid.Cid, markSet MarkSet) error {
+	if !start.Defined() {
+		return s.purge(coldr, checkpoint, markSet)
+	}
+
+	seeking := true
+	batch := make([]cid.Cid, 0, batchSize)
+	deadCids := make([]cid.Cid, 0, batchSize)
+
+	var purgeCnt, liveCnt int
+	defer func() {
+		log.Infow("purged cold objects", "purged", purgeCnt, "live", liveCnt)
+	}()
+
+	deleteBatch := func() error {
+		pc, lc, err := s.purgeBatch(batch, deadCids, checkpoint, markSet)
+
+		purgeCnt += pc
+		liveCnt += lc
+		batch = batch[:0]
+
+		return err
+	}
+
+	err := coldr.ForEach(func(c cid.Cid) error {
+		if seeking {
+			if start.Equals(c) {
+				seeking = false
 			}

-			defer s.txnLk.Unlock()
-
-			for _, c := range cids {
-				live, err := markSet.Has(c)
-				if err != nil {
-					return xerrors.Errorf("error checking for liveness: %w", err)
-				}
-
-				if live {
-					liveCnt++
-					continue
-				}
-
-				deadCids = append(deadCids, c)
-			}
-
-			err := s.hot.DeleteMany(s.ctx, deadCids)
-			if err != nil {
-				return xerrors.Errorf("error purging cold objects: %w", err)
-			}
-
-			s.debug.LogDelete(deadCids)
-
-			purgeCnt += len(deadCids)
 			return nil
-		})
+		}
+
+		batch = append(batch, c)
+		if len(batch) == batchSize {
+			return deleteBatch()
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	if len(batch) > 0 {
+		return deleteBatch()
+	}
+
+	return nil
 }

 // I really don't like having this code, but we seem to have some occasional DAG references with
@@ -0,0 +1,214 @@
+package splitstore
+
+import (
+	"errors"
+	"runtime"
+	"sync/atomic"
+
+	"golang.org/x/xerrors"
+
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
+)
+
+var (
+	errReifyLimit = errors.New("reification limit reached")
+	ReifyLimit    = 16384
+)
+
+func (s *SplitStore) reifyColdObject(c cid.Cid) {
+	if !s.isWarm() {
+		return
+	}
+
+	if isUnitaryObject(c) {
+		return
+	}
+
+	s.reifyMx.Lock()
+	defer s.reifyMx.Unlock()
+
+	_, ok := s.reifyInProgress[c]
+	if ok {
+		return
+	}
+
+	s.reifyPend[c] = struct{}{}
+	s.reifyCond.Broadcast()
+}
+
+func (s *SplitStore) reifyOrchestrator() {
+	workers := runtime.NumCPU() / 4
+	if workers < 2 {
+		workers = 2
+	}
+
+	workch := make(chan cid.Cid, workers)
+	defer close(workch)
+
+	for i := 0; i < workers; i++ {
+		s.reifyWorkers.Add(1)
+		go s.reifyWorker(workch)
+	}
+
+	for {
+		s.reifyMx.Lock()
+		for len(s.reifyPend) == 0 && atomic.LoadInt32(&s.closing) == 0 {
+			s.reifyCond.Wait()
+		}
+
+		if atomic.LoadInt32(&s.closing) != 0 {
+			s.reifyMx.Unlock()
+			return
+		}
+
+		reifyPend := s.reifyPend
+		s.reifyPend = make(map[cid.Cid]struct{})
+		s.reifyMx.Unlock()
+
+		for c := range reifyPend {
+			select {
+			case workch <- c:
+			case <-s.ctx.Done():
+				return
+			}
+		}
+	}
+}
+
+func (s *SplitStore) reifyWorker(workch chan cid.Cid) {
+	defer s.reifyWorkers.Done()
+	for c := range workch {
+		s.doReify(c)
+	}
+}
+
+func (s *SplitStore) doReify(c cid.Cid) {
+	var toreify, totrack, toforget []cid.Cid
+
+	defer func() {
+		s.reifyMx.Lock()
+		defer s.reifyMx.Unlock()
+
+		for _, c := range toreify {
+			delete(s.reifyInProgress, c)
+		}
+		for _, c := range totrack {
+			delete(s.reifyInProgress, c)
+		}
+		for _, c := range toforget {
+			delete(s.reifyInProgress, c)
+		}
+	}()
+
+	s.txnLk.RLock()
+	defer s.txnLk.RUnlock()
+
+	count := 0
+	err := s.walkObjectIncomplete(c, newTmpVisitor(),
+		func(c cid.Cid) error {
+			if isUnitaryObject(c) {
+				return errStopWalk
+			}
+
+			count++
+			if count > ReifyLimit {
+				return errReifyLimit
+			}
+
+			s.reifyMx.Lock()
+			_, inProgress := s.reifyInProgress[c]
+			if !inProgress {
+				s.reifyInProgress[c] = struct{}{}
+			}
+			s.reifyMx.Unlock()
+
+			if inProgress {
+				return errStopWalk
+			}
+
+			has, err := s.hot.Has(s.ctx, c)
+			if err != nil {
+				return xerrors.Errorf("error checking hotstore: %w", err)
+			}
+
+			if has {
+				if s.txnMarkSet != nil {
+					hasMark, err := s.txnMarkSet.Has(c)
+					if err != nil {
+						log.Warnf("error checking markset: %s", err)
+					} else if hasMark {
+						toforget = append(toforget, c)
+						return errStopWalk
+					}
+				} else {
+					totrack = append(totrack, c)
+					return errStopWalk
+				}
+			}
+
+			toreify = append(toreify, c)
+			return nil
+		},
+		func(missing cid.Cid) error {
+			log.Warnf("missing reference while reifying %s: %s", c, missing)
+			return errStopWalk
+		})
+
+	if err != nil {
+		if xerrors.Is(err, errReifyLimit) {
+			log.Debug("reification aborted; reify limit reached")
+			return
+		}
+
+		log.Warnf("error walking cold object for reification (cid: %s): %s", c, err)
+		return
+	}
+
+	log.Debugf("reifying %d objects rooted at %s", len(toreify), c)
+
+	// this should not get too big, maybe some 100s of objects.
+	batch := make([]blocks.Block, 0, len(toreify))
+	for _, c := range toreify {
+		blk, err := s.cold.Get(s.ctx, c)
+		if err != nil {
+			log.Warnf("error retrieving cold object for reification (cid: %s): %s", c, err)
+			continue
+		}
+
+		if err := s.checkClosing(); err != nil {
+			return
+		}
+
+		batch = append(batch, blk)
+	}
+
+	if len(batch) > 0 {
+		err = s.hot.PutMany(s.ctx, batch)
+		if err != nil {
+			log.Warnf("error reifying cold object (cid: %s): %s", c, err)
+			return
+		}
+	}
+
+	if s.txnMarkSet != nil {
+		if len(toreify) > 0 {
+			if err := s.txnMarkSet.MarkMany(toreify); err != nil {
+				log.Warnf("error marking reified objects: %s", err)
+			}
+		}
+		if len(totrack) > 0 {
+			if err := s.txnMarkSet.MarkMany(totrack); err != nil {
+				log.Warnf("error marking tracked objects: %s", err)
+			}
+		}
+	} else {
+		// if txnActive is false these are noops
+		if len(toreify) > 0 {
+			s.trackTxnRefMany(toreify)
+		}
+		if len(totrack) > 0 {
+			s.trackTxnRefMany(totrack)
+		}
+	}
+}
@@ -1,9 +1,11 @@
+//stm: #unit
 package splitstore

 import (
 	"context"
 	"errors"
 	"fmt"
+	"math/rand"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -20,12 +22,14 @@ import (
 	datastore "github.com/ipfs/go-datastore"
 	dssync "github.com/ipfs/go-datastore/sync"
 	logging "github.com/ipfs/go-log/v2"
+	mh "github.com/multiformats/go-multihash"
 )

 func init() {
 	CompactionThreshold = 5
 	CompactionBoundary = 2
 	WarmupBoundary = 0
+	SyncWaitTime = time.Millisecond
 	logging.SetLogLevel("splitstore", "DEBUG")
 }

@@ -80,8 +84,10 @@ func testSplitStore(t *testing.T, cfg *Config) {
 		t.Fatal(err)
 	}

+	path := t.TempDir()
+
 	// open the splitstore
-	ss, err := Open("", ds, hot, cold, cfg)
+	ss, err := Open(path, ds, hot, cold, cfg)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -125,6 +131,10 @@ func testSplitStore(t *testing.T, cfg *Config) {
 	}

 	waitForCompaction := func() {
+		ss.txnSyncMx.Lock()
+		ss.txnSync = true
+		ss.txnSyncCond.Broadcast()
+		ss.txnSyncMx.Unlock()
 		for atomic.LoadInt32(&ss.compacting) == 1 {
 			time.Sleep(100 * time.Millisecond)
 		}
@@ -210,10 +220,16 @@ func testSplitStore(t *testing.T, cfg *Config) {
 }

 func TestSplitStoreCompaction(t *testing.T) {
+	//stm: @SPLITSTORE_SPLITSTORE_OPEN_001, @SPLITSTORE_SPLITSTORE_CLOSE_001
+	//stm: @SPLITSTORE_SPLITSTORE_PUT_001, @SPLITSTORE_SPLITSTORE_ADD_PROTECTOR_001
+	//stm: @SPLITSTORE_SPLITSTORE_CLOSE_001
 	testSplitStore(t, &Config{MarkSetType: "map"})
 }

 func TestSplitStoreCompactionWithBadger(t *testing.T) {
+	//stm: @SPLITSTORE_SPLITSTORE_OPEN_001, @SPLITSTORE_SPLITSTORE_CLOSE_001
+	//stm: @SPLITSTORE_SPLITSTORE_PUT_001, @SPLITSTORE_SPLITSTORE_ADD_PROTECTOR_001
+	//stm: @SPLITSTORE_SPLITSTORE_CLOSE_001
 	bs := badgerMarkSetBatchSize
 	badgerMarkSetBatchSize = 1
 	t.Cleanup(func() {
@@ -223,6 +239,9 @@ func TestSplitStoreCompactionWithBadger(t *testing.T) {
 }

 func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
+	//stm: @SPLITSTORE_SPLITSTORE_OPEN_001, @SPLITSTORE_SPLITSTORE_CLOSE_001
+	//stm: @SPLITSTORE_SPLITSTORE_PUT_001, @SPLITSTORE_SPLITSTORE_ADD_PROTECTOR_001
+	//stm: @SPLITSTORE_SPLITSTORE_CLOSE_001
 	ctx := context.Background()
 	chain := &mockChain{t: t}

@@ -259,8 +278,10 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
 		t.Fatal(err)
 	}

+	path := t.TempDir()
+
 	// open the splitstore
-	ss, err := Open("", ds, hot, cold, &Config{MarkSetType: "map"})
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -305,6 +326,10 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
 	}

 	waitForCompaction := func() {
+		ss.txnSyncMx.Lock()
+		ss.txnSync = true
+		ss.txnSyncCond.Broadcast()
+		ss.txnSyncMx.Unlock()
 		for atomic.LoadInt32(&ss.compacting) == 1 {
 			time.Sleep(100 * time.Millisecond)
 		}
@@ -357,6 +382,221 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
 	}
 }

+func testSplitStoreReification(t *testing.T, f func(context.Context, blockstore.Blockstore, cid.Cid) error) {
+	ds := dssync.MutexWrap(datastore.NewMapDatastore())
+	hot := newMockStore()
+	cold := newMockStore()
+
+	mkRandomBlock := func() blocks.Block {
+		data := make([]byte, 128)
+		_, err := rand.Read(data)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return blocks.NewBlock(data)
+	}
+
+	block1 := mkRandomBlock()
+	block2 := mkRandomBlock()
+	block3 := mkRandomBlock()
+
+	hdr := mock.MkBlock(nil, 0, 0)
+	hdr.Messages = block1.Cid()
+	hdr.ParentMessageReceipts = block2.Cid()
+	hdr.ParentStateRoot = block3.Cid()
+	block4, err := hdr.ToStorageBlock()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	allBlocks := []blocks.Block{block1, block2, block3, block4}
+	for _, blk := range allBlocks {
+		err := cold.Put(context.Background(), blk)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	path := t.TempDir()
+
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close() //nolint
+
+	ss.warmupEpoch = 1
+	go ss.reifyOrchestrator()
+
+	waitForReification := func() {
+		for {
+			ss.reifyMx.Lock()
+			ready := len(ss.reifyPend) == 0 && len(ss.reifyInProgress) == 0
+			ss.reifyMx.Unlock()
+
+			if ready {
+				return
+			}
+
+			time.Sleep(time.Millisecond)
+		}
+	}
+
+	// first access using the standard view
+	err = f(context.Background(), ss, block4.Cid())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// nothing should be reified
+	waitForReification()
+	for _, blk := range allBlocks {
+		has, err := hot.Has(context.Background(), blk.Cid())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("block unexpectedly reified")
+		}
+	}
+
+	// now make the hot/reifying view and ensure access reifies
+	err = f(blockstore.WithHotView(context.Background()), ss, block4.Cid())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// everything should be reified
+	waitForReification()
+	for i, blk := range allBlocks {
+		has, err := hot.Has(context.Background(), blk.Cid())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !has {
+			t.Fatalf("block%d was not reified", i+1)
+		}
+	}
+}
+
+func testSplitStoreReificationLimit(t *testing.T, f func(context.Context, blockstore.Blockstore, cid.Cid) error) {
+	ds := dssync.MutexWrap(datastore.NewMapDatastore())
+	hot := newMockStore()
+	cold := newMockStore()
+
+	mkRandomBlock := func() blocks.Block {
+		data := make([]byte, 128)
+		_, err := rand.Read(data)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return blocks.NewBlock(data)
+	}
+
+	block1 := mkRandomBlock()
+	block2 := mkRandomBlock()
+	block3 := mkRandomBlock()
+
+	hdr := mock.MkBlock(nil, 0, 0)
+	hdr.Messages = block1.Cid()
+	hdr.ParentMessageReceipts = block2.Cid()
+	hdr.ParentStateRoot = block3.Cid()
+	block4, err := hdr.ToStorageBlock()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	allBlocks := []blocks.Block{block1, block2, block3, block4}
+	for _, blk := range allBlocks {
+		err := cold.Put(context.Background(), blk)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	path := t.TempDir()
+
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close() //nolint
+
+	ss.warmupEpoch = 1
+	go ss.reifyOrchestrator()
+
+	waitForReification := func() {
+		for {
+			ss.reifyMx.Lock()
+			ready := len(ss.reifyPend) == 0 && len(ss.reifyInProgress) == 0
+			ss.reifyMx.Unlock()
+
+			if ready {
+				return
+			}
+
+			time.Sleep(time.Millisecond)
+		}
+	}
+
+	// do a hot access -- nothing should be reified as the limit should be exceeded
+	oldReifyLimit := ReifyLimit
+	ReifyLimit = 2
+	t.Cleanup(func() {
+		ReifyLimit = oldReifyLimit
+	})
+
+	err = f(blockstore.WithHotView(context.Background()), ss, block4.Cid())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	waitForReification()
+
+	for _, blk := range allBlocks {
+		has, err := hot.Has(context.Background(), blk.Cid())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("block unexpectedly reified")
+		}
+	}
+
+}
+
+func TestSplitStoreReification(t *testing.T) {
+	t.Log("test reification with Has")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.Has(ctx, c)
+		return err
+	})
+	t.Log("test reification with Get")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.Get(ctx, c)
+		return err
+	})
+	t.Log("test reification with GetSize")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.GetSize(ctx, c)
+		return err
+	})
+	t.Log("test reification with View")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		return s.View(ctx, c, func(_ []byte) error { return nil })
+	})
+	t.Log("test reification limit")
+	testSplitStoreReificationLimit(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.Has(ctx, c)
+		return err
+	})
+}
+
 type mockChain struct {
 	t testing.TB

@@ -426,17 +666,25 @@ func (c *mockChain) SubscribeHeadChanges(change func(revert []*types.TipSet, app

 type mockStore struct {
 	mx  sync.Mutex
-	set map[cid.Cid]blocks.Block
+	set map[string]blocks.Block
 }

 func newMockStore() *mockStore {
-	return &mockStore{set: make(map[cid.Cid]blocks.Block)}
+	return &mockStore{set: make(map[string]blocks.Block)}
+}
+
+func (b *mockStore) keyOf(c cid.Cid) string {
+	return string(c.Hash())
+}
+
+func (b *mockStore) cidOf(k string) cid.Cid {
+	return cid.NewCidV1(cid.Raw, mh.Multihash([]byte(k)))
 }

 func (b *mockStore) Has(_ context.Context, cid cid.Cid) (bool, error) {
 	b.mx.Lock()
 	defer b.mx.Unlock()
-	_, ok := b.set[cid]
+	_, ok := b.set[b.keyOf(cid)]
 	return ok, nil
 }

@@ -446,7 +694,7 @@ func (b *mockStore) Get(_ context.Context, cid cid.Cid) (blocks.Block, error) {
 	b.mx.Lock()
 	defer b.mx.Unlock()

-	blk, ok := b.set[cid]
+	blk, ok := b.set[b.keyOf(cid)]
 	if !ok {
 		return nil, blockstore.ErrNotFound
 	}
@@ -474,7 +722,7 @@ func (b *mockStore) Put(_ context.Context, blk blocks.Block) error {
 	b.mx.Lock()
 	defer b.mx.Unlock()

-	b.set[blk.Cid()] = blk
+	b.set[b.keyOf(blk.Cid())] = blk
 	return nil
 }

@@ -483,7 +731,7 @@ func (b *mockStore) PutMany(_ context.Context, blks []blocks.Block) error {
 	defer b.mx.Unlock()

 	for _, blk := range blks {
-		b.set[blk.Cid()] = blk
+		b.set[b.keyOf(blk.Cid())] = blk
 	}
 	return nil
 }
@@ -492,7 +740,7 @@ func (b *mockStore) DeleteBlock(_ context.Context, cid cid.Cid) error {
 	b.mx.Lock()
 	defer b.mx.Unlock()

-	delete(b.set, cid)
+	delete(b.set, b.keyOf(cid))
 	return nil
 }

@@ -501,7 +749,7 @@ func (b *mockStore) DeleteMany(_ context.Context, cids []cid.Cid) error {
 	defer b.mx.Unlock()

 	for _, c := range cids {
-		delete(b.set, c)
+		delete(b.set, b.keyOf(c))
 	}
 	return nil
 }
@@ -515,7 +763,7 @@ func (b *mockStore) ForEachKey(f func(cid.Cid) error) error {
 	defer b.mx.Unlock()

 	for c := range b.set {
-		err := f(c)
+		err := f(b.cidOf(c))
 		if err != nil {
 			return err
 		}
@@ -62,7 +62,7 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 	xcount := new(int64)
 	missing := new(int64)

-	visitor, err := s.markSetEnv.Create("warmup", 0)
+	visitor, err := s.markSetEnv.New("warmup", 0)
 	if err != nil {
 		return xerrors.Errorf("error creating visitor: %w", err)
 	}
@@ -26,6 +26,10 @@ type tmpVisitor struct {
 var _ ObjectVisitor = (*tmpVisitor)(nil)

 func (v *tmpVisitor) Visit(c cid.Cid) (bool, error) {
+	if isUnitaryObject(c) {
+		return false, nil
+	}
+
 	return v.set.Visit(c), nil
 }

@@ -45,6 +49,10 @@ func newConcurrentVisitor() *concurrentVisitor {
 }

 func (v *concurrentVisitor) Visit(c cid.Cid) (bool, error) {
+	if isUnitaryObject(c) {
+		return false, nil
+	}
+
 	v.mx.Lock()
 	defer v.mx.Unlock()

@@ -1,3 +1,4 @@
+//stm: #unit
 package blockstore

 import (
@@ -13,6 +14,9 @@ import (
 )

 func TestTimedCacheBlockstoreSimple(t *testing.T) {
+	//stm: @SPLITSTORE_TIMED_BLOCKSTORE_START_001
+	//stm: @SPLITSTORE_TIMED_BLOCKSTORE_PUT_001, @SPLITSTORE_TIMED_BLOCKSTORE_HAS_001, @SPLITSTORE_TIMED_BLOCKSTORE_GET_001
+	//stm: @SPLITSTORE_TIMED_BLOCKSTORE_ALL_KEYS_CHAN_001
 	tc := NewTimedCacheBlockstore(10 * time.Millisecond)
 	mClock := clock.NewMock()
 	mClock.Set(time.Now())
@@ -1,3 +1,4 @@
+//stm: #unit
 package blockstore

 import (
@@ -15,6 +16,7 @@ var (
 )

 func TestUnionBlockstore_Get(t *testing.T) {
+	//stm: @SPLITSTORE_UNION_BLOCKSTORE_GET_001
 	ctx := context.Background()
 	m1 := NewMemory()
 	m2 := NewMemory()
@@ -34,6 +36,9 @@ func TestUnionBlockstore_Get(t *testing.T) {
 }

 func TestUnionBlockstore_Put_PutMany_Delete_AllKeysChan(t *testing.T) {
+	//stm: @SPLITSTORE_UNION_BLOCKSTORE_PUT_001, @SPLITSTORE_UNION_BLOCKSTORE_HAS_001
+	//stm: @SPLITSTORE_UNION_BLOCKSTORE_PUT_MANY_001, @SPLITSTORE_UNION_BLOCKSTORE_DELETE_001
+	//stm: @SPLITSTORE_UNION_BLOCKSTORE_ALL_KEYS_CHAN_001
 	ctx := context.Background()
 	m1 := NewMemory()
 	m2 := NewMemory()
@@ -1,2 +1,2 @@
-/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWBdRCBLUeKvoy22u5DcXs61adFn31v8WWCZgmBjDCjbsC
-/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWDUQJBA18njjXnG9RtLxoN3muvdU7PEy55QorUEsdAqdy
+/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWFHDtFx7CVTy4xoCDutVo1cScvSnQjDeaM8UzwVS1qwkh
+/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWKt8cwpkiumkT8x32c3YFxsPRwhV5J8hCYPn9mhUmcAXt
@@ -1,3 +1,4 @@
+//stm: #unit
 package build

 import (
@@ -7,6 +8,7 @@ import (
 )

 func TestOpenRPCDiscoverJSON_Version(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_OPENRPC_VERSION_001
 	// openRPCDocVersion is the current OpenRPC version of the API docs.
 	openRPCDocVersion := "1.2.6"

@@ -42,8 +42,7 @@ const UpgradeTurboHeight = -15
 const UpgradeHyperdriveHeight = -16
 const UpgradeChocolateHeight = -17

-// 2022-01-17T19:00:00Z
-const UpgradeOhSnapHeight = 30262
+const UpgradeOhSnapHeight = 240

 func init() {
 	policy.SetConsensusMinerMinPower(abi.NewStoragePower(2 << 30))
@@ -54,7 +54,8 @@ const UpgradeHyperdriveHeight = 420

 const UpgradeChocolateHeight = 312746

-const UpgradeOhSnapHeight = 99999999
+// 2022-02-10T19:23:00Z
+const UpgradeOhSnapHeight = 682006

 func init() {
 	policy.SetConsensusMinerMinPower(abi.NewStoragePower(32 << 30))
@@ -67,7 +67,8 @@ const UpgradeHyperdriveHeight = 892800
 // 2021-10-26T13:30:00Z
 const UpgradeChocolateHeight = 1231620

-var UpgradeOhSnapHeight = abi.ChainEpoch(999999999999)
+// 2022-03-01T15:00:00Z
+var UpgradeOhSnapHeight = abi.ChainEpoch(1594680)

 func init() {
 	if os.Getenv("LOTUS_USE_TEST_ADDRESSES") != "1" {
@@ -1,4 +1,54 @@
 {
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-61fa69f38b9cc771ba27b670124714b4ea77fbeae05e377fb859c4a43b73a30c.params": {
+    "cid": "Qma5WL6abSqYg9uUQAZ3EHS286bsNsha7oAGsJBD48Bq2q",
+    "digest": "c3ad7bb549470b82ad52ed070aebb4f4",
+    "sector_size": 536870912
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-61fa69f38b9cc771ba27b670124714b4ea77fbeae05e377fb859c4a43b73a30c.vk": {
+    "cid": "QmUa7f9JtJMsqJJ3s3ZXk6WyF4xJLE8FiqYskZGgk8GCDv",
+    "digest": "994c5b7d450ca9da348c910689f2dc7f",
+    "sector_size": 536870912
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-92180959e1918d26350b8e6cfe217bbdd0a2d8de51ebec269078b364b715ad63.params": {
+    "cid": "QmQiT4qBGodrVNEgVTDXxBNDdPbaD8Ag7Sx3ZTq1zHX79S",
+    "digest": "5aedd2cf3e5c0a15623d56a1b43110ad",
+    "sector_size": 8388608
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-92180959e1918d26350b8e6cfe217bbdd0a2d8de51ebec269078b364b715ad63.vk": {
+    "cid": "QmdcpKUQvHM8RFRVKbk1yHfEqMcBzhtFWKRp9SNEmWq37i",
+    "digest": "abd80269054d391a734febdac0d2e687",
+    "sector_size": 8388608
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-fb9e095bebdd77511c0269b967b4d87ba8b8a525edaa0e165de23ba454510194.params": {
+    "cid": "QmYM6Hg7mjmvA3ZHTsqkss1fkdyDju5dDmLiBZGJ5pz9y9",
+    "digest": "311f92a3e75036ced01b1c0025f1fa0c",
+    "sector_size": 2048
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-fb9e095bebdd77511c0269b967b4d87ba8b8a525edaa0e165de23ba454510194.vk": {
+    "cid": "QmaQsTLL3nc5dw6wAvaioJSBfd1jhQrA2o6ucFf7XeV74P",
+    "digest": "eadad9784969890d30f2749708c79771",
+    "sector_size": 2048
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-8-0-3b7f44a9362e3985369454947bc94022e118211e49fd672d52bec1cbfd599d18.params": {
+    "cid": "QmNPc75iEfcahCwNKdqnWLtxnjspUGGR4iscjiz3wP3RtS",
+    "digest": "1b3cfd761a961543f9eb273e435a06a2",
+    "sector_size": 34359738368
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-8-0-3b7f44a9362e3985369454947bc94022e118211e49fd672d52bec1cbfd599d18.vk": {
+    "cid": "QmdFFUe1gcz9MMHc6YW8aoV48w4ckvcERjt7PkydQAMfCN",
+    "digest": "3a6941983754737fde880d29c7094905",
+    "sector_size": 34359738368
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-8-2-102e1444a7e9a97ebf1e3d6855dcc77e66c011ea66f936d9b2c508f87f2f83a7.params": {
+    "cid": "QmUB6xTVjzBQGuDNeyJMrrJ1byk58vhPm8eY2Lv9pgwanp",
+    "digest": "1a392e7b759fb18e036c7559b5ece816",
+    "sector_size": 68719476736
+  },
+  "v28-empty-sector-update-merkletree-poseidon_hasher-8-8-2-102e1444a7e9a97ebf1e3d6855dcc77e66c011ea66f936d9b2c508f87f2f83a7.vk": {
+    "cid": "Qmd794Jty7k26XJ8Eg4NDEks65Qk8G4GVfGkwqvymv8HAg",
+    "digest": "80e366df2f1011953c2d01c7b7c9ee8e",
+    "sector_size": 68719476736
+  },
  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.params": {
    "cid": "QmVxjFRyhmyQaZEtCh7nk2abc7LhFkzhnRX4rcHqCCpikR",
    "digest": "7610b9f82bfc88405b7a832b651ce2f6",
@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }

 // BuildVersion is the local build version
-const BuildVersion = "1.15.0-dev"
+const BuildVersion = "1.15.2-rc1"

 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {
@@ -1,3 +1,4 @@
+//stm: #unit
 package adt

 import (
@@ -44,6 +45,7 @@ func TestDiffAdtArray(t *testing.T) {

 	changes := new(TestDiffArray)

+	//stm: @CHAIN_ADT_ARRAY_DIFF_001
 	assert.NoError(t, DiffAdtArray(arrA, arrB, changes))
 	assert.NotNil(t, changes)

@@ -98,6 +100,7 @@ func TestDiffAdtMap(t *testing.T) {

 	changes := new(TestDiffMap)

+	//stm: @CHAIN_ADT_MAP_DIFF_001
 	assert.NoError(t, DiffAdtMap(mapA, mapB, changes))
 	assert.NotNil(t, changes)

@@ -1,3 +1,4 @@
+//stm: #unit
 package aerrors_test

 import (
@@ -11,6 +12,7 @@ import (
 )

 func TestFatalError(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_ACTOR_ERRORS_001
 	e1 := xerrors.New("out of disk space")
 	e2 := xerrors.Errorf("could not put node: %w", e1)
 	e3 := xerrors.Errorf("could not save head: %w", e2)
@@ -24,6 +26,7 @@ func TestFatalError(t *testing.T) {
 	assert.True(t, IsFatal(aw4), "should be fatal")
 }
 func TestAbsorbeError(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_ACTOR_ERRORS_001
 	e1 := xerrors.New("EOF")
 	e2 := xerrors.Errorf("could not decode: %w", e1)
 	ae := Absorb(e2, 35, "failed to decode CBOR")
@@ -16,6 +16,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/types"
+	verifreg7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg"
 )

 func init() {
@@ -62,6 +63,11 @@ func GetActorCodeID(av actors.Version) (cid.Cid, error) {
 	return cid.Undef, xerrors.Errorf("unknown actor version %d", av)
 }

+type RemoveDataCapProposal = verifreg{{.latestVersion}}.RemoveDataCapProposal
+type RemoveDataCapRequest = verifreg{{.latestVersion}}.RemoveDataCapRequest
+type RemoveDataCapParams = verifreg{{.latestVersion}}.RemoveDataCapParams
+type RmDcProposalID = verifreg{{.latestVersion}}.RmDcProposalID
+const SignatureDomainSeparation_RemoveDataCap = verifreg{{.latestVersion}}.SignatureDomainSeparation_RemoveDataCap

 type State interface {
 	cbor.Marshaler
@@ -69,6 +75,7 @@ type State interface {
 	RootKey() (address.Address, error)
 	VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error)
 	VerifierDataCap(address.Address) (bool, abi.StoragePower, error)
+	RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error)
 	ForEachVerifier(func(addr address.Address, dcap abi.StoragePower) error) error
 	ForEachClient(func(addr address.Address, dcap abi.StoragePower) error) error
 	GetState() interface{}
@@ -61,6 +61,10 @@ func (s *state{{.v}}) VerifierDataCap(addr address.Address) (bool, abi.StoragePo
 	return getDataCap(s.store, actors.Version{{.v}}, s.verifiers, addr)
 }

+func (s *state{{.v}}) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version{{.v}}, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state{{.v}}) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version{{.v}}, s.verifiers, cb)
 }
@@ -77,6 +81,11 @@ func (s *state{{.v}}) verifiers() (adt.Map, error) {
 	return adt{{.v}}.AsMap(s.store, s.Verifiers{{if (ge .v 3)}}, builtin{{.v}}.DefaultHamtBitwidth{{end}})
 }

+func (s *state{{.v}}) removeDataCapProposalIDs() (adt.Map, error) {
+    {{if le .v 6}}return nil, nil
+    {{else}}return adt{{.v}}.AsMap(s.store, s.RemoveDataCapProposalIDs, builtin{{.v}}.DefaultHamtBitwidth){{end}}
+}
+
 func (s *state{{.v}}) GetState() interface{} {
 	return &s.State
 }
@@ -6,6 +6,7 @@ import (
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
+	"github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg"
 	"golang.org/x/xerrors"
 )

@@ -50,3 +51,28 @@ func forEachCap(store adt.Store, ver actors.Version, root rootFunc, cb func(addr
 		return cb(a, dcap)
 	})
 }
+
+func getRemoveDataCapProposalID(store adt.Store, ver actors.Version, root rootFunc, verifier address.Address, client address.Address) (bool, uint64, error) {
+	if verifier.Protocol() != address.ID {
+		return false, 0, xerrors.Errorf("can only look up ID addresses")
+	}
+	if client.Protocol() != address.ID {
+		return false, 0, xerrors.Errorf("can only look up ID addresses")
+	}
+	vh, err := root()
+	if err != nil {
+		return false, 0, xerrors.Errorf("loading verifreg: %w", err)
+	}
+	if vh == nil {
+		return false, 0, xerrors.Errorf("remove data cap proposal hamt not found. you are probably using an incompatible version of actors")
+	}
+
+	var id verifreg.RmDcProposalID
+	if found, err := vh.Get(abi.NewAddrPairKey(verifier, client), &id); err != nil {
+		return false, 0, xerrors.Errorf("looking up addr pair: %w", err)
+	} else if !found {
+		return false, 0, nil
+	}
+
+	return true, id.ProposalID, nil
+}
@@ -53,6 +53,10 @@ func (s *state0) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version0, s.verifiers, addr)
 }

+func (s *state0) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version0, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state0) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version0, s.verifiers, cb)
 }
@@ -69,6 +73,11 @@ func (s *state0) verifiers() (adt.Map, error) {
 	return adt0.AsMap(s.store, s.Verifiers)
 }

+func (s *state0) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state0) GetState() interface{} {
 	return &s.State
 }
@@ -53,6 +53,10 @@ func (s *state2) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version2, s.verifiers, addr)
 }

+func (s *state2) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version2, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state2) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version2, s.verifiers, cb)
 }
@@ -69,6 +73,11 @@ func (s *state2) verifiers() (adt.Map, error) {
 	return adt2.AsMap(s.store, s.Verifiers)
 }

+func (s *state2) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state2) GetState() interface{} {
 	return &s.State
 }
@@ -54,6 +54,10 @@ func (s *state3) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version3, s.verifiers, addr)
 }

+func (s *state3) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version3, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state3) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version3, s.verifiers, cb)
 }
@@ -70,6 +74,11 @@ func (s *state3) verifiers() (adt.Map, error) {
 	return adt3.AsMap(s.store, s.Verifiers, builtin3.DefaultHamtBitwidth)
 }

+func (s *state3) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state3) GetState() interface{} {
 	return &s.State
 }
@@ -54,6 +54,10 @@ func (s *state4) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version4, s.verifiers, addr)
 }

+func (s *state4) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version4, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state4) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version4, s.verifiers, cb)
 }
@@ -70,6 +74,11 @@ func (s *state4) verifiers() (adt.Map, error) {
 	return adt4.AsMap(s.store, s.Verifiers, builtin4.DefaultHamtBitwidth)
 }

+func (s *state4) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state4) GetState() interface{} {
 	return &s.State
 }
@@ -54,6 +54,10 @@ func (s *state5) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version5, s.verifiers, addr)
 }

+func (s *state5) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version5, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state5) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version5, s.verifiers, cb)
 }
@@ -70,6 +74,11 @@ func (s *state5) verifiers() (adt.Map, error) {
 	return adt5.AsMap(s.store, s.Verifiers, builtin5.DefaultHamtBitwidth)
 }

+func (s *state5) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state5) GetState() interface{} {
 	return &s.State
 }
@@ -54,6 +54,10 @@ func (s *state6) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version6, s.verifiers, addr)
 }

+func (s *state6) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version6, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state6) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version6, s.verifiers, cb)
 }
@@ -70,6 +74,11 @@ func (s *state6) verifiers() (adt.Map, error) {
 	return adt6.AsMap(s.store, s.Verifiers, builtin6.DefaultHamtBitwidth)
 }

+func (s *state6) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state6) GetState() interface{} {
 	return &s.State
 }
@@ -54,6 +54,10 @@ func (s *state7) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version7, s.verifiers, addr)
 }

+func (s *state7) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version7, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state7) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version7, s.verifiers, cb)
 }
@@ -70,6 +74,10 @@ func (s *state7) verifiers() (adt.Map, error) {
 	return adt7.AsMap(s.store, s.Verifiers, builtin7.DefaultHamtBitwidth)
 }

+func (s *state7) removeDataCapProposalIDs() (adt.Map, error) {
+	return adt7.AsMap(s.store, s.RemoveDataCapProposalIDs, builtin7.DefaultHamtBitwidth)
+}
+
 func (s *state7) GetState() interface{} {
 	return &s.State
 }
@@ -27,6 +27,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/types"
+	verifreg7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg"
 )

 func init() {
@@ -151,12 +152,20 @@ func GetActorCodeID(av actors.Version) (cid.Cid, error) {
 	return cid.Undef, xerrors.Errorf("unknown actor version %d", av)
 }

+type RemoveDataCapProposal = verifreg7.RemoveDataCapProposal
+type RemoveDataCapRequest = verifreg7.RemoveDataCapRequest
+type RemoveDataCapParams = verifreg7.RemoveDataCapParams
+type RmDcProposalID = verifreg7.RmDcProposalID
+
+const SignatureDomainSeparation_RemoveDataCap = verifreg7.SignatureDomainSeparation_RemoveDataCap
+
 type State interface {
 	cbor.Marshaler

 	RootKey() (address.Address, error)
 	VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error)
 	VerifierDataCap(address.Address) (bool, abi.StoragePower, error)
+	RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error)
 	ForEachVerifier(func(addr address.Address, dcap abi.StoragePower) error) error
 	ForEachClient(func(addr address.Address, dcap abi.StoragePower) error) error
 	GetState() interface{}
@@ -1,3 +1,4 @@
+//stm: #unit
 package policy

 import (
@@ -22,6 +23,7 @@ func TestSupportedProofTypes(t *testing.T) {
 	for t := range miner0.SupportedProofTypes {
 		oldTypes = append(oldTypes, t)
 	}
+	//stm: @BLOCKCHAIN_POLICY_SET_MAX_SUPPORTED_PROOF_TYPES_001
 	t.Cleanup(func() {
 		SetSupportedProofTypes(oldTypes...)
 	})
@@ -33,6 +35,7 @@ func TestSupportedProofTypes(t *testing.T) {
 			abi.RegisteredSealProof_StackedDrg2KiBV1: {},
 		},
 	)
+	//stm: @BLOCKCHAIN_POLICY_ADD_MAX_SUPPORTED_PROOF_TYPES_001
 	AddSupportedProofTypes(abi.RegisteredSealProof_StackedDrg8MiBV1)
 	require.EqualValues(t,
 		miner0.SupportedProofTypes,
@@ -45,6 +48,7 @@ func TestSupportedProofTypes(t *testing.T) {

 // Tests assumptions about policies being the same between actor versions.
 func TestAssumptions(t *testing.T) {
+	//stm: @BLOCKCHAIN_POLICY_ASSUMPTIONS_001
 	require.EqualValues(t, miner0.SupportedProofTypes, miner2.PreCommitSealProofTypesV0)
 	require.Equal(t, miner0.PreCommitChallengeDelay, miner2.PreCommitChallengeDelay)
 	require.Equal(t, miner0.MaxSectorExpirationExtension, miner2.MaxSectorExpirationExtension)
@@ -58,6 +62,7 @@ func TestAssumptions(t *testing.T) {
 }

 func TestPartitionSizes(t *testing.T) {
+	//stm: @CHAIN_ACTOR_PARTITION_SIZES_001
 	for _, p := range abi.SealProofInfos {
 		sizeNew, err := builtin2.PoStProofWindowPoStPartitionSectors(p.WindowPoStProof)
 		require.NoError(t, err)
@@ -71,6 +76,7 @@ func TestPartitionSizes(t *testing.T) {
 }

 func TestPoStSize(t *testing.T) {
+	//stm: @BLOCKCHAIN_POLICY_GET_MAX_POST_PARTITIONS_001
 	v12PoStSize, err := GetMaxPoStPartitions(network.Version12, abi.RegisteredPoStProof_StackedDrgWindow64GiBV1)
 	require.Equal(t, 4, v12PoStSize)
 	require.NoError(t, err)
@@ -142,7 +142,7 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re

 	go func() {
 		start := build.Clock.Now()
-		log.Infow("start fetching randomness", "round", round)
+		log.Debugw("start fetching randomness", "round", round)
 		resp, err := db.client.Get(ctx, round)

 		var br beacon.Response
@@ -152,7 +152,7 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re
 			br.Entry.Round = resp.Round()
 			br.Entry.Data = resp.Signature()
 		}
-		log.Infow("done fetching randomness", "round", round, "took", build.Clock.Since(start))
+		log.Debugw("done fetching randomness", "round", round, "took", build.Clock.Since(start))
 		out <- br
 		close(out)
 	}()
@@ -1,3 +1,5 @@
+//stm: ignore
+//Only tests external library behavior, therefore it should not be annotated
 package drand

 import (
@@ -2,6 +2,7 @@ package filcns

 import (
 	"context"
+	"os"
 	"sync/atomic"

 	"github.com/filecoin-project/lotus/chain/rand"
@@ -32,6 +33,7 @@ import (

 	/* inline-gen end */

+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -92,7 +94,8 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, sm *stmgr.StateManager
 		partDone()
 	}()

-	makeVmWithBaseStateAndEpoch := func(base cid.Cid, e abi.ChainEpoch) (*vm.VM, error) {
+	ctx = blockstore.WithHotView(ctx)
+	makeVmWithBaseStateAndEpoch := func(base cid.Cid, e abi.ChainEpoch) (vm.Interface, error) {
 		vmopt := &vm.VMOpts{
 			StateBase:      base,
 			Epoch:          e,
@@ -106,10 +109,23 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, sm *stmgr.StateManager
 			LookbackState:  stmgr.LookbackStateGetterForTipset(sm, ts),
 		}

+		if os.Getenv("LOTUS_USE_FVM_EXPERIMENTAL") == "1" {
+			// This is needed so that the FVM does not have to duplicate the genesis vesting schedule, one
+			// of the components of the circ supply calc.
+			// This field is NOT needed by the LegacyVM, and also NOT needed by the FVM from v15 onwards.
+			filVested, err := sm.GetFilVested(ctx, e)
+			if err != nil {
+				return nil, err
+			}
+
+			vmopt.FilVested = filVested
+			return vm.NewFVM(ctx, vmopt)
+		}
+
 		return sm.VMConstructor()(ctx, vmopt)
 	}

-	runCron := func(vmCron *vm.VM, epoch abi.ChainEpoch) error {
+	runCron := func(vmCron vm.Interface, epoch abi.ChainEpoch) error {
 		cronMsg := &types.Message{
 			To:         cron.Address,
 			From:       builtin.SystemActorAddr,
@@ -467,7 +467,7 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
 	}

 	nv := filec.sm.GetNetworkVersion(ctx, b.Header.Height)
-	pl := vm.PricelistByEpoch(baseTs.Height())
+	pl := vm.PricelistByEpochAndNetworkVersion(b.Header.Height, nv)
 	var sumGasLimit int64
 	checkMsg := func(msg types.ChainMsg) error {
 		m := msg.VMMessage()
@@ -165,13 +165,8 @@ func DefaultUpgradeSchedule() stmgr.UpgradeSchedule {
 		Migration: UpgradeActorsV7,
 		PreMigrations: []stmgr.PreMigration{{
 			PreMigration:    PreUpgradeActorsV7,
-			StartWithin:     120,
+			StartWithin:     180,
 			DontStartWithin: 60,
-			StopWithin:      35,
-		}, {
-			PreMigration:    PreUpgradeActorsV7,
-			StartWithin:     30,
-			DontStartWithin: 15,
 			StopWithin:      5,
 		}},
 		Expensive: true,
@@ -1264,7 +1259,7 @@ func upgradeActorsV7Common(
 	root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet,
 	config nv15.Config,
 ) (cid.Cid, error) {
-	writeStore := blockstore.NewAutobatch(ctx, sm.ChainStore().StateBlockstore(), units.GiB)
+	writeStore := blockstore.NewAutobatch(ctx, sm.ChainStore().StateBlockstore(), units.GiB/4)
 	// TODO: pretty sure we'd achieve nothing by doing this, confirm in review
 	//buf := blockstore.NewTieredBstore(sm.ChainStore().StateBlockstore(), writeStore)
 	store := store.ActorStore(ctx, writeStore)
@@ -1,3 +1,4 @@
+//stm: #unit
 package events

 import (
@@ -358,6 +359,7 @@ func (fcs *fakeCS) advance(rev, app, drop int, msgs map[int]cid.Cid, nulls ...in
 var _ EventAPI = &fakeCS{}

 func TestAt(t *testing.T) {
+	//stm: @EVENTS_HEIGHT_CHAIN_AT_001, @EVENTS_HEIGHT_REVERT_001
 	fcs := newFakeCS(t)
 	events, err := NewEvents(context.Background(), fcs)
 	require.NoError(t, err)
@@ -418,6 +420,7 @@ func TestAt(t *testing.T) {
 }

 func TestAtNullTrigger(t *testing.T) {
+	//stm: @EVENTS_HEIGHT_CHAIN_AT_001
 	fcs := newFakeCS(t)
 	events, err := NewEvents(context.Background(), fcs)
 	require.NoError(t, err)
@@ -447,6 +450,7 @@ func TestAtNullTrigger(t *testing.T) {
 }

 func TestAtNullConf(t *testing.T) {
+	//stm: @EVENTS_HEIGHT_CHAIN_AT_001, @EVENTS_HEIGHT_REVERT_001
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

@@ -485,6 +489,7 @@ func TestAtNullConf(t *testing.T) {
 }

 func TestAtStart(t *testing.T) {
+	//stm: @EVENTS_HEIGHT_CHAIN_AT_001
 	fcs := newFakeCS(t)

 	events, err := NewEvents(context.Background(), fcs)
@@ -515,6 +520,7 @@ func TestAtStart(t *testing.T) {
 }

 func TestAtStartConfidence(t *testing.T) {
+	//stm: @EVENTS_HEIGHT_CHAIN_AT_001
 	fcs := newFakeCS(t)

 	events, err := NewEvents(context.Background(), fcs)
@@ -541,6 +547,7 @@ func TestAtStartConfidence(t *testing.T) {
 }

 func TestAtChained(t *testing.T) {
+	//stm: @EVENTS_HEIGHT_CHAIN_AT_001
 	fcs := newFakeCS(t)

 	events, err := NewEvents(context.Background(), fcs)
@@ -571,6 +578,7 @@ func TestAtChained(t *testing.T) {
 }

 func TestAtChainedConfidence(t *testing.T) {
+	//stm: @EVENTS_HEIGHT_CHAIN_AT_001
 	fcs := newFakeCS(t)

 	events, err := NewEvents(context.Background(), fcs)
@@ -601,6 +609,7 @@ func TestAtChainedConfidence(t *testing.T) {
 }

 func TestAtChainedConfidenceNull(t *testing.T) {
+	//stm: @EVENTS_HEIGHT_CHAIN_AT_001
 	fcs := newFakeCS(t)

 	events, err := NewEvents(context.Background(), fcs)
@@ -632,6 +641,7 @@ func matchAddrMethod(to address.Address, m abi.MethodNum) func(msg *types.Messag
 }

 func TestCalled(t *testing.T) {
+	//stm: @EVENTS_EVENTS_CALLED_001, @EVENTS_HEIGHT_REVERT_001
 	fcs := newFakeCS(t)

 	events, err := NewEvents(context.Background(), fcs)
@@ -837,6 +847,7 @@ func TestCalled(t *testing.T) {
 }

 func TestCalledTimeout(t *testing.T) {
+	//stm: @EVENTS_EVENTS_CALLED_001, @EVENTS_HEIGHT_REVERT_001
 	fcs := newFakeCS(t)

 	events, err := NewEvents(context.Background(), fcs)
@@ -897,6 +908,7 @@ func TestCalledTimeout(t *testing.T) {
 }

 func TestCalledOrder(t *testing.T) {
+	//stm: @EVENTS_EVENTS_CALLED_001, @EVENTS_HEIGHT_REVERT_001
 	fcs := newFakeCS(t)

 	events, err := NewEvents(context.Background(), fcs)
@@ -953,6 +965,7 @@ func TestCalledOrder(t *testing.T) {
 }

 func TestCalledNull(t *testing.T) {
+	//stm: @EVENTS_EVENTS_CALLED_001, @EVENTS_HEIGHT_REVERT_001
 	fcs := newFakeCS(t)

 	events, err := NewEvents(context.Background(), fcs)
@@ -1011,6 +1024,7 @@ func TestCalledNull(t *testing.T) {
 }

 func TestRemoveTriggersOnMessage(t *testing.T) {
+	//stm: @EVENTS_EVENTS_CALLED_001, @EVENTS_HEIGHT_REVERT_001
 	fcs := newFakeCS(t)

 	events, err := NewEvents(context.Background(), fcs)
@@ -1094,6 +1108,7 @@ type testStateChange struct {
 }

 func TestStateChanged(t *testing.T) {
+	//stm: @EVENTS_EVENTS_CALLED_001, @EVENTS_HEIGHT_REVERT_001
 	fcs := newFakeCS(t)

 	events, err := NewEvents(context.Background(), fcs)
@@ -1179,6 +1194,7 @@ func TestStateChanged(t *testing.T) {
 }

 func TestStateChangedRevert(t *testing.T) {
+	//stm: @EVENTS_EVENTS_CALLED_001, @EVENTS_HEIGHT_REVERT_001
 	fcs := newFakeCS(t)

 	events, err := NewEvents(context.Background(), fcs)
@@ -1255,6 +1271,7 @@ func TestStateChangedRevert(t *testing.T) {
 }

 func TestStateChangedTimeout(t *testing.T) {
+	//stm: @EVENTS_EVENTS_CALLED_001, @EVENTS_HEIGHT_REVERT_001
 	timeoutHeight := abi.ChainEpoch(20)
 	confidence := 3

@@ -1332,6 +1349,7 @@ func TestStateChangedTimeout(t *testing.T) {
 }

 func TestCalledMultiplePerEpoch(t *testing.T) {
+	//stm: @EVENTS_EVENTS_CALLED_001, @EVENTS_HEIGHT_REVERT_001
 	fcs := newFakeCS(t)

 	events, err := NewEvents(context.Background(), fcs)
@@ -1384,6 +1402,7 @@ func TestCalledMultiplePerEpoch(t *testing.T) {
 }

 func TestCachedSameBlock(t *testing.T) {
+	//stm: @EVENTS_EVENTS_CALLED_001, @EVENTS_HEIGHT_REVERT_001
 	fcs := newFakeCS(t)

 	_, err := NewEvents(context.Background(), fcs)
@@ -1418,6 +1437,7 @@ func (t *testObserver) Revert(_ context.Context, from, to *types.TipSet) error {
 }

 func TestReconnect(t *testing.T) {
+	//stm: @EVENTS_EVENTS_CALLED_001, @EVENTS_HEIGHT_REVERT_001
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

@@ -1449,3 +1469,35 @@ func TestReconnect(t *testing.T) {
 	fcs.advance(0, 5, 2, nil, 0, 1, 3)
 	require.True(t, fcs.callNumber["ChainGetPath"] == 4)
 }
+
+func TestUnregister(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	fcs := newFakeCS(t)
+
+	events, err := NewEvents(ctx, fcs)
+	require.NoError(t, err)
+
+	tsObs := &testObserver{t: t}
+	events.Observe(tsObs)
+
+	// observer receives heads as the chain advances
+	fcs.advance(0, 1, 0, nil)
+	headBeforeDeregister := events.lastTs
+	require.Equal(t, tsObs.head, headBeforeDeregister)
+
+	// observer unregistered successfully
+	found := events.Unregister(tsObs)
+	require.True(t, found)
+
+	// observer stops receiving heads as the chain advances
+	fcs.advance(0, 1, 0, nil)
+	require.Equal(t, tsObs.head, headBeforeDeregister)
+	require.NotEqual(t, tsObs.head, events.lastTs)
+
+	// unregistering an invalid observer returns false
+	dneObs := &testObserver{t: t}
+	found = events.Unregister(dneObs)
+	require.False(t, found)
+}
@@ -253,3 +253,27 @@ func (o *observer) Observe(obs TipSetObserver) *types.TipSet {
 	o.observers = append(o.observers, obs)
 	return o.head
 }
+
+// Unregister unregisters an observer. Returns true if we successfully removed the observer.
+//
+// NOTE: The observer _may_ be called after being removed. Observers MUST handle this case
+// internally.
+func (o *observer) Unregister(obs TipSetObserver) (found bool) {
+	o.lk.Lock()
+	defer o.lk.Unlock()
+	// We _copy_ the observers list because we may be concurrently reading it from a headChange
+	// handler.
+	//
+	// This should happen infrequently, so it's fine if we spend a bit of time here.
+	newObservers := make([]TipSetObserver, 0, len(o.observers))
+	for _, existingObs := range o.observers {
+		if existingObs == obs {
+			found = true
+			continue
+		}
+		newObservers = append(newObservers, existingObs)
+	}
+
+	o.observers = newObservers
+	return found
+}
@@ -1,3 +1,4 @@
+//stm: #unit
 package state

 import (
@@ -35,6 +36,12 @@ func init() {
 }

 func TestMarketPredicates(t *testing.T) {
+	//stm: @EVENTS_PREDICATES_ON_ACTOR_STATE_CHANGED_001, @EVENTS_PREDICATES_DEAL_STATE_CHANGED_001
+	//stm: @EVENTS_PREDICATES_DEAL_CHANGED_FOR_IDS
+
+	//stm: @EVENTS_PREDICATES_ON_BALANCE_CHANGED_001, @EVENTS_PREDICATES_BALANCE_CHANGED_FOR_ADDRESS_001
+	//stm: @EVENTS_PREDICATES_ON_DEAL_PROPOSAL_CHANGED_001, @EVENTS_PREDICATES_PROPOSAL_AMT_CHANGED_001
+	//stm: @EVENTS_PREDICATES_DEAL_STATE_CHANGED_001, @EVENTS_PREDICATES_DEAL_AMT_CHANGED_001
 	ctx := context.Background()
 	bs := bstore.NewMemorySync()
 	store := adt2.WrapStore(ctx, cbornode.NewCborStore(bs))
@@ -333,6 +340,8 @@ func TestMarketPredicates(t *testing.T) {
 }

 func TestMinerSectorChange(t *testing.T) {
+	//stm: @EVENTS_PREDICATES_ON_ACTOR_STATE_CHANGED_001, @EVENTS_PREDICATES_MINER_ACTOR_CHANGE_001
+	//stm: @EVENTS_PREDICATES_MINER_SECTOR_CHANGE_001
 	ctx := context.Background()
 	bs := bstore.NewMemorySync()
 	store := adt2.WrapStore(ctx, cbornode.NewCborStore(bs))
@@ -1,3 +1,4 @@
+//stm: #unit
 package events

 import (
@@ -92,6 +93,7 @@ func (h *cacheHarness) skip(n abi.ChainEpoch) {
 }

 func TestTsCache(t *testing.T) {
+	//stm: @EVENTS_CACHE_GET_CHAIN_HEAD_001, @EVENTS_CACHE_GET_001, @EVENTS_CACHE_ADD_001
 	h := newCacheharness(t)

 	for i := 0; i < 9000; i++ {
@@ -104,6 +106,8 @@ func TestTsCache(t *testing.T) {
 }

 func TestTsCacheNulls(t *testing.T) {
+	//stm: @EVENTS_CACHE_GET_CHAIN_HEAD_001, @EVENTS_CACHE_GET_CHAIN_TIPSET_BEFORE_001, @EVENTS_CACHE_GET_CHAIN_TIPSET_AFTER_001
+	//stm: @EVENTS_CACHE_GET_001, @EVENTS_CACHE_ADD_001
 	ctx := context.Background()
 	h := newCacheharness(t)

@@ -182,6 +186,7 @@ func (tc *tsCacheAPIStorageCallCounter) ChainGetTipSet(ctx context.Context, tsk
 }

 func TestTsCacheEmpty(t *testing.T) {
+	//stm: @EVENTS_CACHE_GET_CHAIN_HEAD_001
 	// Calling best on an empty cache should just call out to the chain API
 	callCounter := &tsCacheAPIStorageCallCounter{t: t}
 	tsc := newTSCache(callCounter, 50)
@@ -191,6 +196,7 @@ func TestTsCacheEmpty(t *testing.T) {
 }

 func TestTsCacheSkip(t *testing.T) {
+	//stm: @EVENTS_CACHE_GET_CHAIN_HEAD_001, @EVENTS_CACHE_GET_001, @EVENTS_CACHE_ADD_001
 	h := newCacheharness(t)

 	ts, err := types.NewTipSet([]*types.BlockHeader{{
@@ -1,3 +1,4 @@
+//stm: #unit
 package gen

 import (
@@ -34,6 +35,7 @@ func testGeneration(t testing.TB, n int, msgs int, sectors int) {
 }

 func TestChainGeneration(t *testing.T) {
+	//stm: @CHAIN_GEN_NEW_GEN_WITH_SECTORS_001, @CHAIN_GEN_NEXT_TIPSET_001
 	t.Run("10-20-1", func(t *testing.T) { testGeneration(t, 10, 20, 1) })
 	t.Run("10-20-25", func(t *testing.T) { testGeneration(t, 10, 20, 25) })
 }
@@ -491,12 +491,13 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, sys vm.Sysca
 		Actors:         filcns.NewActorRegistry(),
 		Syscalls:       mkFakedSigSyscalls(sys),
 		CircSupplyCalc: csc,
+		FilVested:      big.Zero(),
 		NetworkVersion: nv,
-		BaseFee:        types.NewInt(0),
+		BaseFee:        big.Zero(),
 	}
-	vm, err := vm.NewVM(ctx, &vmopt)
+	vm, err := vm.NewLegacyVM(ctx, &vmopt)
 	if err != nil {
-		return cid.Undef, xerrors.Errorf("failed to create NewVM: %w", err)
+		return cid.Undef, xerrors.Errorf("failed to create NewLegacyVM: %w", err)
 	}

 	for mi, m := range template.Miners {
@@ -95,12 +95,13 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.Syscal
 		Syscalls:       mkFakedSigSyscalls(sys),
 		CircSupplyCalc: csc,
 		NetworkVersion: nv,
-		BaseFee:        types.NewInt(0),
+		BaseFee:        big.Zero(),
+		FilVested:      big.Zero(),
 	}

-	vm, err := vm.NewVM(ctx, vmopt)
+	vm, err := vm.NewLegacyVM(ctx, vmopt)
 	if err != nil {
-		return cid.Undef, xerrors.Errorf("failed to create NewVM: %w", err)
+		return cid.Undef, xerrors.Errorf("failed to create NewLegacyVM: %w", err)
 	}

 	if len(miners) == 0 {
@@ -520,7 +521,7 @@ func (fr *fakeRand) GetBeaconRandomness(ctx context.Context, personalization cry
 	return out, nil
 }

-func currentTotalPower(ctx context.Context, vm *vm.VM, maddr address.Address) (*power0.CurrentTotalPowerReturn, error) {
+func currentTotalPower(ctx context.Context, vm *vm.LegacyVM, maddr address.Address) (*power0.CurrentTotalPowerReturn, error) {
 	pwret, err := doExecValue(ctx, vm, power.Address, maddr, big.Zero(), builtin0.MethodsPower.CurrentTotalPower, nil)
 	if err != nil {
 		return nil, err
@@ -533,7 +534,7 @@ func currentTotalPower(ctx context.Context, vm *vm.VM, maddr address.Address) (*
 	return &pwr, nil
 }

-func dealWeight(ctx context.Context, vm *vm.VM, maddr address.Address, dealIDs []abi.DealID, sectorStart, sectorExpiry abi.ChainEpoch, av actors.Version) (abi.DealWeight, abi.DealWeight, error) {
+func dealWeight(ctx context.Context, vm *vm.LegacyVM, maddr address.Address, dealIDs []abi.DealID, sectorStart, sectorExpiry abi.ChainEpoch, av actors.Version) (abi.DealWeight, abi.DealWeight, error) {
 	// TODO: This hack should move to market actor wrapper
 	if av <= actors.Version2 {
 		params := &market0.VerifyDealsForActivationParams{
@@ -593,7 +594,7 @@ func dealWeight(ctx context.Context, vm *vm.VM, maddr address.Address, dealIDs [
 	return dealWeights.Sectors[0].DealWeight, dealWeights.Sectors[0].VerifiedDealWeight, nil
 }

-func currentEpochBlockReward(ctx context.Context, vm *vm.VM, maddr address.Address, av actors.Version) (abi.StoragePower, builtin.FilterEstimate, error) {
+func currentEpochBlockReward(ctx context.Context, vm *vm.LegacyVM, maddr address.Address, av actors.Version) (abi.StoragePower, builtin.FilterEstimate, error) {
 	rwret, err := doExecValue(ctx, vm, reward.Address, maddr, big.Zero(), reward.Methods.ThisEpochReward, nil)
 	if err != nil {
 		return big.Zero(), builtin.FilterEstimate{}, err
@@ -628,7 +629,7 @@ func currentEpochBlockReward(ctx context.Context, vm *vm.VM, maddr address.Addre
 	return epochReward.ThisEpochBaselinePower, builtin.FilterEstimate(epochReward.ThisEpochRewardSmoothed), nil
 }

-func circSupply(ctx context.Context, vmi *vm.VM, maddr address.Address) abi.TokenAmount {
+func circSupply(ctx context.Context, vmi *vm.LegacyVM, maddr address.Address) abi.TokenAmount {
 	unsafeVM := &vm.UnsafeVM{VM: vmi}
 	rt := unsafeVM.MakeRuntime(ctx, &types.Message{
 		GasLimit: 1_000_000_000,
@@ -21,7 +21,7 @@ func mustEnc(i cbg.CBORMarshaler) []byte {
 	return enc
 }

-func doExecValue(ctx context.Context, vm *vm.VM, to, from address.Address, value types.BigInt, method abi.MethodNum, params []byte) ([]byte, error) {
+func doExecValue(ctx context.Context, vm *vm.LegacyVM, to, from address.Address, value types.BigInt, method abi.MethodNum, params []byte) ([]byte, error) {
 	act, err := vm.StateTree().GetActor(from)
 	if err != nil {
 		return nil, xerrors.Errorf("doExec failed to get from actor (%s): %w", from, err)
@@ -1,3 +1,4 @@
+//stm: #unit
 package market

 import (
@@ -22,6 +23,7 @@ import (

 // TestFundManagerBasic verifies that the basic fund manager operations work
 func TestFundManagerBasic(t *testing.T) {
+	//stm: @MARKET_RESERVE_FUNDS_001, @MARKET_RELEASE_FUNDS_001, @MARKET_WITHDRAW_FUNDS_001
 	s := setup(t)
 	defer s.fm.Stop()

@@ -106,6 +108,7 @@ func TestFundManagerBasic(t *testing.T) {

 // TestFundManagerParallel verifies that operations can be run in parallel
 func TestFundManagerParallel(t *testing.T) {
+	//stm: @MARKET_RESERVE_FUNDS_001, @MARKET_RELEASE_FUNDS_001, @MARKET_WITHDRAW_FUNDS_001
 	s := setup(t)
 	defer s.fm.Stop()

@@ -197,6 +200,7 @@ func TestFundManagerParallel(t *testing.T) {

 // TestFundManagerReserveByWallet verifies that reserve requests are grouped by wallet
 func TestFundManagerReserveByWallet(t *testing.T) {
+	//stm: @MARKET_RESERVE_FUNDS_001
 	s := setup(t)
 	defer s.fm.Stop()

@@ -290,6 +294,7 @@ func TestFundManagerReserveByWallet(t *testing.T) {
 // TestFundManagerWithdrawal verifies that as many withdraw operations as
 // possible are processed
 func TestFundManagerWithdrawalLimit(t *testing.T) {
+	//stm: @MARKET_RESERVE_FUNDS_001, @MARKET_RELEASE_FUNDS_001, @MARKET_WITHDRAW_FUNDS_001
 	s := setup(t)
 	defer s.fm.Stop()

@@ -384,6 +389,7 @@ func TestFundManagerWithdrawalLimit(t *testing.T) {

 // TestFundManagerWithdrawByWallet verifies that withdraw requests are grouped by wallet
 func TestFundManagerWithdrawByWallet(t *testing.T) {
+	//stm: @MARKET_RESERVE_FUNDS_001, @MARKET_RELEASE_FUNDS_001, @MARKET_WITHDRAW_FUNDS_001
 	s := setup(t)
 	defer s.fm.Stop()

@@ -493,6 +499,7 @@ func TestFundManagerWithdrawByWallet(t *testing.T) {
 // TestFundManagerRestart verifies that waiting for incomplete requests resumes
 // on restart
 func TestFundManagerRestart(t *testing.T) {
+	//stm: @MARKET_RESERVE_FUNDS_001
 	s := setup(t)
 	defer s.fm.Stop()

@@ -559,6 +566,7 @@ func TestFundManagerRestart(t *testing.T) {
 // 3. Deal B completes, reducing addr1 by 7:      reserved       12    available 12 ->  5
 // 4. Deal A releases 5 from addr1:               reserved 12 ->  7    available        5
 func TestFundManagerReleaseAfterPublish(t *testing.T) {
+	//stm: @MARKET_RESERVE_FUNDS_001, @MARKET_RELEASE_FUNDS_001
 	s := setup(t)
 	defer s.fm.Stop()

@@ -1,3 +1,4 @@
+//stm: #unit
 package messagepool

 import (
@@ -8,6 +9,7 @@ import (
 )

 func TestBlockProbability(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_BLOCK_PROB_001
 	mp := &MessagePool{}
 	bp := mp.blockProbabilities(1 - 0.15)
 	t.Logf("%+v\n", bp)
@@ -20,6 +22,7 @@ func TestBlockProbability(t *testing.T) {
 }

 func TestWinnerProba(t *testing.T) {
+	//stm: @OTHER_IMPLEMENTATION_BLOCK_PROB_002
 	rand.Seed(time.Now().UnixNano())
 	const N = 1000000
 	winnerProba := noWinnersProb()
@@ -106,7 +106,7 @@ func (mp *MessagePool) checkMessages(ctx context.Context, msgs []*types.Message,
 	curTs := mp.curTs
 	mp.curTsLk.Unlock()

-	epoch := curTs.Height()
+	epoch := curTs.Height() + 1

 	var baseFee big.Int
 	if len(curTs.Blocks()) > 0 {
@@ -281,12 +281,11 @@ func (mp *MessagePool) checkMessages(ctx context.Context, msgs []*types.Message,
 		// gas checks

 		// 4. Min Gas
-		minGas := vm.PricelistByEpoch(epoch).OnChainMessage(m.ChainLength())
+		minGas := vm.PricelistByEpochAndNetworkVersion(epoch, nv).OnChainMessage(m.ChainLength())

 		check = api.MessageCheckStatus{
 			Cid: m.Cid(),
-			CheckStatus: api.CheckStatus{
-				Code: api.CheckStatusMessageMinGas,
+			CheckStatus: api.CheckStatus{Code: api.CheckStatusMessageMinGas,
 				Hint: map[string]interface{}{
 					"minGas": minGas,
 				},
@@ -0,0 +1,224 @@
+//stm: #unit
+package messagepool
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/ipfs/go-datastore"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/consensus/filcns"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/types/mock"
+	"github.com/filecoin-project/lotus/chain/wallet"
+	_ "github.com/filecoin-project/lotus/lib/sigs/bls"
+	_ "github.com/filecoin-project/lotus/lib/sigs/secp"
+)
+
+func init() {
+	_ = logging.SetLogLevel("*", "INFO")
+}
+
+func getCheckMessageStatus(statusCode api.CheckStatusCode, msgStatuses []api.MessageCheckStatus) (*api.MessageCheckStatus, error) {
+	for i := 0; i < len(msgStatuses); i++ {
+		iMsgStatuses := msgStatuses[i]
+		if iMsgStatuses.CheckStatus.Code == statusCode {
+			return &iMsgStatuses, nil
+		}
+	}
+	return nil, fmt.Errorf("Could not find CheckStatusCode %s", statusCode)
+}
+
+func TestCheckMessages(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_CHECK_MESSAGES_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sender, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(sender, 1000e15)
+	target := mock.Address(1001)
+
+	var protos []*api.MessagePrototype
+	for i := 0; i < 5; i++ {
+		msg := &types.Message{
+			To:         target,
+			From:       sender,
+			Value:      types.NewInt(1),
+			Nonce:      uint64(i),
+			GasLimit:   50000000,
+			GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+			GasPremium: types.NewInt(1),
+			Params:     make([]byte, 2<<10),
+		}
+		proto := &api.MessagePrototype{
+			Message:    *msg,
+			ValidNonce: true,
+		}
+		protos = append(protos, proto)
+	}
+
+	messageStatuses, err := mp.CheckMessages(context.TODO(), protos)
+	assert.NoError(t, err)
+	for i := 0; i < len(messageStatuses); i++ {
+		iMsgStatuses := messageStatuses[i]
+		for j := 0; j < len(iMsgStatuses); j++ {
+			jStatus := iMsgStatuses[i]
+			assert.True(t, jStatus.OK)
+		}
+	}
+}
+
+func TestCheckPendingMessages(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_CHECK_PENDING_MESSAGES_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sender, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(sender, 1000e15)
+	target := mock.Address(1001)
+
+	// add a valid message to the pool
+	msg := &types.Message{
+		To:         target,
+		From:       sender,
+		Value:      types.NewInt(1),
+		Nonce:      0,
+		GasLimit:   50000000,
+		GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+		GasPremium: types.NewInt(1),
+		Params:     make([]byte, 2<<10),
+	}
+
+	sig, err := w.WalletSign(context.TODO(), sender, msg.Cid().Bytes(), api.MsgMeta{})
+	if err != nil {
+		panic(err)
+	}
+	sm := &types.SignedMessage{
+		Message:   *msg,
+		Signature: *sig,
+	}
+	mustAdd(t, mp, sm)
+
+	messageStatuses, err := mp.CheckPendingMessages(context.TODO(), sender)
+	assert.NoError(t, err)
+	for i := 0; i < len(messageStatuses); i++ {
+		iMsgStatuses := messageStatuses[i]
+		for j := 0; j < len(iMsgStatuses); j++ {
+			jStatus := iMsgStatuses[i]
+			assert.True(t, jStatus.OK)
+		}
+	}
+}
+
+func TestCheckReplaceMessages(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_CHECK_REPLACE_MESSAGES_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sender, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(sender, 1000e15)
+	target := mock.Address(1001)
+
+	// add a valid message to the pool
+	msg := &types.Message{
+		To:         target,
+		From:       sender,
+		Value:      types.NewInt(1),
+		Nonce:      0,
+		GasLimit:   50000000,
+		GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+		GasPremium: types.NewInt(1),
+		Params:     make([]byte, 2<<10),
+	}
+
+	sig, err := w.WalletSign(context.TODO(), sender, msg.Cid().Bytes(), api.MsgMeta{})
+	if err != nil {
+		panic(err)
+	}
+	sm := &types.SignedMessage{
+		Message:   *msg,
+		Signature: *sig,
+	}
+	mustAdd(t, mp, sm)
+
+	// create a new message with the same data, except that it is too big
+	var msgs []*types.Message
+	invalidmsg := &types.Message{
+		To:         target,
+		From:       sender,
+		Value:      types.NewInt(1),
+		Nonce:      0,
+		GasLimit:   50000000,
+		GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+		GasPremium: types.NewInt(1),
+		Params:     make([]byte, 128<<10),
+	}
+	msgs = append(msgs, invalidmsg)
+
+	{
+		messageStatuses, err := mp.CheckReplaceMessages(context.TODO(), msgs)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for i := 0; i < len(messageStatuses); i++ {
+			iMsgStatuses := messageStatuses[i]
+
+			status, err := getCheckMessageStatus(api.CheckStatusMessageSize, iMsgStatuses)
+			if err != nil {
+				t.Fatal(err)
+			}
+			// the replacement message should cause a status error
+			assert.False(t, status.OK)
+		}
+	}
+
+}
@@ -628,8 +628,12 @@ func (mp *MessagePool) addLocal(ctx context.Context, m *types.SignedMessage) err
 // For non local messages, if the message cannot be included in the next 20 blocks it returns
 // a (soft) validation error.
 func (mp *MessagePool) verifyMsgBeforeAdd(m *types.SignedMessage, curTs *types.TipSet, local bool) (bool, error) {
-	epoch := curTs.Height()
-	minGas := vm.PricelistByEpoch(epoch).OnChainMessage(m.ChainLength())
+	epoch := curTs.Height() + 1
+	nv, err := mp.getNtwkVersion(epoch)
+	if err != nil {
+		return false, xerrors.Errorf("getting network version: %w", err)
+	}
+	minGas := vm.PricelistByEpochAndNetworkVersion(epoch, nv).OnChainMessage(m.ChainLength())

 	if err := m.VMMessage().ValidForBlockInclusion(minGas.Total(), build.NewestNetworkVersion); err != nil {
 		return false, xerrors.Errorf("message will not be included in a block: %w", err)
@@ -9,6 +9,7 @@ import (

 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
 	logging "github.com/ipfs/go-log/v2"
@@ -226,6 +227,8 @@ func mustAdd(t *testing.T, mp *MessagePool, msg *types.SignedMessage) {
 }

 func TestMessagePool(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_GET_NONCE_001
+
 	tma := newTestMpoolAPI()

 	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
@@ -327,6 +330,7 @@ func TestCheckMessageBig(t *testing.T) {
 			Message:   *msg,
 			Signature: *sig,
 		}
+		//stm: @CHAIN_MEMPOOL_PUSH_001
 		err = mp.Add(context.TODO(), sm)
 		assert.ErrorIs(t, err, ErrMessageTooBig)
 	}
@@ -760,3 +764,314 @@ func TestUpdates(t *testing.T) {
 		t.Fatal("expected closed channel, but got an update instead")
 	}
 }
+
+func TestMessageBelowMinGasFee(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	// fee is just below minimum gas fee
+	fee := minimumBaseFee.Uint64() - 1
+	{
+		msg := &types.Message{
+			To:         to,
+			From:       from,
+			Value:      types.NewInt(1),
+			Nonce:      0,
+			GasLimit:   50000000,
+			GasFeeCap:  types.NewInt(fee),
+			GasPremium: types.NewInt(1),
+			Params:     make([]byte, 32<<10),
+		}
+
+		sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{})
+		if err != nil {
+			panic(err)
+		}
+		sm := &types.SignedMessage{
+			Message:   *msg,
+			Signature: *sig,
+		}
+		err = mp.Add(context.TODO(), sm)
+		assert.ErrorIs(t, err, ErrGasFeeCapTooLow)
+	}
+}
+
+func TestMessageValueTooHigh(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	totalFil := types.TotalFilecoinInt
+	extra := types.NewInt(1)
+
+	value := types.BigAdd(totalFil, extra)
+	{
+		msg := &types.Message{
+			To:         to,
+			From:       from,
+			Value:      value,
+			Nonce:      0,
+			GasLimit:   50000000,
+			GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+			GasPremium: types.NewInt(1),
+			Params:     make([]byte, 32<<10),
+		}
+
+		sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{})
+		if err != nil {
+			panic(err)
+		}
+		sm := &types.SignedMessage{
+			Message:   *msg,
+			Signature: *sig,
+		}
+		err = mp.Add(context.TODO(), sm)
+		assert.Error(t, err)
+	}
+}
+
+func TestMessageSignatureInvalid(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		msg := &types.Message{
+			To:         to,
+			From:       from,
+			Value:      types.NewInt(1),
+			Nonce:      0,
+			GasLimit:   50000000,
+			GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+			GasPremium: types.NewInt(1),
+			Params:     make([]byte, 32<<10),
+		}
+
+		badSig := &crypto.Signature{
+			Type: crypto.SigTypeSecp256k1,
+			Data: make([]byte, 0),
+		}
+		sm := &types.SignedMessage{
+			Message:   *msg,
+			Signature: *badSig,
+		}
+		err = mp.Add(context.TODO(), sm)
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "invalid signature length")
+	}
+}
+
+func TestAddMessageTwice(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		msg := &types.Message{
+			To:         to,
+			From:       from,
+			Value:      types.NewInt(1),
+			Nonce:      0,
+			GasLimit:   50000000,
+			GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+			GasPremium: types.NewInt(1),
+			Params:     make([]byte, 32<<10),
+		}
+
+		sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{})
+		if err != nil {
+			panic(err)
+		}
+		sm := &types.SignedMessage{
+			Message:   *msg,
+			Signature: *sig,
+		}
+		mustAdd(t, mp, sm)
+
+		err = mp.Add(context.TODO(), sm)
+		assert.Contains(t, err.Error(), "with nonce 0 already in mpool")
+	}
+}
+
+func TestAddMessageTwiceNonceGap(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		// create message with invalid nonce (1)
+		sm := makeTestMessage(w, from, to, 1, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		// then try to add message again
+		err = mp.Add(context.TODO(), sm)
+		assert.Contains(t, err.Error(), "unfulfilled nonce gap")
+	}
+}
+
+func TestAddMessageTwiceCidDiff(t *testing.T) {
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		// Create message with different data, so CID is different
+		sm2 := makeTestMessage(w, from, to, 0, 50_000_001, minimumBaseFee.Uint64())
+
+		//stm: @CHAIN_MEMPOOL_PUSH_001
+		// then try to add message again
+		err = mp.Add(context.TODO(), sm2)
+		// assert.Contains(t, err.Error(), "replace by fee has too low GasPremium")
+		assert.Error(t, err)
+	}
+}
+
+func TestAddMessageTwiceCidDiffReplaced(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		// Create message with different data, so CID is different
+		sm2 := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64()*2)
+		mustAdd(t, mp, sm2)
+	}
+}
+
+func TestRemoveMessage(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		//stm: @CHAIN_MEMPOOL_REMOVE_001
+		// remove message for sender
+		mp.Remove(context.TODO(), from, sm.Message.Nonce, true)
+
+		//stm: @CHAIN_MEMPOOL_PENDING_FOR_001
+		// check messages in pool: should be none present
+		msgs := mp.pendingFor(context.TODO(), from)
+		assert.Len(t, msgs, 0)
+	}
+}
@@ -1,3 +1,4 @@
+//stm: #unit
 package messagepool

 import (
@@ -16,6 +17,7 @@ import (
 )

 func TestRepubMessages(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001
 	oldRepublishBatchDelay := RepublishBatchDelay
 	RepublishBatchDelay = time.Microsecond
 	defer func() {
@@ -57,6 +59,7 @@ func TestRepubMessages(t *testing.T) {

 	for i := 0; i < 10; i++ {
 		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		//stm: @CHAIN_MEMPOOL_PUSH_001
 		_, err := mp.Push(context.TODO(), m)
 		if err != nil {
 			t.Fatal(err)
@@ -781,6 +781,7 @@ func (mp *MessagePool) createMessageChains(actor address.Address, mset map[uint6
 	//   cannot exceed the block limit; drop all messages that exceed the limit
 	// - the total gasReward cannot exceed the actor's balance; drop all messages that exceed
 	//   the balance
+
 	a, err := mp.api.GetActorAfter(actor, ts)
 	if err != nil {
 		log.Errorf("failed to load actor state, not building chain for %s: %v", actor, err)
@@ -793,6 +794,12 @@ func (mp *MessagePool) createMessageChains(actor address.Address, mset map[uint6
 	skip := 0
 	i := 0
 	rewards := make([]*big.Int, 0, len(msgs))
+
+	nv, err := mp.getNtwkVersion(ts.Height())
+	if err != nil {
+		log.Errorf("getting network version: %v", err)
+		return nil
+	}
 	for i = 0; i < len(msgs); i++ {
 		m := msgs[i]

@@ -808,7 +815,7 @@ func (mp *MessagePool) createMessageChains(actor address.Address, mset map[uint6
 		}
 		curNonce++

-		minGas := vm.PricelistByEpoch(ts.Height()).OnChainMessage(m.ChainLength()).Total()
+		minGas := vm.PricelistByEpochAndNetworkVersion(ts.Height(), nv).OnChainMessage(m.ChainLength()).Total()
 		if m.Message.GasLimit < minGas {
 			break
 		}
@@ -1,3 +1,4 @@
+//stm: #unit
 package messagepool

 import (
@@ -74,6 +75,8 @@ func makeTestMpool() (*MessagePool, *testMpoolAPI) {
 }

 func TestMessageChains(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001
+	//stm: @CHAIN_MEMPOOL_CREATE_MSG_CHAINS_001
 	mp, tma := makeTestMpool()

 	// the actors
@@ -310,6 +313,8 @@ func TestMessageChains(t *testing.T) {
 }

 func TestMessageChainSkipping(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_CREATE_MSG_CHAINS_001
+
 	// regression test for chain skip bug

 	mp, tma := makeTestMpool()
@@ -382,6 +387,7 @@ func TestMessageChainSkipping(t *testing.T) {
 }

 func TestBasicMessageSelection(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	oldMaxNonceGap := MaxNonceGap
 	MaxNonceGap = 1000
 	defer func() {
@@ -532,6 +538,7 @@ func TestBasicMessageSelection(t *testing.T) {
 }

 func TestMessageSelectionTrimmingGas(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()

 	// the actors
@@ -595,6 +602,7 @@ func TestMessageSelectionTrimmingGas(t *testing.T) {
 }

 func TestMessageSelectionTrimmingMsgsBasic(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()

 	// the actors
@@ -641,6 +649,7 @@ func TestMessageSelectionTrimmingMsgsBasic(t *testing.T) {
 }

 func TestMessageSelectionTrimmingMsgsTwoSendersBasic(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()

 	// the actors
@@ -707,6 +716,7 @@ func TestMessageSelectionTrimmingMsgsTwoSendersBasic(t *testing.T) {
 }

 func TestMessageSelectionTrimmingMsgsTwoSendersAdvanced(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()

 	// the actors
@@ -788,6 +798,7 @@ func TestMessageSelectionTrimmingMsgsTwoSendersAdvanced(t *testing.T) {
 }

 func TestPriorityMessageSelection(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()

 	// the actors
@@ -867,6 +878,7 @@ func TestPriorityMessageSelection(t *testing.T) {
 }

 func TestPriorityMessageSelection2(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()

 	// the actors
@@ -934,6 +946,7 @@ func TestPriorityMessageSelection2(t *testing.T) {
 }

 func TestPriorityMessageSelection3(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()

 	// the actors
@@ -1028,6 +1041,8 @@ func TestPriorityMessageSelection3(t *testing.T) {
 }

 func TestOptimalMessageSelection1(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	// this test uses just a single actor sending messages with a low tq
 	// the chain depenent merging algorithm should pick messages from the actor
 	// from the start
@@ -1094,6 +1109,8 @@ func TestOptimalMessageSelection1(t *testing.T) {
 }

 func TestOptimalMessageSelection2(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	// this test uses two actors sending messages to each other, with the first
 	// actor paying (much) higher gas premium than the second.
 	// We select with a low ticket quality; the chain depenent merging algorithm should pick
@@ -1173,6 +1190,8 @@ func TestOptimalMessageSelection2(t *testing.T) {
 }

 func TestOptimalMessageSelection3(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	// this test uses 10 actors sending a block of messages to each other, with the the first
 	// actors paying higher gas premium than the subsequent actors.
 	// We select with a low ticket quality; the chain dependent merging algorithm should pick
@@ -1416,6 +1435,8 @@ func makeZipfPremiumDistribution(rng *rand.Rand) func() uint64 {
 }

 func TestCompetitiveMessageSelectionExp(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	if testing.Short() {
 		t.Skip("skipping in short mode")
 	}
@@ -1439,6 +1460,8 @@ func TestCompetitiveMessageSelectionExp(t *testing.T) {
 }

 func TestCompetitiveMessageSelectionZipf(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	if testing.Short() {
 		t.Skip("skipping in short mode")
 	}
@@ -1462,6 +1485,7 @@ func TestCompetitiveMessageSelectionZipf(t *testing.T) {
 }

 func TestGasReward(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_GET_GAS_REWARD_001
 	tests := []struct {
 		Premium   uint64
 		FeeCap    uint64
@@ -1494,6 +1518,8 @@ func TestGasReward(t *testing.T) {
 }

 func TestRealWorldSelection(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @TOKEN_WALLET_SIGN_001, @CHAIN_MEMPOOL_SELECT_001
+
 	// load test-messages.json.gz and rewrite the messages so that
 	// 1) we map each real actor to a test actor so that we can sign the messages
 	// 2) adjust the nonces so that they start from 0
@@ -1,3 +1,4 @@
+//stm: #unit
 package state

 import (
@@ -18,6 +19,7 @@ import (
 )

 func BenchmarkStateTreeSet(b *testing.B) {
+	//stm: @CHAIN_STATETREE_SET_ACTOR_001
 	cst := cbor.NewMemCborStore()
 	st, err := NewStateTree(cst, types.StateTreeVersion1)
 	if err != nil {
@@ -45,6 +47,7 @@ func BenchmarkStateTreeSet(b *testing.B) {
 }

 func BenchmarkStateTreeSetFlush(b *testing.B) {
+	//stm: @CHAIN_STATETREE_SET_ACTOR_001
 	cst := cbor.NewMemCborStore()
 	sv, err := VersionForNetwork(build.NewestNetworkVersion)
 	if err != nil {
@@ -80,6 +83,8 @@ func BenchmarkStateTreeSetFlush(b *testing.B) {
 }

 func TestResolveCache(t *testing.T) {
+	//stm: @CHAIN_STATETREE_SET_ACTOR_001, @CHAIN_STATETREE_GET_ACTOR_001, @CHAIN_STATETREE_VERSION_FOR_NETWORK_001
+	//stm: @CHAIN_STATETREE_SNAPSHOT_001, @CHAIN_STATETREE_SNAPSHOT_CLEAR_001
 	cst := cbor.NewMemCborStore()
 	sv, err := VersionForNetwork(build.NewestNetworkVersion)
 	if err != nil {
@@ -182,6 +187,8 @@ func TestResolveCache(t *testing.T) {
 }

 func BenchmarkStateTree10kGetActor(b *testing.B) {
+	//stm: @CHAIN_STATETREE_SET_ACTOR_001, @CHAIN_STATETREE_GET_ACTOR_001, @CHAIN_STATETREE_VERSION_FOR_NETWORK_001
+	//stm: @CHAIN_STATETREE_FLUSH_001
 	cst := cbor.NewMemCborStore()
 	sv, err := VersionForNetwork(build.NewestNetworkVersion)
 	if err != nil {
@@ -229,6 +236,7 @@ func BenchmarkStateTree10kGetActor(b *testing.B) {
 }

 func TestSetCache(t *testing.T) {
+	//stm: @CHAIN_STATETREE_SET_ACTOR_001, @CHAIN_STATETREE_GET_ACTOR_001, @CHAIN_STATETREE_VERSION_FOR_NETWORK_001
 	cst := cbor.NewMemCborStore()
 	sv, err := VersionForNetwork(build.NewestNetworkVersion)
 	if err != nil {
@@ -270,6 +278,8 @@ func TestSetCache(t *testing.T) {
 }

 func TestSnapshots(t *testing.T) {
+	//stm: @CHAIN_STATETREE_SET_ACTOR_001, @CHAIN_STATETREE_GET_ACTOR_001, @CHAIN_STATETREE_VERSION_FOR_NETWORK_001
+	//stm: @CHAIN_STATETREE_FLUSH_001, @CHAIN_STATETREE_SNAPSHOT_REVERT_001, CHAIN_STATETREE_SNAPSHOT_CLEAR_001
 	ctx := context.Background()
 	cst := cbor.NewMemCborStore()

@@ -360,6 +370,7 @@ func assertNotHas(t *testing.T, st *StateTree, addr address.Address) {
 }

 func TestStateTreeConsistency(t *testing.T) {
+	//stm: @CHAIN_STATETREE_SET_ACTOR_001, @CHAIN_STATETREE_VERSION_FOR_NETWORK_001, @CHAIN_STATETREE_FLUSH_001
 	cst := cbor.NewMemCborStore()

 	// TODO: ActorUpgrade: this test tests pre actors v2
@@ -5,6 +5,12 @@ import (
 	"errors"
 	"fmt"

+	"github.com/filecoin-project/lotus/blockstore"
+
+	cbor "github.com/ipfs/go-ipld-cbor"
+
+	"github.com/filecoin-project/lotus/chain/state"
+
 	"github.com/filecoin-project/lotus/chain/rand"

 	"github.com/filecoin-project/go-address"
@@ -64,6 +70,8 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
 		pheight = ts.Height() - 1
 	}

+	// Since we're simulating a future message, pretend we're applying it in the "next" tipset
+	vmHeight := pheight + 1
 	bstate := ts.ParentState()

 	// Run the (not expensive) migration.
@@ -72,9 +80,14 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
 		return nil, fmt.Errorf("failed to handle fork: %w", err)
 	}

+	filVested, err := sm.GetFilVested(ctx, vmHeight)
+	if err != nil {
+		return nil, err
+	}
+
 	vmopt := &vm.VMOpts{
 		StateBase:      bstate,
-		Epoch:          pheight + 1,
+		Epoch:          vmHeight,
 		Rand:           rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon, sm.GetNetworkVersion),
 		Bstore:         sm.cs.StateBlockstore(),
 		Actors:         sm.tsExec.NewActorRegistry(),
@@ -82,6 +95,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
 		CircSupplyCalc: sm.GetVMCirculatingSupply,
 		NetworkVersion: sm.GetNetworkVersion(ctx, pheight+1),
 		BaseFee:        types.NewInt(0),
+		FilVested:      filVested,
 		LookbackState:  LookbackStateGetterForTipset(sm, ts),
 	}

@@ -112,7 +126,12 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
 		)
 	}

-	fromActor, err := vmi.StateTree().GetActor(msg.From)
+	stTree, err := sm.StateTree(bstate)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to load state tree: %w", err)
+	}
+
+	fromActor, err := stTree.GetActor(msg.From)
 	if err != nil {
 		return nil, xerrors.Errorf("call raw get actor: %s", err)
 	}
@@ -175,13 +194,16 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 		}
 	}

-	state, _, err := sm.TipSetState(ctx, ts)
+	// Since we're simulating a future message, pretend we're applying it in the "next" tipset
+	vmHeight := ts.Height() + 1
+
+	stateCid, _, err := sm.TipSetState(ctx, ts)
 	if err != nil {
 		return nil, xerrors.Errorf("computing tipset state: %w", err)
 	}

 	// Technically, the tipset we're passing in here should be ts+1, but that may not exist.
-	state, err = sm.HandleStateForks(ctx, state, ts.Height(), nil, ts)
+	stateCid, err = sm.HandleStateForks(ctx, stateCid, ts.Height(), nil, ts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to handle fork: %w", err)
 	}
@@ -196,16 +218,23 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 		)
 	}

+	filVested, err := sm.GetFilVested(ctx, vmHeight)
+	if err != nil {
+		return nil, err
+	}
+
+	buffStore := blockstore.NewTieredBstore(sm.cs.StateBlockstore(), blockstore.NewMemorySync())
 	vmopt := &vm.VMOpts{
-		StateBase:      state,
-		Epoch:          ts.Height() + 1,
+		StateBase:      stateCid,
+		Epoch:          vmHeight,
 		Rand:           r,
-		Bstore:         sm.cs.StateBlockstore(),
+		Bstore:         buffStore,
 		Actors:         sm.tsExec.NewActorRegistry(),
 		Syscalls:       sm.Syscalls,
 		CircSupplyCalc: sm.GetVMCirculatingSupply,
 		NetworkVersion: sm.GetNetworkVersion(ctx, ts.Height()+1),
 		BaseFee:        ts.Blocks()[0].ParentBaseFee,
+		FilVested:      filVested,
 		LookbackState:  LookbackStateGetterForTipset(sm, ts),
 	}
 	vmi, err := sm.newVM(ctx, vmopt)
@@ -219,7 +248,19 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 		}
 	}

-	fromActor, err := vmi.StateTree().GetActor(msg.From)
+	// We flush to get the VM's view of the state tree after applying the above messages
+	// This is needed to get the correct nonce from the actor state to match the VM
+	stateCid, err = vmi.Flush(ctx)
+	if err != nil {
+		return nil, xerrors.Errorf("flushing vm: %w", err)
+	}
+
+	stTree, err := state.LoadStateTree(cbor.NewCborStore(buffStore), stateCid)
+	if err != nil {
+		return nil, xerrors.Errorf("loading state tree: %w", err)
+	}
+
+	fromActor, err := stTree.GetActor(msg.From)
 	if err != nil {
 		return nil, xerrors.Errorf("call raw get actor: %s", err)
 	}
@@ -8,6 +8,8 @@ import (
 	"sync"
 	"time"

+	"github.com/filecoin-project/lotus/build"
+
 	"github.com/filecoin-project/specs-actors/v7/actors/migration/nv15"

 	"github.com/ipfs/go-cid"
@@ -161,7 +163,8 @@ func (us UpgradeSchedule) GetNtwkVersion(e abi.ChainEpoch) (network.Version, err
 			return u.Network, nil
 		}
 	}
-	return network.Version0, xerrors.Errorf("Epoch %d has no defined network version", e)
+
+	return build.GenesisNetworkVersion, nil
 }

 func (sm *StateManager) HandleStateForks(ctx context.Context, root cid.Cid, height abi.ChainEpoch, cb ExecMonitor, ts *types.TipSet) (cid.Cid, error) {
@@ -1,3 +1,4 @@
+//stm: #integration
 package stmgr_test

 import (
@@ -106,6 +107,9 @@ func (ta *testActor) TestMethod(rt rt2.Runtime, params *abi.EmptyValue) *abi.Emp
 }

 func TestForkHeightTriggers(t *testing.T) {
+	//stm: @CHAIN_STATETREE_GET_ACTOR_001, @CHAIN_STATETREE_FLUSH_001, @TOKEN_WALLET_SIGN_001
+	//stm: @CHAIN_GEN_NEXT_TIPSET_001
+	//stm: @CHAIN_STATE_RESOLVE_TO_KEY_ADDR_001, @CHAIN_STATE_SET_VM_CONSTRUCTOR_001
 	logging.SetAllLoggers(logging.LevelInfo)

 	ctx := context.TODO()
@@ -166,8 +170,8 @@ func TestForkHeightTriggers(t *testing.T) {
 	inv := filcns.NewActorRegistry()
 	inv.Register(nil, testActor{})

-	sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (*vm.VM, error) {
-		nvm, err := vm.NewVM(ctx, vmopt)
+	sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (vm.Interface, error) {
+		nvm, err := vm.NewLegacyVM(ctx, vmopt)
 		if err != nil {
 			return nil, err
 		}
@@ -241,6 +245,8 @@ func TestForkHeightTriggers(t *testing.T) {
 }

 func TestForkRefuseCall(t *testing.T) {
+	//stm: @CHAIN_GEN_NEXT_TIPSET_001, @CHAIN_GEN_NEXT_TIPSET_FROM_MINERS_001
+	//stm: @CHAIN_STATE_RESOLVE_TO_KEY_ADDR_001, @CHAIN_STATE_SET_VM_CONSTRUCTOR_001, @CHAIN_STATE_CALL_001
 	logging.SetAllLoggers(logging.LevelInfo)

 	for after := 0; after < 3; after++ {
@@ -281,8 +287,8 @@ func testForkRefuseCall(t *testing.T, nullsBefore, nullsAfter int) {
 	inv := filcns.NewActorRegistry()
 	inv.Register(nil, testActor{})

-	sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (*vm.VM, error) {
-		nvm, err := vm.NewVM(ctx, vmopt)
+	sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (vm.Interface, error) {
+		nvm, err := vm.NewLegacyVM(ctx, vmopt)
 		if err != nil {
 			return nil, err
 		}
@@ -360,6 +366,8 @@ func testForkRefuseCall(t *testing.T, nullsBefore, nullsAfter int) {
 }

 func TestForkPreMigration(t *testing.T) {
+	//stm: @CHAIN_GEN_NEXT_TIPSET_001,
+	//stm: @CHAIN_STATE_RESOLVE_TO_KEY_ADDR_001, @CHAIN_STATE_SET_VM_CONSTRUCTOR_001
 	logging.SetAllLoggers(logging.LevelInfo)

 	cg, err := gen.NewGenerator()
@@ -500,8 +508,8 @@ func TestForkPreMigration(t *testing.T) {
 	inv := filcns.NewActorRegistry()
 	inv.Register(nil, testActor{})

-	sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (*vm.VM, error) {
-		nvm, err := vm.NewVM(ctx, vmopt)
+	sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (vm.Interface, error) {
+		nvm, err := vm.NewLegacyVM(ctx, vmopt)
 		if err != nil {
 			return nil, err
 		}
@@ -1,3 +1,4 @@
+//stm: #unit
 package stmgr_test

 import (
@@ -12,6 +13,8 @@ import (
 )

 func TestSearchForMessageReplacements(t *testing.T) {
+	//stm: @CHAIN_GEN_NEXT_TIPSET_001
+	//stm: @CHAIN_STATE_SEARCH_MSG_001
 	ctx := context.Background()
 	cg, err := gen.NewGenerator()
 	if err != nil {
@@ -84,7 +84,7 @@ type StateManager struct {
 	compWait            map[string]chan struct{}
 	stlk                sync.Mutex
 	genesisMsigLk       sync.Mutex
-	newVM               func(context.Context, *vm.VMOpts) (*vm.VM, error)
+	newVM               func(context.Context, *vm.VMOpts) (vm.Interface, error)
 	Syscalls            vm.SyscallBuilder
 	preIgnitionVesting  []msig0.State
 	postIgnitionVesting []msig0.State
@@ -347,12 +347,12 @@ func (sm *StateManager) ValidateChain(ctx context.Context, ts *types.TipSet) err
 	return nil
 }

-func (sm *StateManager) SetVMConstructor(nvm func(context.Context, *vm.VMOpts) (*vm.VM, error)) {
+func (sm *StateManager) SetVMConstructor(nvm func(context.Context, *vm.VMOpts) (vm.Interface, error)) {
 	sm.newVM = nvm
 }

-func (sm *StateManager) VMConstructor() func(context.Context, *vm.VMOpts) (*vm.VM, error) {
-	return func(ctx context.Context, opts *vm.VMOpts) (*vm.VM, error) {
+func (sm *StateManager) VMConstructor() func(context.Context, *vm.VMOpts) (vm.Interface, error) {
+	return func(ctx context.Context, opts *vm.VMOpts) (vm.Interface, error) {
 		return sm.newVM(ctx, opts)
 	}
 }
@@ -196,8 +196,32 @@ func (sm *StateManager) setupPostCalicoVesting(ctx context.Context) error {
 // GetVestedFunds returns all funds that have "left" actors that are in the genesis state:
 // - For Multisigs, it counts the actual amounts that have vested at the given epoch
 // - For Accounts, it counts max(currentBalance - genesisBalance, 0).
-func (sm *StateManager) GetFilVested(ctx context.Context, height abi.ChainEpoch, st *state.StateTree) (abi.TokenAmount, error) {
+func (sm *StateManager) GetFilVested(ctx context.Context, height abi.ChainEpoch) (abi.TokenAmount, error) {
 	vf := big.Zero()
+
+	sm.genesisMsigLk.Lock()
+	defer sm.genesisMsigLk.Unlock()
+
+	// TODO: combine all this?
+	if sm.preIgnitionVesting == nil || sm.genesisPledge.IsZero() || sm.genesisMarketFunds.IsZero() {
+		err := sm.setupGenesisVestingSchedule(ctx)
+		if err != nil {
+			return vf, xerrors.Errorf("failed to setup pre-ignition vesting schedule: %w", err)
+		}
+	}
+	if sm.postIgnitionVesting == nil {
+		err := sm.setupPostIgnitionVesting(ctx)
+		if err != nil {
+			return vf, xerrors.Errorf("failed to setup post-ignition vesting schedule: %w", err)
+		}
+	}
+	if sm.postCalicoVesting == nil {
+		err := sm.setupPostCalicoVesting(ctx)
+		if err != nil {
+			return vf, xerrors.Errorf("failed to setup post-calico vesting schedule: %w", err)
+		}
+	}
+
 	if height <= build.UpgradeIgnitionHeight {
 		for _, v := range sm.preIgnitionVesting {
 			au := big.Sub(v.InitialBalance, v.AmountLocked(height))
@@ -282,7 +306,7 @@ func getFilPowerLocked(ctx context.Context, st *state.StateTree) (abi.TokenAmoun
 	return pst.TotalLocked()
 }

-func (sm *StateManager) GetFilLocked(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) {
+func GetFilLocked(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) {

 	filMarketLocked, err := getFilMarketLocked(ctx, st)
 	if err != nil {
@@ -316,28 +340,7 @@ func (sm *StateManager) GetVMCirculatingSupply(ctx context.Context, height abi.C
 }

 func (sm *StateManager) GetVMCirculatingSupplyDetailed(ctx context.Context, height abi.ChainEpoch, st *state.StateTree) (api.CirculatingSupply, error) {
-	sm.genesisMsigLk.Lock()
-	defer sm.genesisMsigLk.Unlock()
-	if sm.preIgnitionVesting == nil || sm.genesisPledge.IsZero() || sm.genesisMarketFunds.IsZero() {
-		err := sm.setupGenesisVestingSchedule(ctx)
-		if err != nil {
-			return api.CirculatingSupply{}, xerrors.Errorf("failed to setup pre-ignition vesting schedule: %w", err)
-		}
-	}
-	if sm.postIgnitionVesting == nil {
-		err := sm.setupPostIgnitionVesting(ctx)
-		if err != nil {
-			return api.CirculatingSupply{}, xerrors.Errorf("failed to setup post-ignition vesting schedule: %w", err)
-		}
-	}
-	if sm.postCalicoVesting == nil {
-		err := sm.setupPostCalicoVesting(ctx)
-		if err != nil {
-			return api.CirculatingSupply{}, xerrors.Errorf("failed to setup post-calico vesting schedule: %w", err)
-		}
-	}
-
-	filVested, err := sm.GetFilVested(ctx, height, st)
+	filVested, err := sm.GetFilVested(ctx, height)
 	if err != nil {
 		return api.CirculatingSupply{}, xerrors.Errorf("failed to calculate filVested: %w", err)
 	}
@@ -360,7 +363,7 @@ func (sm *StateManager) GetVMCirculatingSupplyDetailed(ctx context.Context, heig
 		return api.CirculatingSupply{}, xerrors.Errorf("failed to calculate filBurnt: %w", err)
 	}

-	filLocked, err := sm.GetFilLocked(ctx, st)
+	filLocked, err := GetFilLocked(ctx, st)
 	if err != nil {
 		return api.CirculatingSupply{}, xerrors.Errorf("failed to calculate filLocked: %w", err)
 	}
@@ -79,6 +79,11 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
 		// future. It's not guaranteed to be accurate... but that's fine.
 	}

+	filVested, err := sm.GetFilVested(ctx, height)
+	if err != nil {
+		return cid.Undef, nil, err
+	}
+
 	r := rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon, sm.GetNetworkVersion)
 	vmopt := &vm.VMOpts{
 		StateBase:      base,
@@ -90,6 +95,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
 		CircSupplyCalc: sm.GetVMCirculatingSupply,
 		NetworkVersion: sm.GetNetworkVersion(ctx, height),
 		BaseFee:        ts.Blocks()[0].ParentBaseFee,
+		FilVested:      filVested,
 		LookbackState:  LookbackStateGetterForTipset(sm, ts),
 	}
 	vmi, err := sm.newVM(ctx, vmopt)
@@ -1,3 +1,5 @@
+//stm: #unit
+
 package store

 import (
@@ -10,6 +12,7 @@ import (
 )

 func TestBaseFee(t *testing.T) {
+	//stm: @CHAIN_STORE_COMPUTE_NEXT_BASE_FEE_001
 	tests := []struct {
 		basefee             uint64
 		limitUsed           int64
@@ -1,3 +1,4 @@
+//stm: #unit
 package store_test

 import (
@@ -10,6 +11,9 @@ import (
 )

 func TestChainCheckpoint(t *testing.T) {
+	//stm: @CHAIN_GEN_NEXT_TIPSET_FROM_MINERS_001
+	//stm: @CHAIN_STORE_GET_TIPSET_FROM_KEY_001, @CHAIN_STORE_SET_HEAD_001, @CHAIN_STORE_GET_HEAVIEST_TIPSET_001
+	//stm: @CHAIN_STORE_SET_CHECKPOINT_001, @CHAIN_STORE_MAYBE_TAKE_HEAVIER_TIPSET_001, @CHAIN_STORE_REMOVE_CHECKPOINT_001
 	ctx := context.Background()

 	cg, err := gen.NewGenerator()
@@ -1,3 +1,4 @@
+//stm: #unit
 package store

 import (
@@ -9,6 +10,7 @@ import (
 )

 func TestHeadChangeCoalescer(t *testing.T) {
+	//stm: @CHAIN_STORE_COALESCE_HEAD_CHANGE_001
 	notif := make(chan headChange, 1)
 	c := NewHeadChangeCoalescer(func(revert, apply []*types.TipSet) error {
 		notif <- headChange{apply: apply, revert: revert}
@@ -1,3 +1,4 @@
+//stm: #unit
 package store_test

 import (
@@ -17,6 +18,9 @@ import (
 )

 func TestIndexSeeks(t *testing.T) {
+	//stm: @CHAIN_STORE_IMPORT_001
+	//stm: @CHAIN_STORE_GET_TIPSET_BY_HEIGHT_001, @CHAIN_STORE_PUT_TIPSET_001, @CHAIN_STORE_SET_GENESIS_BLOCK_001
+	//stm: @CHAIN_STORE_CLOSE_001
 	cg, err := gen.NewGenerator()
 	if err != nil {
 		t.Fatal(err)
--- a/Show More
+++ b/Show More