Merge pull request #9273 from filecoin-project/release/v1.17.1

build: release: v1.17.1

.circleci/config.yml

@@ -46,6 +46,35 @@ commands:
           steps:
             - run: sudo apt-get update
             - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
+      - when:
+          condition: <<parameters.darwin>>
+          steps:
+            - run:
+                name: Install Go
+                command: |
+                  curl https://dl.google.com/go/go1.17.9.darwin-amd64.pkg -o /tmp/go.pkg && \
+                  sudo installer -pkg /tmp/go.pkg -target /
+            - run:
+                name: Export Go
+                command: |
+                  echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
+            - run: go version
+            - run:
+                name: Install pkg-config, goreleaser, and sha512sum
+                command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils
+            - run:
+                name: Install Rust
+                command: |
+                  curl https://sh.rustup.rs -sSf | sh -s -- -y
+            - run:
+                name: Install hwloc
+                command: |
+                  mkdir ~/hwloc
+                  curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
+                  cd ~/hwloc
+                  tar -xvzpf hwloc-2.4.1.tar.gz
+                  cd hwloc-2.4.1
+                  ./configure && make && sudo make install
       - run: git submodule sync
       - run: git submodule update --init
   download-params:
@@ -77,13 +106,22 @@ commands:
           tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz
           mv go-ipfs/ipfs /usr/local/bin/ipfs
           chmod +x /usr/local/bin/ipfs
+  install_ipfs_macos:
+    steps:
+      - run: |
+          curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz
+          tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz
+          pushd kubo
+          sudo bash install.sh
+          popd
+          rm -rf kubo/
+          rm kubo_v0.14.0_darwin-amd64.tar.gz
   git_fetch_all_tags:
     steps:
       - run:
           name: fetch all tags
           command: |
             git fetch --all
-
   packer_build:
     description: "Run a packer build"
     parameters:
@@ -116,7 +154,7 @@ jobs:
           command: |
             git --no-pager diff go.mod go.sum
             git --no-pager diff --quiet go.mod go.sum
-  build-all:
+  build-linux:
     executor: golang
     steps:
       - install-deps
@@ -174,16 +212,6 @@ jobs:
         type: string
         default: standard-verbose
         description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
-      coverage:
-        type: string
-        default: -coverprofile=coverage.txt -coverpkg=github.com/filecoin-project/lotus/...
-        description: Coverage flag. Set to the empty string to disable.
-      codecov-upload:
-        type: boolean
-        default: true
-        description: |
-          Upload coverage report to https://codecov.io/. Requires the codecov API token to be
-          set as an environment variable for private projects.
     executor: << parameters.executor >>
     steps:
       - install-deps
@@ -197,6 +225,7 @@ jobs:
           environment:
             TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >>
             SKIP_CONFORMANCE: "1"
+            LOTUS_SRC_DIR: /home/circleci/project
           command: |
             mkdir -p /tmp/test-reports/<< parameters.suite >>
             mkdir -p /tmp/test-artifacts
@@ -205,7 +234,6 @@ jobs:
               --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
               --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \
               -- \
-              << parameters.coverage >> \
               << parameters.go-test-flags >> \
               << parameters.target >>
           no_output_timeout: 30m
@@ -213,17 +241,10 @@ jobs:
           path: /tmp/test-reports
       - store_artifacts:
           path: /tmp/test-artifacts/<< parameters.suite >>.json
-      - when:
-          condition: << parameters.codecov-upload >>
-          steps:
-            - run:
-                shell: /bin/bash -eo pipefail
-                command: |
-                  bash <(curl -s https://codecov.io/bash)
 
   test-conformance:
     description: |
-      Run tests using a corpus of interoperable test vectors for Filecoin 
+      Run tests using a corpus of interoperable test vectors for Filecoin
       implementations to test their correctness and compliance with the Filecoin
       specifications.
     parameters:
@@ -353,58 +374,42 @@ jobs:
       - run:
           name: "trigger payment channel stress testplan on taas"
           command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-
   build-macos:
     description: build darwin lotus binary
+    parameters:
+      publish:
+        default: false
+        description: publish github release and homebrew?
+        type: boolean
     macos:
-      xcode: "12.5.0"
+      xcode: "13.4.1"
     working_directory: ~/go/src/github.com/filecoin-project/lotus
     steps:
       - prepare:
           linux: false
           darwin: true
-      - run:
-          name: Install go
-          command: |
-            curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \
-            sudo installer -pkg go1.17.9.darwin-amd64.pkg -target /
-      - run:
-          name: Install pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
-      - run: go version
-      - run:
-          name: Install Rust
-          command: |
-            curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - run:
-          name: Install hwloc
-          command: |
-            mkdir ~/hwloc
-            curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-            cd ~/hwloc
-            tar -xvzpf hwloc-2.4.1.tar.gz
-            cd hwloc-2.4.1
-            ./configure && make && sudo make install
+      - install_ipfs_macos
       - restore_cache:
           name: restore cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-      - run:
-          command: make build
-          no_output_timeout: 30m
-      - run:
-          name: check tag and version output match
-          command: ./scripts/version-check.sh ./lotus
+      - when:
+          condition: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist
+            - run: ./scripts/generate-checksums.sh
+            - run: ./scripts/publish-checksums.sh
+      - when:
+          condition:
+            not: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist --snapshot
+            - run: ./scripts/generate-checksums.sh
       - store_artifacts:
-          path: lotus
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
+          path: dist
       - persist_to_workspace:
           root: "."
           paths:
-            - darwin
+            - dist
       - save_cache:
           name: save cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
@@ -536,6 +541,15 @@ jobs:
   publish:
     description: publish binary artifacts
     executor: ubuntu
+    parameters:
+      linux:
+        default: false
+        description: publish linux binaries?
+        type: boolean
+      appimage:
+        default: false
+        description: publish appimage binaries?
+        type: boolean
     steps:
       - run:
           name: Install git jq curl
@@ -546,12 +560,16 @@ jobs:
       - install_ipfs
       - attach_workspace:
           at: "."
-      - run:
-          name: Create bundles
-          command: ./scripts/build-bundle.sh
-      - run:
-          name: Publish release
-          command: ./scripts/publish-release.sh
+      - when:
+          condition: << parameters.linux >>
+          steps:
+            - run: ./scripts/build-arch-bundle.sh linux
+            - run: ./scripts/publish-arch-release.sh linux
+      - when:
+          condition: << parameters.appimage >>
+          steps:
+            - run: ./scripts/build-appimage-bundle.sh
+            - run: ./scripts/publish-arch-release.sh appimage
 
   publish-snapcraft:
     description: build and push snapcraft
@@ -563,22 +581,29 @@ jobs:
         type: string
         default: "edge"
         description: snapcraft channel
+      snap-name:
+        type: string
+        default: 'lotus-filecoin'
+        description: name of snap in snap store
     steps:
       - checkout
       - run:
-          name: install snapcraft
+          name: Install snapcraft
           command: sudo snap install snapcraft --classic
       - run:
-          name: create snapcraft config file
+          name: Build << parameters.snap-name >> snap
           command: |
-            mkdir -p ~/.config/snapcraft
-            echo "$SNAPCRAFT_LOGIN_FILE" | base64 -d > ~/.config/snapcraft/snapcraft.cfg
+            if [ "<< parameters.snap-name >>" != 'lotus-filecoin' ]; then
+              cat snap/snapcraft.yaml | sed 's/lotus-filecoin/lotus/' > edited-snapcraft.yaml
+              mv edited-snapcraft.yaml snap/snapcraft.yaml
+            fi
+
+            snapcraft --use-lxd --debug
       - run:
-          name: build snap
-          command: snapcraft --use-lxd
-      - run:
-          name: publish snap
-          command: snapcraft push *.snap --release << parameters.channel >>
+          name: Publish snap to << parameters.channel >> channel
+          shell: /bin/bash -o pipefail
+          command: |
+            snapcraft upload *.snap --release << parameters.channel >>
 
   build-and-push-image:
     description: build and push docker images to public AWS ECR registry
@@ -730,26 +755,32 @@ jobs:
           name: docker build
           command: |
             docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus .
+            docker build --target lotus-gateway -t filecoin/lotus-gateway:<< parameters.tag >> -f Dockerfile.lotus .
             docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus .
             if [[ ! -z $CIRCLE_SHA1 ]]; then
               docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus .
+              docker build --target lotus-gateway -t filecoin/lotus-gateway:$CIRCLE_SHA1 -f Dockerfile.lotus .
               docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus .
             fi
             if [[ ! -z $CIRCLE_TAG ]]; then
               docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus .
+              docker build --target lotus-gateway -t filecoin/lotus-gateway:$CIRCLE_TAG -f Dockerfile.lotus .
               docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus .
             fi
       - run:
           name: docker push
           command: |
             docker push filecoin/lotus:<< parameters.tag >>
+            docker push filecoin/lotus-gateway:<< parameters.tag >>
             docker push filecoin/lotus-all-in-one:<< parameters.tag >>
             if [[ ! -z $CIRCLE_SHA1 ]]; then
               docker push filecoin/lotus:$CIRCLE_SHA1
+              docker push filecoin/lotus-gateway:$CIRCLE_SHA1
               docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1
             fi
             if [[ ! -z $CIRCLE_TAG ]]; then
               docker push filecoin/lotus:$CIRCLE_TAG
+              docker push filecoin/lotus-gateway:$CIRCLE_TAG
               docker push filecoin/lotus-all-in-one:$CIRCLE_TAG
             fi
 
@@ -803,6 +834,11 @@ workflows:
           suite: itest-deals_concurrent
           target: "./itests/deals_concurrent_test.go"
       
+      - test:
+          name: test-itest-deals_invalid_utf8_label
+          suite: itest-deals_invalid_utf8_label
+          target: "./itests/deals_invalid_utf8_label_test.go"
+      
       - test:
           name: test-itest-deals_max_staging_deals
           suite: itest-deals_max_staging_deals
@@ -853,6 +889,11 @@ workflows:
           suite: itest-deals
           target: "./itests/deals_test.go"
       
+      - test:
+          name: test-itest-gas_estimation
+          suite: itest-gas_estimation
+          target: "./itests/gas_estimation_test.go"
+      
       - test:
           name: test-itest-gateway
           suite: itest-gateway
@@ -863,6 +904,11 @@ workflows:
           suite: itest-get_messages_in_ts
           target: "./itests/get_messages_in_ts_test.go"
       
+      - test:
+          name: test-itest-lite_migration
+          suite: itest-lite_migration
+          target: "./itests/lite_migration_test.go"
+      
       - test:
           name: test-itest-lookup_robust_address
           suite: itest-lookup_robust_address
@@ -878,11 +924,26 @@ workflows:
           suite: itest-multisig
           target: "./itests/multisig_test.go"
       
+      - test:
+          name: test-itest-net
+          suite: itest-net
+          target: "./itests/net_test.go"
+      
       - test:
           name: test-itest-nonce
           suite: itest-nonce
           target: "./itests/nonce_test.go"
       
+      - test:
+          name: test-itest-path_detach_redeclare
+          suite: itest-path_detach_redeclare
+          target: "./itests/path_detach_redeclare_test.go"
+      
+      - test:
+          name: test-itest-path_type_filters
+          suite: itest-path_type_filters
+          target: "./itests/path_type_filters_test.go"
+      
       - test:
           name: test-itest-paych_api
           suite: itest-paych_api
@@ -938,6 +999,11 @@ workflows:
           suite: itest-self_sent_txn
           target: "./itests/self_sent_txn_test.go"
       
+      - test:
+          name: test-itest-splitstore
+          suite: itest-splitstore
+          target: "./itests/splitstore_test.go"
+      
       - test:
           name: test-itest-tape
           suite: itest-tape
@@ -948,16 +1014,31 @@ workflows:
           suite: itest-verifreg
           target: "./itests/verifreg_test.go"
       
+      - test:
+          name: test-itest-wdpost_config
+          suite: itest-wdpost_config
+          target: "./itests/wdpost_config_test.go"
+      
       - test:
           name: test-itest-wdpost_dispute
           suite: itest-wdpost_dispute
           target: "./itests/wdpost_dispute_test.go"
       
+      - test:
+          name: test-itest-wdpost_no_miner_storage
+          suite: itest-wdpost_no_miner_storage
+          target: "./itests/wdpost_no_miner_storage_test.go"
+      
       - test:
           name: test-itest-wdpost
           suite: itest-wdpost
           target: "./itests/wdpost_test.go"
       
+      - test:
+          name: test-itest-wdpost_worker_config
+          suite: itest-wdpost_worker_config
+          target: "./itests/wdpost_worker_config_test.go"
+      
       - test:
           name: test-itest-worker
           suite: itest-worker
@@ -982,15 +1063,13 @@ workflows:
       - test:
           go-test-flags: "-run=TestMulticoreSDR"
           suite: multicore-sdr-check
-          target: "./extern/sector-storage/ffiwrapper"
+          target: "./storage/sealer/ffiwrapper"
           proofs-log-test: "1"
       - test-conformance:
           suite: conformance
-          codecov-upload: false
           target: "./conformance"
       - test-conformance:
           name: test-conformance-bleeding-edge
-          codecov-upload: false
           suite: conformance-bleeding-edge
           target: "./conformance"
           vectors-branch: specs-actors-v7
@@ -1000,7 +1079,7 @@ workflows:
               only:
                 - master
       - build-debug
-      - build-all:
+      - build-linux:
           filters:
             tags:
               only:
@@ -1017,11 +1096,33 @@ workflows:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-lotus-soup
       - build-macos:
+          name: publish-macos
+          publish: true
           filters:
+            branches:
+              ignore:
+                - /.*/
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-macos:
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-appimage:
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - publish:
+          name: publish-linux
+          linux: true
+          requires:
+            - build-linux
           filters:
             branches:
               ignore:
@@ -1030,9 +1131,9 @@ workflows:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - publish:
+          name: publish-appimage
+          appimage: true
           requires:
-            - build-all
-            - build-macos
             - build-appimage
           filters:
             branches:
@@ -1042,27 +1143,63 @@ workflows:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-and-push-image:
+          name: build-and-push/lotus-all-in-one
           dockerfile: Dockerfile.lotus
           path: .
           repo: lotus-dev
           tag: '${CIRCLE_SHA1:0:8}'
           target: lotus-all-in-one
       - build-and-push-image:
+          name: build-and-push/lotus-test
           dockerfile: Dockerfile.lotus
           path: .
           repo: lotus-test
           tag: '${CIRCLE_SHA1:0:8}'
           target: lotus-test
       - publish-snapcraft:
-          name: publish-snapcraft-stable
+          name: "Publish Snapcraft (lotus-filecoin / candidate)"
           channel: stable
+          snap-name: lotus-filecoin
           filters:
             branches:
               ignore:
                 - /.*/
             tags:
               only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+                - /^v\d+\.\d+\.\d+$/
+      - publish-snapcraft:
+          name: "Publish Snapcraft (lotus-filecoin / candidate)"
+          channel: candidate
+          snap-name: lotus-filecoin
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+-rc\d+$/
+      - publish-snapcraft:
+          name: "Publish Snapcraft (lotus / stable)"
+          channel: stable
+          snap-name: lotus
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+$/
+      - publish-snapcraft:
+          name: "Publish Snapcraft (lotus / candidate)"
+          channel: candidate
+          snap-name: lotus
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+-rc\d+$/
       - publish-dockerhub:
           name: publish-dockerhub
           tag: stable
@@ -1084,8 +1221,13 @@ workflows:
                 - master
     jobs:
       - publish-snapcraft:
-          name: publish-snapcraft-nightly
+          name: "Publish Snapcraft Nightly (lotus-filecoin / edge)"
           channel: edge
+          snap-name: lotus-filecoin
+      - publish-snapcraft:
+          name: "Publish Snapcraft Nightly (lotus / edge)"
+          channel: edge
+          snap-name: lotus
       - publish-dockerhub:
           name: publish-dockerhub-nightly
           tag: nightly
@@ -1098,4 +1240,4 @@ workflows:
               only:
                 - master
     jobs:
-      - publish-packer-snap
+      - publish-packer-snap

.circleci/template.yml

@@ -46,6 +46,35 @@ commands:
           steps:
             - run: sudo apt-get update
             - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
+      - when:
+          condition: <<parameters.darwin>>
+          steps:
+            - run:
+                name: Install Go
+                command: |
+                  curl https://dl.google.com/go/go1.17.9.darwin-amd64.pkg -o /tmp/go.pkg && \
+                  sudo installer -pkg /tmp/go.pkg -target /
+            - run:
+                name: Export Go
+                command: |
+                  echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
+            - run: go version
+            - run:
+                name: Install pkg-config, goreleaser, and sha512sum
+                command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils
+            - run:
+                name: Install Rust
+                command: |
+                  curl https://sh.rustup.rs -sSf | sh -s -- -y
+            - run:
+                name: Install hwloc
+                command: |
+                  mkdir ~/hwloc
+                  curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
+                  cd ~/hwloc
+                  tar -xvzpf hwloc-2.4.1.tar.gz
+                  cd hwloc-2.4.1
+                  ./configure && make && sudo make install
       - run: git submodule sync
       - run: git submodule update --init
   download-params:
@@ -77,13 +106,22 @@ commands:
           tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz
           mv go-ipfs/ipfs /usr/local/bin/ipfs
           chmod +x /usr/local/bin/ipfs
+  install_ipfs_macos:
+    steps:
+      - run: |
+          curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz
+          tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz
+          pushd kubo
+          sudo bash install.sh
+          popd
+          rm -rf kubo/
+          rm kubo_v0.14.0_darwin-amd64.tar.gz
   git_fetch_all_tags:
     steps:
       - run:
           name: fetch all tags
           command: |
             git fetch --all
-
   packer_build:
     description: "Run a packer build"
     parameters:
@@ -116,7 +154,7 @@ jobs:
           command: |
             git --no-pager diff go.mod go.sum
             git --no-pager diff --quiet go.mod go.sum
-  build-all:
+  build-linux:
     executor: golang
     steps:
       - install-deps
@@ -174,16 +212,6 @@ jobs:
         type: string
         default: standard-verbose
         description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
-      coverage:
-        type: string
-        default: -coverprofile=coverage.txt -coverpkg=github.com/filecoin-project/lotus/...
-        description: Coverage flag. Set to the empty string to disable.
-      codecov-upload:
-        type: boolean
-        default: true
-        description: |
-          Upload coverage report to https://codecov.io/. Requires the codecov API token to be
-          set as an environment variable for private projects.
     executor: << parameters.executor >>
     steps:
       - install-deps
@@ -197,6 +225,7 @@ jobs:
           environment:
             TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >>
             SKIP_CONFORMANCE: "1"
+            LOTUS_SRC_DIR: /home/circleci/project
           command: |
             mkdir -p /tmp/test-reports/<< parameters.suite >>
             mkdir -p /tmp/test-artifacts
@@ -205,7 +234,6 @@ jobs:
               --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
               --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \
               -- \
-              << parameters.coverage >> \
               << parameters.go-test-flags >> \
               << parameters.target >>
           no_output_timeout: 30m
@@ -213,17 +241,10 @@ jobs:
           path: /tmp/test-reports
       - store_artifacts:
           path: /tmp/test-artifacts/<< parameters.suite >>.json
-      - when:
-          condition: << parameters.codecov-upload >>
-          steps:
-            - run:
-                shell: /bin/bash -eo pipefail
-                command: |
-                  bash <(curl -s https://codecov.io/bash)
 
   test-conformance:
     description: |
-      Run tests using a corpus of interoperable test vectors for Filecoin 
+      Run tests using a corpus of interoperable test vectors for Filecoin
       implementations to test their correctness and compliance with the Filecoin
       specifications.
     parameters:
@@ -353,58 +374,42 @@ jobs:
       - run:
           name: "trigger payment channel stress testplan on taas"
           command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-
   build-macos:
     description: build darwin lotus binary
+    parameters:
+      publish:
+        default: false
+        description: publish github release and homebrew?
+        type: boolean
     macos:
-      xcode: "12.5.0"
+      xcode: "13.4.1"
     working_directory: ~/go/src/github.com/filecoin-project/lotus
     steps:
       - prepare:
           linux: false
           darwin: true
-      - run:
-          name: Install go
-          command: |
-            curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \
-            sudo installer -pkg go1.17.9.darwin-amd64.pkg -target /
-      - run:
-          name: Install pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
-      - run: go version
-      - run:
-          name: Install Rust
-          command: |
-            curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - run:
-          name: Install hwloc
-          command: |
-            mkdir ~/hwloc
-            curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-            cd ~/hwloc
-            tar -xvzpf hwloc-2.4.1.tar.gz
-            cd hwloc-2.4.1
-            ./configure && make && sudo make install
+      - install_ipfs_macos
       - restore_cache:
           name: restore cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-      - run:
-          command: make build
-          no_output_timeout: 30m
-      - run:
-          name: check tag and version output match
-          command: ./scripts/version-check.sh ./lotus
+      - when:
+          condition: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist
+            - run: ./scripts/generate-checksums.sh
+            - run: ./scripts/publish-checksums.sh
+      - when:
+          condition:
+            not: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist --snapshot
+            - run: ./scripts/generate-checksums.sh
       - store_artifacts:
-          path: lotus
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
+          path: dist
       - persist_to_workspace:
           root: "."
           paths:
-            - darwin
+            - dist
       - save_cache:
           name: save cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
@@ -536,6 +541,15 @@ jobs:
   publish:
     description: publish binary artifacts
     executor: ubuntu
+    parameters:
+      linux:
+        default: false
+        description: publish linux binaries?
+        type: boolean
+      appimage:
+        default: false
+        description: publish appimage binaries?
+        type: boolean
     steps:
       - run:
           name: Install git jq curl
@@ -546,12 +560,16 @@ jobs:
       - install_ipfs
       - attach_workspace:
           at: "."
-      - run:
-          name: Create bundles
-          command: ./scripts/build-bundle.sh
-      - run:
-          name: Publish release
-          command: ./scripts/publish-release.sh
+      - when:
+          condition: << parameters.linux >>
+          steps:
+            - run: ./scripts/build-arch-bundle.sh linux
+            - run: ./scripts/publish-arch-release.sh linux
+      - when:
+          condition: << parameters.appimage >>
+          steps:
+            - run: ./scripts/build-appimage-bundle.sh
+            - run: ./scripts/publish-arch-release.sh appimage
 
   publish-snapcraft:
     description: build and push snapcraft
@@ -563,22 +581,29 @@ jobs:
         type: string
         default: "edge"
         description: snapcraft channel
+      snap-name:
+        type: string
+        default: 'lotus-filecoin'
+        description: name of snap in snap store
     steps:
       - checkout
       - run:
-          name: install snapcraft
+          name: Install snapcraft
           command: sudo snap install snapcraft --classic
       - run:
-          name: create snapcraft config file
+          name: Build << parameters.snap-name >> snap
           command: |
-            mkdir -p ~/.config/snapcraft
-            echo "$SNAPCRAFT_LOGIN_FILE" | base64 -d > ~/.config/snapcraft/snapcraft.cfg
+            if [ "<< parameters.snap-name >>" != 'lotus-filecoin' ]; then
+              cat snap/snapcraft.yaml | sed 's/lotus-filecoin/lotus/' > edited-snapcraft.yaml
+              mv edited-snapcraft.yaml snap/snapcraft.yaml
+            fi
+
+            snapcraft --use-lxd --debug
       - run:
-          name: build snap
-          command: snapcraft --use-lxd
-      - run:
-          name: publish snap
-          command: snapcraft push *.snap --release << parameters.channel >>
+          name: Publish snap to << parameters.channel >> channel
+          shell: /bin/bash -o pipefail
+          command: |
+            snapcraft upload *.snap --release << parameters.channel >>
 
   build-and-push-image:
     description: build and push docker images to public AWS ECR registry
@@ -730,26 +755,32 @@ jobs:
           name: docker build
           command: |
             docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus .
+            docker build --target lotus-gateway -t filecoin/lotus-gateway:<< parameters.tag >> -f Dockerfile.lotus .
             docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus .
             if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then
               docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus .
+              docker build --target lotus-gateway -t filecoin/lotus-gateway:$CIRCLE_SHA1 -f Dockerfile.lotus .
               docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus .
             fi
             if [["[[ ! -z $CIRCLE_TAG ]]"]]; then
               docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus .
+              docker build --target lotus-gateway -t filecoin/lotus-gateway:$CIRCLE_TAG -f Dockerfile.lotus .
               docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus .
             fi
       - run:
           name: docker push
           command: |
             docker push filecoin/lotus:<< parameters.tag >>
+            docker push filecoin/lotus-gateway:<< parameters.tag >>
             docker push filecoin/lotus-all-in-one:<< parameters.tag >>
             if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then
               docker push filecoin/lotus:$CIRCLE_SHA1
+              docker push filecoin/lotus-gateway:$CIRCLE_SHA1
               docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1
             fi
             if [["[[ ! -z $CIRCLE_TAG ]]"]]; then
               docker push filecoin/lotus:$CIRCLE_TAG
+              docker push filecoin/lotus-gateway:$CIRCLE_TAG
               docker push filecoin/lotus-all-in-one:$CIRCLE_TAG
             fi
 
@@ -782,15 +813,13 @@ workflows:
       - test:
           go-test-flags: "-run=TestMulticoreSDR"
           suite: multicore-sdr-check
-          target: "./extern/sector-storage/ffiwrapper"
+          target: "./storage/sealer/ffiwrapper"
           proofs-log-test: "1"
       - test-conformance:
           suite: conformance
-          codecov-upload: false
           target: "./conformance"
       - test-conformance:
           name: test-conformance-bleeding-edge
-          codecov-upload: false
           suite: conformance-bleeding-edge
           target: "./conformance"
           vectors-branch: specs-actors-v7
@@ -800,7 +829,7 @@ workflows:
               only:
                 - master
       - build-debug
-      - build-all:
+      - build-linux:
           filters:
             tags:
               only:
@@ -817,11 +846,33 @@ workflows:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-lotus-soup
       - build-macos:
+          name: publish-macos
+          publish: true
           filters:
+            branches:
+              ignore:
+                - /.*/
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-macos:
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-appimage:
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - publish:
+          name: publish-linux
+          linux: true
+          requires:
+            - build-linux
           filters:
             branches:
               ignore:
@@ -830,9 +881,9 @@ workflows:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - publish:
+          name: publish-appimage
+          appimage: true
           requires:
-            - build-all
-            - build-macos
             - build-appimage
           filters:
             branches:
@@ -842,27 +893,63 @@ workflows:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-and-push-image:
+          name: build-and-push/lotus-all-in-one
           dockerfile: Dockerfile.lotus
           path: .
           repo: lotus-dev
           tag: '${CIRCLE_SHA1:0:8}'
           target: lotus-all-in-one
       - build-and-push-image:
+          name: build-and-push/lotus-test
           dockerfile: Dockerfile.lotus
           path: .
           repo: lotus-test
           tag: '${CIRCLE_SHA1:0:8}'
           target: lotus-test
       - publish-snapcraft:
-          name: publish-snapcraft-stable
+          name: "Publish Snapcraft (lotus-filecoin / candidate)"
           channel: stable
+          snap-name: lotus-filecoin
           filters:
             branches:
               ignore:
                 - /.*/
             tags:
               only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+                - /^v\d+\.\d+\.\d+$/
+      - publish-snapcraft:
+          name: "Publish Snapcraft (lotus-filecoin / candidate)"
+          channel: candidate
+          snap-name: lotus-filecoin
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+-rc\d+$/
+      - publish-snapcraft:
+          name: "Publish Snapcraft (lotus / stable)"
+          channel: stable
+          snap-name: lotus
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+$/
+      - publish-snapcraft:
+          name: "Publish Snapcraft (lotus / candidate)"
+          channel: candidate
+          snap-name: lotus
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+-rc\d+$/
       - publish-dockerhub:
           name: publish-dockerhub
           tag: stable
@@ -884,8 +971,13 @@ workflows:
                 - master
     jobs:
       - publish-snapcraft:
-          name: publish-snapcraft-nightly
+          name: "Publish Snapcraft Nightly (lotus-filecoin / edge)"
           channel: edge
+          snap-name: lotus-filecoin
+      - publish-snapcraft:
+          name: "Publish Snapcraft Nightly (lotus / edge)"
+          channel: edge
+          snap-name: lotus
       - publish-dockerhub:
           name: publish-dockerhub-nightly
           tag: nightly
@@ -898,4 +990,4 @@ workflows:
               only:
                 - master
     jobs:
-      - publish-packer-snap
+      - publish-packer-snap

.codecov.yml

@@ -1,74 +0,0 @@
-ignore:
-  # Auto generated
-  - "^.*_gen.go$"
-  - "^.*/mock_full.go$"
-  # Old actors.
-  - "^chain/actors/builtin/[^/]*/(message|state|v)[0-4]\\.go$" # We test the latest version only.
-  # Tests
-  - "api/test/**"
-  - "conformance/**"
-  # Generators
-  - "gen/**"
-  - "chain/actors/agen/**"
-  # Non-critical utilities
-  - "api/docgen/**"
-  - "api/docgen-openrpc/**"
-coverage:
-  status:
-    patch: off
-    project:
-      threshold: 1%
-      tools-and-tests:
-        target: auto
-        threshold: 1%
-        informational: true
-        paths:
-          - "testplans"
-          - "tools"
-          - "system"
-          - "snap"
-          - "lotuspond"
-          - "conformance"
-          - "scripts"
-          - "gen"
-          - "build"
-      markets: 
-        target: auto
-        threshold: 1%
-        informational: true        
-        paths: 
-          - "markets"
-          - "paychmgr"
-      miner: 
-        target: auto
-        threshold: 1.5%
-        informational: true
-        paths: 
-          - "miner"
-          - "storage"
-      chain:
-        target: auto
-        threshold: 1%
-        informational: true
-        paths:
-          - "chain"
-      node:
-        target: auto
-        threshold: 1%
-        informational: true
-        paths: 
-          - "node"
-          - "blockstore"
-          - "metrics"
-          - "lib"
-          - "genesis"
-          - "gateway"
-          - "api"
-          - "journal"
-      cli: 
-        target: auto
-        threshold: 1%
-        informational: true
-        paths: 
-          - "cli"
-          - "cmd"

.github/workflows/testground-on-push.yml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: testground run
-        uses: coryschwartz/testground-github-action@v1.1
+        uses: testground/testground-github-action@v1
         with:
           backend_addr: ${{ matrix.backend_addr }}
           backend_proto: ${{ matrix.backend_proto }}

.gitignore

@@ -41,7 +41,6 @@ build/paramfetch.sh
 /darwin
 /linux
 *.snap
-build/builtin-actors
 
 *-fuzz.zip
 /chain/types/work_msg/
@@ -49,3 +48,8 @@ bin/ipget
 bin/tmp/*
 .idea
 scratchpad
+
+build/builtin-actors/v*
+build/builtin-actors/*.car
+
+dist/

.goreleaser.yaml

@@ -0,0 +1,165 @@
+project_name: lotus
+before:
+  hooks:
+    - go mod tidy
+    - make deps
+
+universal_binaries:
+  - id: lotus
+    replace: true
+    name_template: lotus
+    ids:
+      - lotus_darwin_amd64
+      - lotus_darwin_arm64
+  - id: lotus-miner
+    replace: true
+    name_template: lotus-miner
+    ids:
+      - lotus-miner_darwin_amd64
+      - lotus-miner_darwin_arm64
+  - id: lotus-worker
+    replace: true
+    name_template: lotus-worker
+    ids:
+      - lotus-worker_darwin_amd64
+      - lotus-worker_darwin_arm64
+
+builds:
+  - id: lotus_darwin_amd64
+    main: ./cmd/lotus
+    binary: lotus
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-miner_darwin_amd64
+    main: ./cmd/lotus-miner
+    binary: lotus-miner
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-worker_darwin_amd64
+    main: ./cmd/lotus-worker
+    binary: lotus-worker
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus_darwin_arm64
+    main: ./cmd/lotus
+    binary: lotus
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-miner_darwin_arm64
+    main: ./cmd/lotus-miner
+    binary: lotus-miner
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-worker_darwin_arm64
+    main: ./cmd/lotus-worker
+    binary: lotus-worker
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+#  - id: linux
+#    main: ./cmd/lotus
+#    binary: lotus
+#    goos:
+#      - linux
+#    goarch:
+#      - amd64
+#    env:
+#      - CGO_ENABLED=1
+#    ldflags:
+#      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+
+archives:
+  - id: primary
+    format: tar.gz
+    wrap_in_directory: true
+    files:
+      # this is a dumb but required hack so it doesn't include the default files
+      # https://github.com/goreleaser/goreleaser/issues/602
+      - _n_o_n_e_*
+
+release:
+  github:
+    owner: filecoin-project
+    name: lotus
+  prerelease: auto
+  name_template: "Release v{{.Version}}"
+
+
+brews:
+  - tap:
+      owner: filecoin-project
+      name: homebrew-lotus
+      branch: master
+    ids:
+      - primary
+    install: |
+      bin.install "lotus"
+      bin.install "lotus-miner"
+      bin.install "lotus-worker"
+    test: |
+      system "#{bin}/lotus --version"
+      system "#{bin}/lotus-miner --version"
+      system "#{bin}/lotus-worker --version"
+    folder: Formula
+    homepage: "https://filecoin.io"
+    description: "A homebrew cask for installing filecoin-project/lotus on MacOS"
+    license: MIT
+    dependencies:
+      - name: pkg-config
+      - name: jq
+      - name: bzr
+      - name: hwloc
+
+# produced manually so we can include cid checksums
+checksum:
+  disable: true
+
+snapshot:
+  name_template: "{{ .Tag }}"

AppImageBuilder.yml

@@ -68,6 +68,4 @@ AppDir:
       use_host_x: false
 AppImage:
   arch: x86_64
-  update-information: guess
-  sign-key: None
 

CHANGELOG.md

@@ -1,5 +1,399 @@
 # Lotus changelog
 
+# v1.17.1 / 2022-09-06
+
+This is an optional release of Lotus. This release introduces the [Splitstore v2 - beta](https://github.com/filecoin-project/lotus/blob/master/blockstore/splitstore/README.md)(beta). Splitstore aims to reduce the node performance impact that's caused by the Filecoin's very large, and continuously growing datastore. Splitstore v2 introduces  the coldstore auto prune/GC feature & some improvements for the hotstore.  We welcome all lotus users to join the early testers and try the new Splitstore out, you can leave any feedback or report issues in [this discussion](https://github.com/filecoin-project/lotus/discussions/9179) or create an issue. As always,  multiple small bug fixes, new features & improvements are also included in this release.
+
+
+## New features
+
+- feat:chain:splitstore auto prune ([filecoin-project/lotus#9123](https://github.com/filecoin-project/lotus/pull/9123))
+  - Trigger SplitStore chain prune on head events. [Link to the documentation](https://lotus.filecoin.io/lotus/manage/chain-management/#cold-store-garbage-collection)
+- feat:chain:splitstore chain prune ([filecoin-project/lotus#9056](https://github.com/filecoin-project/lotus/pull/9056))
+  - Adds `chain prune` command to trigger manual garbage collection. [Link to the documentation](https://lotus.filecoin.io/lotus/manage/chain-management/#cold-store-garbage-collection)
+- feat: storage: Path type filters ([filecoin-project/lotus#9013](https://github.com/filecoin-project/lotus/pull/9013))
+  - Adds new fields to `sectorstore.json` to allow file type filtering. [Link to the documentation](https://lotus.filecoin.io/storage-providers/operate/custom-storage-layout/#filter-sector-types-1)
+- feat: sealing: storage redeclare/detach ([filecoin-project/lotus#9032](https://github.com/filecoin-project/lotus/pull/9032))
+  - Adds new Lotus commands to detach and redeclare storage paths. [Link to the documentation](https://lotus.filecoin.io/storage-providers/operate/custom-storage-layout/#detach-storage-paths)
+- feat: worker: Add stop cmd for lotus worker ([filecoin-project/lotus#9101](https://github.com/filecoin-project/lotus/pull/9101))
+  - Adds new `lotus-worker stop` command. [Link to the documentation](https://lotus.filecoin.io/storage-providers/seal-workers/seal-workers/#stop-the-worker)
+- feat: market: Add lotus-shed cmd to get total active deal storage ([filecoin-project/lotus#9113](https://github.com/filecoin-project/lotus/pull/9113))
+  - `get-deals-total-storage` - View the total storage available in all active market deals
+- feat: wdpost: Envvar for limiting recovering sectors ([filecoin-project/lotus#9106](https://github.com/filecoin-project/lotus/pull/9106))
+  - Adds new envvar to limit the number of sectors declared in the recover message
+
+
+## Improvements
+
+- feat: sealing: Allow overriding worker hostname ([filecoin-project/lotus#9116](https://github.com/filecoin-project/lotus/pull/9116))
+- feat: build: run fiximports on make actors-gen ([filecoin-project/lotus#9114](https://github.com/filecoin-project/lotus/pull/9114))
+- feat: FVM: always enable tracing for user-triggered executions ([filecoin-project/lotus#9036](https://github.com/filecoin-project/lotus/pull/9036))
+- feat: miner cli: proving deadline command enchantments ([filecoin-project/lotus#9109](https://github.com/filecoin-project/lotus/pull/9109))
+- FVM: Use MaxInt64 for Implicit Message gas limits ([filecoin-project/lotus#9037](https://github.com/filecoin-project/lotus/pull/9037))
+- lotus shed addr decode
+- push lotus-gateway to docker hub ([filecoin-project/lotus#8969](https://github.com/filecoin-project/lotus/pull/8969))
+- Review Response
+- test: net: net and conngater tests ([filecoin-project/lotus#8084](https://github.com/filecoin-project/lotus/pull/8084))
+- Update FFI ([filecoin-project/lotus#9139](https://github.com/filecoin-project/lotus/pull/9139))
+
+## Bug Fixes
+
+- backport: 9153: detach storage on worker shutdown ([filecoin-project/lotus#9127](https://github.com/filecoin-project/lotus/pull/9165))
+- fix makegen
+- fix: build: use GOCC when building lotus-fountain ([filecoin-project/lotus#9127](https://github.com/filecoin-project/lotus/pull/9127))
+- fix: ci: Forgot a .sh on the end of a the new publish script ([filecoin-project/lotus#9088](https://github.com/filecoin-project/lotus/pull/9088))
+- fix: cli: ./lotus-miner actor control list, if the owner is not account ([filecoin-project/lotus#9072](https://github.com/filecoin-project/lotus/pull/9072))
+- fix: deps: update FFI to fix a slow memory leak ([filecoin-project/lotus#9042](https://github.com/filecoin-project/lotus/pull/9042))
+- fix: FVM: record message applied metrics ([filecoin-project/lotus#9052](https://github.com/filecoin-project/lotus/pull/9052))
+- fix: gas: estimate gas with a zero base-fee ([filecoin-project/lotus#8991](https://github.com/filecoin-project/lotus/pull/8991))
+- fix: post: restrict recoveries per deadline ([filecoin-project/lotus#9111](https://github.com/filecoin-project/lotus/pull/9111))
+- fix: sealing: Workaround for sealing bug ([filecoin-project/lotus#9043](https://github.com/filecoin-project/lotus/pull/9043))
+- fix: storage: don't panic in getCommitCutoff when precommit is not found ([filecoin-project/lotus#9141](https://github.com/filecoin-project/lotus/pull/9141))
+- fix: test: deflake TestQuotePriceForUnsealedRetrieval ([filecoin-project/lotus#9084](https://github.com/filecoin-project/lotus/pull/9084))
+
+## Dependency Updates
+
+- github.com/multiformats/go-multibase (v0.0.3 -> v0.1.1)
+
+## Others
+
+- chore: ci: Update xcode version for macos builds ([filecoin-project/lotus#9164)](https://github.com/filecoin-project/lotus/pull/9164))
+- Merge branch 'docs/ysrotciv-desc'
+- Merge branch 'feat/f8-worker-env'
+- Merge branch 'LexLuthr-feat/minerWithdrawBalanceAPI'
+- Merge branch 'LexLuthr-feat/SchedRemoveRequest'
+- base256emoji ([filecoin-project/lotus#9038)](https://github.com/filecoin-project/lotus/pull/9038))
+- chore: interop: update interop assets ([filecoin-project/lotus#9093)](https://github.com/filecoin-project/lotus/pull/9093))
+- chore: merge: releases (v1.17.0) to master ([filecoin-project/lotus#9146)](https://github.com/filecoin-project/lotus/pull/9146))
+- chore: sealer: Fixup typos ([filecoin-project/lotus#9040)](https://github.com/filecoin-project/lotus/pull/9040))
+- chore:docs:remove readme reference to deprecated specs-actors ([filecoin-project/lotus#8984)](https://github.com/filecoin-project/lotus/pull/8984))
+- ci : Change default shell options for snapcraft publish ([filecoin-project/lotus#9122)](https://github.com/filecoin-project/lotus/pull/9122))
+- ci: More tweaks to snapcraft release process ([filecoin-project/lotus#9090)](https://github.com/filecoin-project/lotus/pull/9090))
+- ci: Publish to both lotus and lotus-filecoin for snap ([filecoin-project/lotus#9119)](https://github.com/filecoin-project/lotus/pull/9119))
+- ci: Run snap builds for lotus and lotus-filecoin in parallel ([filecoin-project/lotus#9133)](https://github.com/filecoin-project/lotus/pull/9133))
+- ci: Switches goreleaser notes back to default (keep-existing) ([filecoin-project/lotus#9120)](https://github.com/filecoin-project/lotus/pull/9120))
+- ci: update snapcraft and release flow logic ([filecoin-project/lotus#8994)](https://github.com/filecoin-project/lotus/pull/8994))
+- ci: Use goreleaser to build macos universal binaries (including M1 macs) ([filecoin-project/lotus#9096)](https://github.com/filecoin-project/lotus/pull/9096))
+- ci:testing:remove codecov ([filecoin-project/lotus#9062)](https://github.com/filecoin-project/lotus/pull/9062))
+
+
+Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| Łukasz Magiera | 34 | +2329/-317 | 163 |
+| ZenGround0 | 2 | +1527/-89 | 38 |
+| Ian Davis | 14 | +751/-232 | 30 |
+| LexLuthr | 17 | +480/-225 | 63 |
+| TheMenko | 4 | +323/-61 | 5 |
+| Aayush | 10 | +285/-92 | 30 |
+| beck | 3 | +143/-93 | 3 |
+| Steven Allen | 4 | +95/-75 | 9 |
+| zenground0 | 5 | +44/-116 | 9 |
+| Shrenuj Bansal | 7 | +136/-7 | 16 |
+| Patrick Deuse | 3 | +76/-57 | 3 |
+| Jennifer Wang | 3 | +6/-52 | 11 |
+| zl | 2 | +20/-16 | 2 |
+| Aayush Rajasekaran | 2 | +6/-6 | 2 |
+| Clint Armstrong | 1 | +7/-3 | 1 |
+| Cory Schwartz | 2 | +9/-0 | 2 |
+| Jorropo | 1 | +3/-2 | 2 |
+| Geoff Stuart | 1 | +5/-0 | 1 |
+| Frank Y | 1 | +2/-2 | 2 |
+| Aloxaf | 1 | +2/-2 | 1 |
+
+
+# v1.17.0 / 2022-08-02
+
+This is an optional release of Lotus. This feature release introduces a lot of new sealing and scheduler improvements, and many other functionalities and bug fixes.
+
+PSA: Markets related features, enhancements and fixes is now lower priority for Lotus, and is going to be in the hands of [Boost](https://boost.filecoin.io), built by the amazing [Bedrock team](https://www.notion.so/pl-strflt/Bedrock-2e956d5d8143432080a1d84435ccf0ff). You can find Lotus mission scope [here](https://www.notion.so/Lotus-8352bbb6c321431abd8790a7a3401ed3#805c645f592840ad893c272723362d3d)
+
+## New features
+
+- feat: worker: lotus-worker run --no-default ([filecoin-project/lotus#8672](https://github.com/filecoin-project/lotus/pull/8672))
+  - Makes it very easy to spin up workers that make the compute tasks opt-in, instead of the default opt-out. Also makes it very easy to create storage-only workers. [Link to the documentation](https://lotus.filecoin.io/storage-providers/seal-workers/seal-workers/#run-the-worker)
+- feat: sched: Per worker concurrent task count limits ([filecoin-project/lotus#8725](https://github.com/filecoin-project/lotus/pull/8725))
+  - Set the maximum number of tasks running it parallel on workers by exporting env-variables: `[short task type]_[sector size]_MAX_CONCURRENT=[limit]`. [Link to documentation](https://lotus.filecoin.io/storage-providers/seal-workers/seal-workers/#limit-tasks-run-in-parallel)
+- feat: sched: Finalize* move selectors ([filecoin-project/lotus#8710](https://github.com/filecoin-project/lotus/pull/8710))
+  - Allows you to force all Finalize tasks to run on workers with local access to both long-term storage and the sealing path containing the sector.
+- feat: sched: Add scheduler interfaces, configurable assigner ([filecoin-project/lotus#8700](https://github.com/filecoin-project/lotus/pull/8700)) 
+  - Introduce a new simpler worker assigning logic which will attempt to assign tasks to as many workers as possible and ignore worker utilization. [Link to documentation](https://lotus.filecoin.io/storage-providers/advanced-configurations/sealing/#worker-assigning-logic)
+- feat: bench: simple sealing operations commands ([filecoin-project/lotus#8373](https://github.com/filecoin-project/lotus/pull/8373))
+  - Allows you to only test the performance of a single task. [Read the documentation](https://lotus.filecoin.io/storage-providers/operate/benchmarks/#single-task-benchmark).
+- feat: miner cli: sealing data-cid command ([filecoin-project/lotus#8715](https://github.com/filecoin-project/lotus/pull/8715))
+  - Makes it possible to compute data CIDs on lotus workers.
+- feat: precommits info ([filecoin-project/lotus#8696](https://github.com/filecoin-project/lotus/pull/8696))
+  - Check the on-chain precommit info with the `lotus-miner sectors precommits` command.
+- feat: dagstore: add dagstore register-shard command ([filecoin-project/lotus#8645](https://github.com/filecoin-project/lotus/pull/8645))
+  - If you have broken indexes in your market’s dagstore, you can try to recover it by using `lotus-miner dagstore register-shard`. [Link to documentation](https://lotus.filecoin.io/kb/dagstore-register/)
+- feat: Implement cli command for compactPartitions ([filecoin-project/lotus#8637](https://github.com/filecoin-project/lotus/pull/8637))
+  - Remove dead sectors from partitions and reduce the number of partitions used if possible with the `lotus-miner sectors compact-partitions`command. [Link to documentation](https://lotus.filecoin.io/storage-providers/operate/daily-chores/#compacting-partitions)
+- feat: recovery: Config for maximum partition count per message ([filecoin-project/lotus#8986](https://github.com/filecoin-project/lotus/pull/8986)
+  - Adds config for setting the maximum amount of partitions to declare in a DeclareFaultsRecovered message [Link to documentation](https://lotus.filecoin.io/storage-providers/seal-workers/post-workers/#multiple-partitions)
+- feat: wdpost: Config for maximum partition count per message ([filecoin-project/lotus#8982](https://github.com/filecoin-project/lotus/pull/8982))
+  - Adds config for setting the maximum amount of partitions to prove in a SubmitWindowPoSt message [Link to documentation](https://lotus.filecoin.io/storage-providers/seal-workers/post-workers/#multiple-partitions)
+- feat: sealer: Config for disabling builtin PoSt / PoSt pre-checks ([filecoin-project/lotus#8959](https://github.com/filecoin-project/lotus/pull/8959))
+  - Adds the ability to fully disable PoSt tasks on the `lotus-miner` and disabling windowPoSt pre-checks. [Link to documentation](https://lotus.filecoin.io/storage-providers/advanced-configurations/workers/#post-computations)
+- feat: add create ledger wallet address by account index command ([filecoin-project/lotus#8657](https://github.com/filecoin-project/lotus/pull/8657))
+ 
+## Improvements
+
+- feat: wdpost: Ignore faults in lotus-miner proving compute window-post ([filecoin-project/lotus#8737](https://github.com/filecoin-project/lotus/pull/8737))
+- feat: cli: Nicer net stat ([filecoin-project/lotus#8797](https://github.com/filecoin-project/lotus/pull/8797))
+- feat: networking: add healthz and livez endpoints ([filecoin-project/lotus#8692](https://github.com/filecoin-project/lotus/pull/8692))
+- feat: Snap Deals full unseal ([filecoin-project/lotus#8478](https://github.com/filecoin-project/lotus/pull/8478))
+- feat: cli: Hide sector nums in 'proving deadline' by default ([filecoin-project/lotus#8952](https://github.com/filecoin-project/lotus/pull/8952))
+- feat: Add rate limiting to the lotus gateway ([filecoin-project/lotus#8517](https://github.com/filecoin-project/lotus/pull/8517))
+- feat: networking: disconnect cmd ([filecoin-project/lotus#8955](https://github.com/filecoin-project/lotus/pull/8955))
+- feat: dagstore: Add DestroyShard() in dagstore wrapper ([filecoin-project/lotus#9010](https://github.com/filecoin-project/lotus/pull/9010))
+- feat: shed: report the "user version" ([filecoin-project/lotus#8864](https://github.com/filecoin-project/lotus/pull/8864))
+- feat: lotus-shed get remote peer hello message ([filecoin-project/lotus#8787](https://github.com/filecoin-project/lotus/pull/8787))
+- feat: only enable rcmgr by default in full nodes ([filecoin-project/lotus#8769](https://github.com/filecoin-project/lotus/pull/8769))
+- feat: refactor: actor bundling system (#8838) ([filecoin-project/lotus#8838](https://github.com/filecoin-project/lotus/pull/8838))
+- feat: migration: Implement function to migrate actors with only code changes ([filecoin-project/lotus#8843](https://github.com/filecoin-project/lotus/pull/8843))
+- feat: conformance & tvx: support ReportConsensusFault messages ([filecoin-project/lotus#8302](https://github.com/filecoin-project/lotus/pull/8302))
+- feat: wdpost: Envvar for limiting recovering sectors ([filecoin-project/lotus#9106](https://github.com/filecoin-project/lotus/pull/9106))
+- fix: post: restrict recoveries per deadline ([filecoin-project/lotus#9111](https://github.com/filecoin-project/lotus/pull/9111))
+- ux: print absolute time for proving period start in proving cli ([filecoin-project/lotus#8954](https://github.com/filecoin-project/lotus/pull/8954))
+- chore: storage refactors part 1 ([filecoin-project/lotus#8858](https://github.com/filecoin-project/lotus/pull/8858))
+- chore: sealing pipeline: Remove adapter code (storage refactors part 2) ([filecoin-project/lotus#8871](https://github.com/filecoin-project/lotus/pull/8871))
+- refactor: remove old BlockSyncProtocolID ([filecoin-project/lotus#8820](https://github.com/filecoin-project/lotus/pull/8820))
+
+## Bug Fixes
+
+- fix: format error log ([filecoin-project/lotus#8854](https://github.com/filecoin-project/lotus/pull/8854))
+- fix: build: really make macos compatible (#8853) ([filecoin-project/lotus#8853](https://github.com/filecoin-project/lotus/pull/8853))
+- fix: build: fix pack script and add calibrationnet to bundle ([filecoin-project/lotus#8852](https://github.com/filecoin-project/lotus/pull/8852))
+- fix: build: fix 2k build params ([filecoin-project/lotus#8835](https://github.com/filecoin-project/lotus/pull/8835))
+- Fix: PaychGetRestartAfterAddFundsMsg may stuck in forever waiting ([filecoin-project/lotus#8829](https://github.com/filecoin-project/lotus/pull/8829))
+- fix: paych: Print "waiting for confirmation.." ([filecoin-project/lotus#8823](https://github.com/filecoin-project/lotus/pull/8823))
+- fix: build: genesis miner network version ([filecoin-project/lotus#8756](https://github.com/filecoin-project/lotus/pull/8756))
+- fix: bench: consistency in description ([filecoin-project/lotus#8777](https://github.com/filecoin-project/lotus/pull/8777))
+- fix: worker: don't log normal storage stat calls ([filecoin-project/lotus#8744](https://github.com/filecoin-project/lotus/pull/8744))
+- fix: worker: don't check params with --no-default when not needed ([filecoin-project/lotus#8741](https://github.com/filecoin-project/lotus/pull/8741))
+- fix: Deduplicate parallel stat requests ([filecoin-project/lotus#8589](https://github.com/filecoin-project/lotus/pull/8589))
+- fix: Delegate storage auth on market nodes ([filecoin-project/lotus#8978](https://github.com/filecoin-project/lotus/pull/8978))
+- fix: post workers: check proving params on startup ([filecoin-project/lotus#8736](https://github.com/filecoin-project/lotus/pull/8736))
+- fix: rpc: readd rpc.discover aliases; lotus-gateway openrpc ([filecoin-project/lotus#8738](https://github.com/filecoin-project/lotus/pull/8738))
+- fix: change 1475 bootstrap peer ([filecoin-project/lotus#9008](https://github.com/filecoin-project/lotus/pull/9008))
+- fix: verifreg: update deprecation log ([filecoin-project/lotus#8690](https://github.com/filecoin-project/lotus/pull/8690))
+- fix: vm: support raw blocks in chain export ([filecoin-project/lotus#8691](https://github.com/filecoin-project/lotus/pull/8691))
+- fix: deps: restore butterfly network genesis from v1.14.4 ([filecoin-project/lotus#8708](https://github.com/filecoin-project/lotus/pull/8708))
+- fix: improve error message when maxPrice is too low ([filecoin-project/lotus#8818](https://github.com/filecoin-project/lotus/pull/8818))
+- fix: msig cli: Check for existing signers in add-propose ([filecoin-project/lotus#8833](https://github.com/filecoin-project/lotus/pull/8833))
+- Add new proofs version type ([filecoin-project/lotus#8848](https://github.com/filecoin-project/lotus/pull/8848))
+- fix: Make cli deal command get Block Delay specific to build ([filecoin-project/lotus#8896](https://github.com/filecoin-project/lotus/pull/8896))
+- fix: cli: Break out of retrieval if provider cancels ([filecoin-project/lotus#8912](https://github.com/filecoin-project/lotus/pull/8912))
+- fix: appimage build ([filecoin-project/lotus#8931](https://github.com/filecoin-project/lotus/pull/8931))
+- fix: incorrect usage of peer.IDFromString (should be peer.Decode) ([filecoin-project/lotus#8993](https://github.com/filecoin-project/lotus/pull/8993))
+- fix: deps: update FFI to fix a slow memory leak ([filecoin-project/lotus#9044](https://github.com/filecoin-project/lotus/pull/9044))
+- fix: sealing: hacky sealing fix backport ([filecoin-project/lotus#9055](https://github.com/filecoin-project/lotus/pull/9055))
+
+## Dependency Updates
+
+- github.com/filecoin-project/go-address (v0.0.6 -> v1.0.0)
+- github.com/filecoin-project/go-fil-markets (v1.20.1 -> v1.23.1)
+- github.com/filecoin-project/go-indexer-core (v0.2.8 -> v0.2.9)
+- github.com/filecoin-project/go-data-transfer (v1.15.1 -> v1.15.2)
+- github.com/filecoin-project/go-legs (v0.3.7 -> v0.4.4)
+- github.com/filecoin-project/go-state-types (v0.1.8 -> v0.1.10)
+- github.com/filecoin-project/index-provider (v0.5.0 -> v0.8.1)
+- github.com/filecoin-project/specs-actors (v0.9.14 -> v0.9.15)
+- github.com/filecoin-project/specs-actors/v3 (v3.1.1 -> v3.1.2)
+- github.com/filecoin-project/specs-actors/v4 (v4.0.1 -> v4.0.2)
+- github.com/filecoin-project/specs-actors/v5 (v5.0.4 -> v5.0.6)
+- github.com/filecoin-project/specs-actors/v6 (v6.0.1 -> v6.0.2)
+- github.com/filecoin-project/specs-actors/v7 (v7.0.0 -> v7.0.1)
+- github.com/filecoin-project/specs-actors/v8 (null -> v8.0.1)
+- github.com/filecoin-project/specs-storage (v0.2.4 -> v0.4.1)
+- github.com/filecoin-project/storetheindex (v0.3.5 -> v0.4.17)
+- deps: libp2p: update to the latest golibp2p tag ([filecoin-project/lotus#8704](https://github.com/filecoin-project/lotus/pull/8704))
+- chore: update and fix libp2p ([filecoin-project/lotus#8996](https://github.com/filecoin-project/lotus/pull/8996))
+
+## Others
+
+- chore: fix imports conflict ([filecoin-project/lotus#8863](https://github.com/filecoin-project/lotus/pull/8863))
+- chore: Fix imports ([filecoin-project/lotus#8859](https://github.com/filecoin-project/lotus/pull/8859))
+- chore: backport: release v1.16.0 back to master ([filecoin-project/lotus#8855](https://github.com/filecoin-project/lotus/pull/8855))
+- chore: bundle: remove wrongly committed bundle cars ([filecoin-project/lotus#8762](https://github.com/filecoin-project/lotus/pull/8762))
+- docs:sealing:fix default miner config comments ([filecoin-project/lotus#8689](https://github.com/filecoin-project/lotus/pull/8689))
+- ci: deps: Use testground-github-action from testground org ([filecoin-project/lotus#8490](https://github.com/filecoin-project/lotus/pull/8490))
+ 
+Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| Masih H. Derkani | 153 | +15515/-16832 | 660 |
+| Łukasz Magiera | 92 | +10429/-8024 | 1580 |
+| Andrew Gillis | 43 | +4149/-1765 | 208 |
+| Jennifer Wang | 10 | +1441/-1138 | 34 |
+| Geoff Stuart | 18 | +1348/-859 | 113 |
+| dirkmc | 11 | +1827/-210 | 70 |
+| Aayush | 21 | +1134/-894 | 69 |
+| Steven Allen | 9 | +743/-889 | 66 |
+| Marco Munizaga | 15 | +990/-252 | 36 |
+| gammazero | 47 | +681/-411 | 104 |
+| Will | 4 | +514/-246 | 29 |
+| web3-bot | 15 | +409/-348 | 20 |
+| Steven Fraser | 1 | +671/-0 | 36 |
+| Cory Schwartz | 27 | +520/-89 | 36 |
+| Hannah Howard | 3 | +318/-105 | 8 |
+| Piotr Galar | 2 | +337/-59 | 7 |
+| swift-mx | 14 | +264/-131 | 17 |
+| vyzo | 7 | +357/-15 | 16 |
+| Petar Maymounkov | 6 | +221/-23 | 14 |
+| LexLuthr | 7 | +182/-21 | 14 |
+| Aayush Rajasekaran | 5 | +97/-70 | 33 |
+| Raúl Kripalani | 5 | +87/-45 | 7 |
+| unknown | 1 | +114/-0 | 8 |
+| sti-bot | 44 | +54/-60 | 44 |
+| Aarsh Shah | 2 | +61/-50 | 8 |
+| Lucas Molas | 1 | +74/-27 | 3 |
+| zenground0 | 8 | +80/-18 | 14 |
+| Dirk McCormick | 3 | +52/-33 | 8 |
+| frank | 3 | +73/-7 | 3 |
+| Will Scott | 4 | +45/-11 | 5 |
+| kaola526 | 5 | +44/-11 | 5 |
+| dependabot[bot] | 3 | +16/-10 | 8 |
+| zl | 1 | +15/-4 | 4 |
+| Phi | 5 | +12/-6 | 6 |
+| Marcin Rataj | 1 | +11/-7 | 1 |
+| github-actions[bot] | 7 | +8/-8 | 7 |
+| Anton Evangelatov | 2 | +13/-0 | 4 |
+| Nicolas Gimenez | 1 | +12/-0 | 1 |
+| Marten Seemann | 2 | +5/-7 | 5 |
+| Chris Harden | 1 | +10/-0 | 2 |
+| jennijuju | 1 | +4/-4 | 7 |
+| Travis Person | 2 | +2/-6 | 2 |
+| Rod Vagg | 1 | +3/-3 | 2 |
+| Rob Quist | 1 | +3/-3 | 1 |
+| Jiaying Wang | 1 | +2/-3 | 2 |
+| zengroundumbass | 1 | +3/-1 | 1 |
+| lifei | 1 | +1/-1 | 1 |
+| Mike | 1 | +2/-0 | 1 |
+| Hubert | 1 | +1/-1 | 1 |
+| Daniel N | 1 | +1/-1 | 1 |
+| BMZ | 1 | +1/-1 | 1 |
+
+# 1.16.1 / 2022-07-07
+
+This is an OPTIONAL PATCH releases for storage providers who have failed to publish `SubmitWindowedPoSt` due to out of gas error. The error log looks like `/wdpost_run.go:xxx	estimating gas	{"error": "estimating gas used: message execution failed: exit SysErrOutOfGas(7)...`.
+
+## New Features
+
+- feat: declare fault recovery: Config for maximum partition count per message (#8988 / #8986)
+  - configure `MaxPartitionsPerRecoveryMessage` in miner configuration setting.
+- feat: wdpost: Config for maximum partition count per message (#8982 / #8986)
+  - configure `MaxPartitionsPerPoStMessage` in miner configuration setting.
+  
+# 1.16.0 / 2022-06-24
+
+This is a MANDATORY release of Lotus that introduces [Filecoin network v16,
+codenamed the Skyr upgrade](https://github.com/filecoin-project/community/discussions/74?sort=new#discussioncomment-2392151).
+
+The network is scheduled to upgrade to nv16 at epoch 1960320, on July 6th at 2022-07-06T14:00:00Z. All node operators, including storage providers, must upgrade to this release (or a later release) before that time. Storage providers must update their daemons, miners, market and worker(s).
+
+Your lotus node will switch from the Legacy VM (that depended on go-based specs-actors) to Filecoin Virtual Machine FVM (that uses Rust-based builtin-actors) atomically upon the upgrade.
+
+The Skyr upgrade introduces the following FIPs, enhancements and bug fixes, delivered in [built-actors v8](https://github.com/filecoin-project/builtin-actors/releases/tag/v8.0.0) and [ref-fvm v1.0.0](https://github.com/filecoin-project/ref-fvm/releases/tag/fvm%40v1.0.0):,
+
+- [FIP-0030](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0030.md)
+- [FIP-0031](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0031.md)
+- [FIP-0032](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0032.md))
+- [FIP-0027](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0027.md)
+- [Bug fix for the ProveReplicaUpdates method](https://github.com/filecoin-project/builtin-actors/pull/138)
+- [New proofs version for SnarkPack](https://github.com/filecoin-project/builtin-actors/pull/474/commits/3027c365f516e1cba6f156d4fb9dbd8c893d5b62)
+
+
+## 🆕 Things you may wanna know
+
+### Actor Code CIDs
+
+As stated in [FIP-0031](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0031.md)- [structure of the code cid](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0031.md#structure-of-code-cids), system actors' code CIDs will be real content-addressing
+
+For lotus users, we are making the change minimal for you. This means the `CODE` output when you run `lotus state get-actor` will now be the actual CID that represents the executable code for the actor, followed by wrapped synthetic id like the ones you've got before, i.e `fil/8/system`.
+
+Moreover, this also means that in the future, whenever the actor code changes, the CID changes accordingly, and a network upgrade is needed for the network participants to have consensus over what executable code we should use for each system actor.
+
+### Built-in actor bundles
+
+As the network introduces FVM, it's also switching from spec-actor (written in GoLang) to [built-in actor](https://github.com/filecoin-project/builtin-actors) (written in rust), in which the latter comes with[ importable bundles](https://github.com/filecoin-project/builtin-actors#importable-bundle). This means, like filecoin proof parameters, node operators now also need to fetch the actor bundles according to the network versions for the nodes to remain operational.
+
+Bundles for all networks(mainnet, calibnet, and etc) are included in the lotus source tree (`build/actors/`) and embedded on build. Lotus verifies that the bundle CIDs are the right ones upon build & upgrade against the values in `build/builtin_actors_gen.go`, according to the network you are building. You may also check the bundle manifest CID matches the bundle gen-ed values by running `lotus state actor-cids --network-version 16`.
+
+The manifest CID & full list of actor code CIDs for nv16 using v8.0.0 is:
+
+```
+    "_manifest":         "bafy2bzacebogjbpiemi7npzxchgcjjki3tfxon4ims55obfyfleqntteljsea"
+	"account":          "bafk2bzacedudbf7fc5va57t3tmo63snmt3en4iaidv4vo3qlyacbxaa6hlx6y"
+	"cron":             "bafk2bzacecqb3eolfurehny6yp7tgmapib4ocazo5ilkopjce2c7wc2bcec62"
+	"init":             "bafk2bzaceaipvjhoxmtofsnv3aj6gj5ida4afdrxa4ewku2hfipdlxpaektlw"
+	"multisig":         "bafk2bzacebhldfjuy4o5v7amrhp5p2gzv2qo5275jut4adnbyp56fxkwy5fag"
+	"paymentchannel":   "bafk2bzacebalad3f72wyk7qyilvfjijcwubdspytnyzlrhvn73254gqis44rq"
+	"reward":           "bafk2bzacecwzzxlgjiavnc3545cqqil3cmq4hgpvfp2crguxy2pl5ybusfsbe"
+	"storagemarket":    "bafk2bzacediohrxkp2fbsl4yj4jlupjdkgsiwqb4zuezvinhdo2j5hrxco62q"
+	"storageminer":     "bafk2bzacecgnynvd3tene3bvqoknuspit56canij5bpra6wl4mrq2mxxwriyu"
+	"storagepower":     "bafk2bzacebjvqva6ppvysn5xpmiqcdfelwbbcxmghx5ww6hr37cgred6dyrpm"
+	"system":           "bafk2bzacedwq5uppsw7vp55zpj7jdieizirmldceehu6wvombw3ixq2tcq57w"
+	"verifiedregistry": "bafk2bzaceb3zbkjz3auizmoln2unmxep7dyfcmsre64vnqfhdyh7rkqfoxlw4"
+```
+
+All bundles are also available at https://github.com/filecoin-project/builtin-actors/releases, thus you can also manually download the bundles and place them in the right path.
+
+Note: use customized bundle will risk you to lose sync with the network!
+
+To get Code CIDs:
+- api:`StateActorCodeCIDs`
+- cli: `lotus state actor-cids`
+- cli: `lotus-shed cid inspect-bundle`
+
+### Execution Trace
+
+For developers that are dependent on lotus execution trace, you will need  to enable `LOTUS_VM_ENABLE_TRACING` envvar to get the exact execution trace response as before. Without the envvar enabled, `Subcall` details, Duration` and `GasCharges` fields will be missing from the new FVM trace.
+
+### Deal Proposal Migration
+
+All deal proposals with non-utf8 string as the label in the metadata store will perform a light migration to new format as defined in [v1.1.1](https://github.com/filecoin-project/go-fil-markets/pull/721).
+
+## Others
+- Resource manager is now only enabled by default on full daemon node. You can enable it manually for other nodes by setting env var `LOTUS_RCMGR` to `1`.
+- Fix: drand: calculation of round from Filecoin epochs ([filecoin-project/lotus#8606](https://github.com/filecoin-project/lotus/pull/8606))
+
+## Dependency Update
+- chore: deps: update to go-libp2p v0.19.4 ([filecoin-project/lotus#8801](https://github.com/filecoin-project/lotus/pull/8801))
+- github.com/filecoin-project/go-fil-markets (v1.20.1 -> v1.20.1-v16-2):
+- github.com/filecoin-project/go-legs (v0.3.7 -> v0.3.10):
+- github.com/filecoin-project/go-state-types (v0.1.8 -> v0.1.10):
+- github.com/filecoin-project/specs-storage (v0.2.4 -> v0.4.1):
+
+## Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| @geoff-vball | 37 | +10565/-8454 | 150 |
+| @arajasek | 33 | +7964/-6340 | 473 |
+| @arajasek | 37 | +6220/-6976 | 457 |
+| @vyzo | 135 | +7287/-5380 | 546 |
+| @ZenGround0 | 19 | +5958/-2920 | 226 |
+| @stebalien | 18 | +1566/-1101 | 116 |
+| Alex | 5 | +323/-2304 | 32 |
+| @zenground0 | 9 | +583/-358 | 56 |
+| @jennijuju | 5 | +853/-27 | 19 |
+| @jennijuju | 24 | +392/-201 | 60 |
+| Marco Munizaga | 3 | +236/-83 | 5 |
+| @raulk| 9 | +93/-15 | 15 |
+| @travisperson | 3 | +37/-37 | 12 |
+| @Kubuxu | 1 | +41/-5 | 1 |
+| @koalacxr | 1 | +29/-13 | 3 |
+| @gammazero | 2 | +18/-10 | 4 |
+| Peter Rabbitson | 1 | +5/-3 | 1 |
+| Steve Loeppky | 1 | +6/-0 | 1 |
+| @masih | 1 | +3/-3 | 2 |
+| @magik6k | 1 | +4/-0 | 1 |
+| @jennijuju | 1 | +2/-2 | 1 |
+| tian zhou | 1 | +1/-1 | 1 |
+
 # 1.15.3 / 2022-05-31
 
 This is an optional release of lotus that include new APIs, some improvements and bug fixes.
@@ -75,7 +469,7 @@ This is an optional release of lotus that include new APIs, some improvements an
 
 # 1.15.2 / 2022-05-06
 
-This is a highly recommended feature lotus release v1.15.2. This feature release introduces many new features and  for SPs, including PoSt workers, sealing scheduler, snap deal queue and so on. 
+This is a highly recommended feature lotus release v1.15.2. This feature release introduces many new features and  for SPs, including PoSt workers, sealing scheduler, snap deal queue and so on.
 
 Note: You need to be using go v1.17.9&up from this release onwards.
 
@@ -233,31 +627,31 @@ This feature release introduces Index Provider, GraphSync v2, and many other lat
 
 ### 🔥🔥🔥 FVM (Experimental) 🔥🔥🔥
 - feat: fvm: FVM integration  ([filecoin-project/lotus#8332](https://github.com/filecoin-project/lotus/pull/8332))
-The lotus team is excited to announce the launch of experimental non-programmable FVM on mainnet. By enabling `"LOTUS_USE_FVM_EXPERIMENTAL=1` envvar, the lotus daemon will be running the [WASM-compiled built-in actors](https://github.com/filecoin-project/builtin-actors) that is compatible with the existing chain(Network v15 OhSnap). If you are trying it out and having any questions or feedbacks, please leave a comment [here](https://github.com/filecoin-project/lotus/discussions/8334)!
+  The lotus team is excited to announce the launch of experimental non-programmable FVM on mainnet. By enabling `"LOTUS_USE_FVM_EXPERIMENTAL=1` envvar, the lotus daemon will be running the [WASM-compiled built-in actors](https://github.com/filecoin-project/builtin-actors) that is compatible with the existing chain(Network v15 OhSnap). If you are trying it out and having any questions or feedbacks, please leave a comment [here](https://github.com/filecoin-project/lotus/discussions/8334)!
   - chore: FVM: log when fvm is used([filecoin-project/lotus#8363](https://github.com/filecoin-project/lotus/pull/8363))
   - chore: ffi: the latest fvm release([filecoin-project/lotus#8382](https://github.com/filecoin-project/lotus/pull/8382))
-  
+
 ### 🌟🌟🌟 Index Provider (Production Ready!) 🌟🌟🌟
 - feat: markets: Integrate index ingest protocol and retrieve by any CID ([filecoin-project/lotus#7313](https://github.com/filecoin-project/lotus/pull/7313))
 
-More and more useful data is being stored on Filecoin via deals made by clients to Storage Providers. The goal is that this content is discoverable when people need them. To achieve that goal, one of the projects [the Bedrock team](https://www.notion.so/pl-strflt/Bedrock-2e956d5d8143432080a1d84435ccf0ff) is working on is building an Indexer Ecosystem, a project that's focus on content addressing on Filecoin, then potentially have interoperability with IPFS in the future and eventually serve the retrieval market. The Indexer Ecosystem high level architecture overview diagram can be found [here](https://github.com/filecoin-project/storetheindex/blob/main/doc/indexer_ecosys.png) and a detailed write up about can be found [here](https://www.notion.so/pl-strflt/Introducing-Indexer-to-SP-90bf296794174a8281c121d4ce6747a0).  
+More and more useful data is being stored on Filecoin via deals made by clients to Storage Providers. The goal is that this content is discoverable when people need them. To achieve that goal, one of the projects [the Bedrock team](https://www.notion.so/pl-strflt/Bedrock-2e956d5d8143432080a1d84435ccf0ff) is working on is building an Indexer Ecosystem, a project that's focus on content addressing on Filecoin, then potentially have interoperability with IPFS in the future and eventually serve the retrieval market. The Indexer Ecosystem high level architecture overview diagram can be found [here](https://github.com/filecoin-project/storetheindex/blob/main/doc/indexer_ecosys.png) and a detailed write up about can be found [here](https://www.notion.so/pl-strflt/Introducing-Indexer-to-SP-90bf296794174a8281c121d4ce6747a0).
 
 That being said, with this release, lotus Storage Providers can easily become an Index Provider and serve the Indexer Content Addressing System. Index Providers generate advertisements from the deals made by a storage provider and announces the data to the indexer nodes for further processing:
 - To learn more about *what is an Index Provider and how to be an Index Provider*, read it [here](https://lotus.filecoin.io/storage-providers/operate/index-provider/) in lotus docuementation.
 - An [one-off migration](https://lotus.filecoin.io/storage-providers/operate/index-provider/#first-time-migration) is needed in order for a Storage Provider to become an Index Provider and announce the proper formatted index. It's *highly recommended* for all Index Provider to do a [force bulk initialization](https://lotus.filecoin.io/storage-providers/configure/dagstore/#forcing-bulk-initialization) to enable index announcement on all existing deals.
   - Note that the Initialization places IO workload on your storage system. SP should set a proper `concurrency` based on your hardware or can stop/start initialization at their wish/convenience as proving deadlines approach and elapse, to avoid IOPS starvation or competition with window PoSt.
-- After the first one-time migration, being an Index Provider barely puts any extra usage on SP's market system.  
+- After the first one-time migration, being an Index Provider barely puts any extra usage on SP's market system.
   - You can find the testing result by SPX fellows [here](https://github.com/filecoin-project/lotus/discussions/8087).
- 
-We recommend all Storage Providers that are serving deals in the Filecoin network to become a Index Provider, make the data you are storing discoverable for the retrieval market and retrieval clients! 
-  - If you have any questions about becoming an index provider, or the indexer system in general, leave a comment [here](https://github.com/filecoin-project/lotus/discussions/8341).
-  - Follow the indexer project at https://github.com/filecoin-project/go-indexer-core.
-  - If you have any feature request or bug reports of running an index provider, create an issue in https://github.com/filecoin-project/index-provider.
-  - You may also join the #storetheindex channel in the Filecoin Slack to engage with the team & the community!
+
+We recommend all Storage Providers that are serving deals in the Filecoin network to become a Index Provider, make the data you are storing discoverable for the retrieval market and retrieval clients!
+- If you have any questions about becoming an index provider, or the indexer system in general, leave a comment [here](https://github.com/filecoin-project/lotus/discussions/8341).
+- Follow the indexer project at https://github.com/filecoin-project/go-indexer-core.
+- If you have any feature request or bug reports of running an index provider, create an issue in https://github.com/filecoin-project/index-provider.
+- You may also join the #storetheindex channel in the Filecoin Slack to engage with the team & the community!
 
 ### ❗️❗️❗️ Dag Migration For New CAR index format in DagStore ❗️❗️❗️
 The index provider leverages the latest CARv2 indexing format `MultihashIndexSorted`, which stores the multihash code as well as the digest of all CIDs in a CAR file. Thus, all Storage Providers SHOULD perform an one-off DAG mirgation to regenerate DagStore CARv2 indices. You have to do it to become an index provider, failing to do so may also impact your future deal making.
-Follow the instruction [here](https://lotus.filecoin.io/storage-providers/operate/index-provider/) to perform the migration. 
+Follow the instruction [here](https://lotus.filecoin.io/storage-providers/operate/index-provider/) to perform the migration.
 
 ## New Features
 - feat: sealing: Sector upgrade queue ([filecoin-project/lotus#8333](https://github.com/filecoin-project/lotus/pull/8333))
@@ -297,7 +691,7 @@ Follow the instruction [here](https://lotus.filecoin.io/storage-providers/operat
 ## Bug Fixes
 - fix: FVM: add finality check for consensus faults #8452
 - fix: market: Reuse the market PubSub in index provider #8451
-- fix: market: set all index provider options based on lotus config #8444  
+- fix: market: set all index provider options based on lotus config #8444
 - fix: sealing: Fix PR1 worker selection (#8421)
 - fix: miner: dead loop on removing sector (#8421)
 - fix: sealing: Remove sector copies from workers after snapdeals ([filecoin-project/lotus#8331](https://github.com/filecoin-project/lotus/pull/8331))

Makefile

@@ -64,7 +64,7 @@ CLEAN+=build/.update-modules
 deps: $(BUILD_DEPS)
 .PHONY: deps
 
-build-devnets: build lotus-seed lotus-shed lotus-wallet lotus-gateway
+build-devnets: build lotus-seed lotus-shed lotus-wallet lotus-gateway lotus-fountain lotus-stats
 .PHONY: build-devnets
 
 debug: GOFLAGS+=-tags=debug
@@ -133,6 +133,18 @@ install-worker:
 install-app:
 	install -C ./$(APP) /usr/local/bin/$(APP)
 
+uninstall: uninstall-daemon uninstall-miner uninstall-worker
+.PHONY: uninstall
+
+uninstall-daemon:
+	rm -f /usr/local/bin/lotus
+
+uninstall-miner:
+	rm -f /usr/local/bin/lotus-miner
+
+uninstall-worker:
+	rm -f /usr/local/bin/lotus-worker
+
 # TOOLS
 
 lotus-seed: $(BUILD_DEPS)
@@ -162,14 +174,14 @@ lotus-pond-app: lotus-pond-front lotus-pond
 
 lotus-fountain:
 	rm -f lotus-fountain
-	go build $(GOFLAGS) -o lotus-fountain ./cmd/lotus-fountain
-	go run github.com/GeertJohan/go.rice/rice append --exec lotus-fountain -i ./cmd/lotus-fountain -i ./build
+	$(GOCC) build $(GOFLAGS) -o lotus-fountain ./cmd/lotus-fountain
+	$(GOCC) run github.com/GeertJohan/go.rice/rice append --exec lotus-fountain -i ./cmd/lotus-fountain -i ./build
 .PHONY: lotus-fountain
 BINS+=lotus-fountain
 
 lotus-bench:
 	rm -f lotus-bench
-	$(GOCC) build -o lotus-bench ./cmd/lotus-bench
+	$(GOCC) build $(GOFLAGS) -o lotus-bench ./cmd/lotus-bench
 .PHONY: lotus-bench
 BINS+=lotus-bench
 
@@ -290,11 +302,20 @@ type-gen: api-gen
 method-gen: api-gen
 	(cd ./lotuspond/front/src/chain && $(GOCC) run ./methodgen.go)
 
-actors-gen:
+actors-code-gen:
 	$(GOCC) run ./gen/inline-gen . gen/inlinegen-data.json
 	$(GOCC) run ./chain/actors/agen
 	$(GOCC) fmt ./...
 
+actors-gen: actors-code-gen fiximports
+.PHONY: actors-gen
+
+bundle-gen:
+	$(GOCC) run ./gen/bundle
+	$(GOCC) fmt ./build/...
+.PHONY: bundle-gen
+
+
 api-gen:
 	$(GOCC) run ./gen/api
 	goimports -w api
@@ -313,7 +334,7 @@ appimage: lotus
 	cp ./lotus AppDir/usr/bin/
 	appimage-builder
 
-docsgen: docsgen-md docsgen-openrpc
+docsgen: docsgen-md docsgen-openrpc fiximports
 
 docsgen-md-bin: api-gen actors-gen
 	$(GOCC) build $(GOFLAGS) -o docgen-md ./api/docgen/cmd
@@ -330,7 +351,7 @@ docsgen-md-storage: docsgen-md-bin
 docsgen-md-worker: docsgen-md-bin
 	./docgen-md "api/api_worker.go" "Worker" "api" "./api" > documentation/en/api-v0-methods-worker.md
 
-docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker
+docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker docsgen-openrpc-gateway
 
 docsgen-openrpc-full: docsgen-openrpc-bin
 	./docgen-openrpc "api/api_full.go" "FullNode" "api" "./api" -gzip > build/openrpc/full.json.gz
@@ -338,10 +359,15 @@ docsgen-openrpc-storage: docsgen-openrpc-bin
 	./docgen-openrpc "api/api_storage.go" "StorageMiner" "api" "./api" -gzip > build/openrpc/miner.json.gz
 docsgen-openrpc-worker: docsgen-openrpc-bin
 	./docgen-openrpc "api/api_worker.go" "Worker" "api" "./api" -gzip > build/openrpc/worker.json.gz
+docsgen-openrpc-gateway: docsgen-openrpc-bin
+	./docgen-openrpc "api/api_gateway.go" "Gateway" "api" "./api" -gzip > build/openrpc/gateway.json.gz
 
 .PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin
 
-gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci
+fiximports:
+	./scripts/fiximports
+
+gen: actors-code-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci bundle-gen fiximports
 	@echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO MAKE docsgen-cli"
 .PHONY: gen
 

README.md

@@ -20,7 +20,7 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more
 
 > Note: The default `master` branch is the dev branch, please use with caution. For the latest stable version, checkout the most recent [`Latest release`](https://github.com/filecoin-project/lotus/releases).
  
-For complete instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/). Basic build instructions can be found further down in this readme.
+For complete instructions on how to build, install and setup lotus, please visit [https://lotus.filecoin.io](https://lotus.filecoin.io/lotus/install/prerequisites/#supported-platforms). Basic build instructions can be found further down in this readme.
 
 ## Reporting a Vulnerability
 
@@ -31,7 +31,7 @@ Please send an email to security@filecoin.org. See our [security policy](SECURIT
 These repos are independent and reusable modules, but are tightly integrated into Lotus to make up a fully featured Filecoin implementation:
 
 - [go-fil-markets](https://github.com/filecoin-project/go-fil-markets) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/markets-shared-components-5daa144a7046a60001c6e253/board)
-- [specs-actors](https://github.com/filecoin-project/specs-actors) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/actors-5ee6f3aa87591f0016c05685/board)
+- [builtin-actors](https://github.com/filecoin-project/builtin-actors)
 
 ## Contribute
 
@@ -64,7 +64,7 @@ sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config cu
 
 Fedora:
 ```
-sudo dnf -y install gcc make git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm wget hwloc libhwloc-dev
+sudo dnf -y install gcc make git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm wget hwloc hwloc-devel
 ```
 
 For other distributions you can find the required dependencies [here.](https://docs.filecoin.io/get-started/lotus/installation/#system-specific) For instructions specific to macOS, you can find them [here.](https://docs.filecoin.io/get-started/lotus/installation/#macos)

api/api_common.go

@@ -4,12 +4,12 @@ import (
 	"context"
 	"fmt"
 
-	apitypes "github.com/filecoin-project/lotus/api/types"
-	"github.com/filecoin-project/lotus/journal/alerting"
-
 	"github.com/google/uuid"
 
 	"github.com/filecoin-project/go-jsonrpc/auth"
+
+	apitypes "github.com/filecoin-project/lotus/api/types"
+	"github.com/filecoin-project/lotus/journal/alerting"
 )
 
 //                       MODIFYING THE API INTERFACE

api/api_full.go

@@ -6,25 +6,27 @@ import (
 	"fmt"
 	"time"
 
+	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	"github.com/libp2p/go-libp2p-core/peer"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/go-state-types/crypto"
-	"github.com/filecoin-project/go-state-types/dline"
-
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/builtin/v8/market"
+	"github.com/filecoin-project/go-state-types/builtin/v8/miner"
+	"github.com/filecoin-project/go-state-types/builtin/v8/paych"
+	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/go-state-types/dline"
+	abinetwork "github.com/filecoin-project/go-state-types/network"
 
 	apitypes "github.com/filecoin-project/lotus/api/types"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
+	lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
@@ -37,6 +39,7 @@ import (
 type ChainIO interface {
 	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
+	ChainPutObj(context.Context, blocks.Block) error
 }
 
 const LookbackNoLimit = abi.ChainEpoch(-1)
@@ -122,6 +125,9 @@ type FullNode interface {
 	// ChainHasObj checks if a given CID exists in the chain blockstore.
 	ChainHasObj(context.Context, cid.Cid) (bool, error) //perm:read
 
+	// ChainPutObj puts a given object into the block store
+	ChainPutObj(context.Context, blocks.Block) error //perm:admin
+
 	// ChainStatObj returns statistics about the graph referenced by 'obj'.
 	// If 'base' is also specified, then the returned stat will be a diff
 	// between the two objects.
@@ -163,6 +169,10 @@ type FullNode interface {
 	// If oldmsgskip is set, messages from before the requested roots are also not included.
 	ChainExport(ctx context.Context, nroots abi.ChainEpoch, oldmsgskip bool, tsk types.TipSetKey) (<-chan []byte, error) //perm:read
 
+	// ChainPrune prunes the stored chain state and garbage collects; only supported if you
+	// are using the splitstore
+	ChainPrune(ctx context.Context, opts PruneOpts) error //perm:admin
+
 	// ChainCheckBlockstore performs an (asynchronous) health check on the chain/state blockstore
 	// if supported by the underlying implementation.
 	ChainCheckBlockstore(context.Context) error //perm:admin
@@ -170,14 +180,6 @@ type FullNode interface {
 	// ChainBlockstoreInfo returns some basic information about the blockstore
 	ChainBlockstoreInfo(context.Context) (map[string]interface{}, error) //perm:read
 
-	// MethodGroup: Beacon
-	// The Beacon method group contains methods for interacting with the random beacon (DRAND)
-
-	// BeaconGetEntry returns the beacon entry for the given filecoin epoch. If
-	// the entry has not yet been produced, the call will block until the entry
-	// becomes available
-	BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) //perm:read
-
 	// GasEstimateFeeCap estimates gas fee cap
 	GasEstimateFeeCap(context.Context, *types.Message, int64, types.TipSetKey) (types.BigInt, error) //perm:read
 
@@ -404,7 +406,7 @@ type FullNode interface {
 	StateCall(context.Context, *types.Message, types.TipSetKey) (*InvocResult, error) //perm:read
 	// StateReplay replays a given message, assuming it was included in a block in the specified tipset.
 	//
-	// If a tipset key is provided, and a replacing message is found on chain,
+	// If a tipset key is provided, and a replacing message is not found on chain,
 	// the method will return an error saying that the message wasn't found
 	//
 	// If no tipset key is provided, the appropriate tipset is looked up, and if
@@ -443,7 +445,7 @@ type FullNode interface {
 	// StateMinerPower returns the power of the indicated miner
 	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error) //perm:read
 	// StateMinerInfo returns info about the indicated miner
-	StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) //perm:read
+	StateMinerInfo(context.Context, address.Address, types.TipSetKey) (MinerInfo, error) //perm:read
 	// StateMinerDeadlines returns all the proving deadlines for the given miner
 	StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]Deadline, error) //perm:read
 	// StateMinerPartitions returns all partitions in the specified deadline
@@ -460,18 +462,23 @@ type FullNode interface {
 	StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) //perm:read
 	// StateMinerAvailableBalance returns the portion of a miner's balance that can be withdrawn or spent
 	StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) //perm:read
-	// StateMinerSectorAllocated checks if a sector is allocated
+	// StateMinerSectorAllocated checks if a sector number is marked as allocated.
 	StateMinerSectorAllocated(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (bool, error) //perm:read
-	// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector
-	StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) //perm:read
+	// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector.
+	// Returns nil and no error if the sector isn't precommitted.
+	//
+	// Note that the sector number may be allocated while PreCommitInfo is nil. This means that either allocated sector
+	// numbers were compacted, and the sector number was marked as allocated in order to reduce size of the allocated
+	// sectors bitfield, or that the sector was precommitted, but the precommit has expired.
+	StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorPreCommitOnChainInfo, error) //perm:read
 	// StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found
 	// NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate
 	// expiration epoch
 	StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) //perm:read
 	// StateSectorExpiration returns epoch at which given sector will expire
-	StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error) //perm:read
+	StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*lminer.SectorExpiration, error) //perm:read
 	// StateSectorPartition finds deadline/partition with the specified sector
-	StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) //perm:read
+	StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*lminer.SectorLocation, error) //perm:read
 	// StateSearchMsg looks back up to limit epochs in the chain for a message, and returns its receipt and the tipset where it was executed
 	//
 	// NOTE: If a replacing message is found on chain, this method will return
@@ -517,9 +524,11 @@ type FullNode interface {
 	// StateMarketParticipants returns the Escrow and Locked balances of every participant in the Storage Market
 	StateMarketParticipants(context.Context, types.TipSetKey) (map[string]MarketBalance, error) //perm:read
 	// StateMarketDeals returns information about every deal in the Storage Market
-	StateMarketDeals(context.Context, types.TipSetKey) (map[string]MarketDeal, error) //perm:read
+	StateMarketDeals(context.Context, types.TipSetKey) (map[string]*MarketDeal, error) //perm:read
 	// StateMarketStorageDeal returns information about the indicated deal
 	StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*MarketDeal, error) //perm:read
+	// StateComputeDataCID computes DataCID from a set of on-chain deals
+	StateComputeDataCID(ctx context.Context, maddr address.Address, sectorType abi.RegisteredSealProof, deals []abi.DealID, tsk types.TipSetKey) (cid.Cid, error) //perm:read
 	// StateLookupID retrieves the ID address of the given address
 	StateLookupID(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
 	// StateAccountKey returns the public key address of the given ID address for secp and bls accounts
@@ -586,12 +595,19 @@ type FullNode interface {
 	StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (CirculatingSupply, error) //perm:read
 	// StateNetworkVersion returns the network version at the given tipset
 	StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) //perm:read
+	// StateActorCodeCIDs returns the CIDs of all the builtin actors for the given network version
+	StateActorCodeCIDs(context.Context, abinetwork.Version) (map[string]cid.Cid, error) //perm:read
 
 	// StateGetRandomnessFromTickets is used to sample the chain for randomness.
 	StateGetRandomnessFromTickets(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
 	// StateGetRandomnessFromBeacon is used to sample the beacon for randomness.
 	StateGetRandomnessFromBeacon(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
 
+	// StateGetBeaconEntry returns the beacon entry for the given filecoin epoch. If
+	// the entry has not yet been produced, the call will block until the entry
+	// becomes available
+	StateGetBeaconEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) //perm:read
+
 	// StateGetNetworkParams return current network params
 	StateGetNetworkParams(ctx context.Context) (*NetworkParams, error) //perm:read
 
@@ -1207,3 +1223,8 @@ type MsigTransaction struct {
 
 	Approved []address.Address
 }
+
+type PruneOpts struct {
+	MovingGC    bool
+	RetainState int64
+}

api/api_gateway.go

@@ -3,14 +3,15 @@ package api
 import (
 	"context"
 
+	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/builtin/v8/miner"
 	"github.com/filecoin-project/go-state-types/dline"
 
 	apitypes "github.com/filecoin-project/lotus/api/types"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
@@ -30,6 +31,7 @@ import (
 
 type Gateway interface {
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
+	ChainPutObj(context.Context, blocks.Block) error
 	ChainHead(ctx context.Context) (*types.TipSet, error)
 	ChainGetParentMessages(context.Context, cid.Cid) ([]Message, error)
 	ChainGetParentReceipts(context.Context, cid.Cid) ([]*types.MessageReceipt, error)
@@ -56,7 +58,7 @@ type Gateway interface {
 	StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
 	StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MarketBalance, error)
 	StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*MarketDeal, error)
-	StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error)
+	StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (MinerInfo, error)
 	StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
 	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error)
 	StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error)
@@ -66,4 +68,5 @@ type Gateway interface {
 	StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
 	WalletBalance(context.Context, address.Address) (types.BigInt, error)
 	Version(context.Context) (APIVersion, error)
+	Discover(context.Context) (apitypes.OpenRPCDocument, error)
 }

api/api_net.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"time"
 
-	metrics "github.com/libp2p/go-libp2p-core/metrics"
+	"github.com/libp2p/go-libp2p-core/metrics"
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/libp2p/go-libp2p-core/protocol"

api/api_storage.go

@@ -5,28 +5,26 @@ import (
 	"context"
 	"time"
 
-	"github.com/filecoin-project/lotus/chain/actors/builtin"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
-
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
 	"github.com/libp2p/go-libp2p-core/peer"
 
 	"github.com/filecoin-project/go-address"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-state-types/abi"
-	abinetwork "github.com/filecoin-project/go-state-types/network"
-	"github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
-	"github.com/filecoin-project/specs-storage/storage"
-
 	"github.com/filecoin-project/go-fil-markets/piecestore"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/filecoin-project/go-jsonrpc/auth"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/builtin/v8/market"
+	"github.com/filecoin-project/go-state-types/builtin/v8/miner"
+	abinetwork "github.com/filecoin-project/go-state-types/network"
 
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-	"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
+	"github.com/filecoin-project/lotus/storage/pipeline/sealiface"
+	"github.com/filecoin-project/lotus/storage/sealer/fsutil"
+	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
 //                       MODIFYING THE API INTERFACE
@@ -50,11 +48,16 @@ type StorageMiner interface {
 	ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error) //perm:read
 	ActorAddressConfig(ctx context.Context) (AddressConfig, error)            //perm:read
 
+	// WithdrawBalance allows to withdraw balance from miner actor to owner address
+	// Specify amount as "0" to withdraw full balance. This method returns a message CID
+	// and does not wait for message execution
+	ActorWithdrawBalance(ctx context.Context, amount abi.TokenAmount) (cid.Cid, error) //perm:admin
+
 	MiningBase(context.Context) (*types.TipSet, error) //perm:read
 
 	ComputeWindowPoSt(ctx context.Context, dlIdx uint64, tsk types.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) //perm:admin
 
-	ComputeDataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) //perm:admin
+	ComputeDataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (abi.PieceInfo, error) //perm:admin
 
 	// Temp api for testing
 	PledgeSector(context.Context) (abi.SectorID, error) //perm:write
@@ -65,9 +68,9 @@ type StorageMiner interface {
 	// Add piece to an open sector. If no sectors with enough space are open,
 	// either a new sector will be created, or this call will block until more
 	// sectors can be created.
-	SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storage.Data, d PieceDealInfo) (SectorOffset, error) //perm:admin
+	SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storiface.Data, d PieceDealInfo) (SectorOffset, error) //perm:admin
 
-	SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error //perm:admin
+	SectorsUnsealPiece(ctx context.Context, sector storiface.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error //perm:admin
 
 	// List all staged sectors
 	SectorsList(context.Context) ([]abi.SectorNumber, error) //perm:read
@@ -126,46 +129,66 @@ type StorageMiner interface {
 	WorkerJobs(context.Context) (map[uuid.UUID][]storiface.WorkerJob, error)  //perm:admin
 
 	//storiface.WorkerReturn
-	ReturnDataCid(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error                                       //perm:admin retry:true
-	ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error                                      //perm:admin retry:true
-	ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storage.PreCommit1Out, err *storiface.CallError) error                       //perm:admin retry:true
-	ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storage.SectorCids, err *storiface.CallError) error                       //perm:admin retry:true
-	ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storage.Commit1Out, err *storiface.CallError) error                             //perm:admin retry:true
-	ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storage.Proof, err *storiface.CallError) error                                //perm:admin retry:true
-	ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                  //perm:admin retry:true
-	ReturnReplicaUpdate(ctx context.Context, callID storiface.CallID, out storage.ReplicaUpdateOut, err *storiface.CallError) error                     //perm:admin retry:true
-	ReturnProveReplicaUpdate1(ctx context.Context, callID storiface.CallID, vanillaProofs storage.ReplicaVanillaProofs, err *storiface.CallError) error //perm:admin retry:true
-	ReturnProveReplicaUpdate2(ctx context.Context, callID storiface.CallID, proof storage.ReplicaUpdateProof, err *storiface.CallError) error           //perm:admin retry:true
-	ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                       //perm:admin retry:true
-	ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                           //perm:admin retry:true
-	ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                 //perm:admin retry:true
-	ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                     //perm:admin retry:true
-	ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                     //perm:admin retry:true
-	ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err *storiface.CallError) error                                              //perm:admin retry:true
-	ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                           //perm:admin retry:true
+	ReturnDataCid(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error                                         //perm:admin retry:true
+	ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error                                        //perm:admin retry:true
+	ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storiface.PreCommit1Out, err *storiface.CallError) error                       //perm:admin retry:true
+	ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storiface.SectorCids, err *storiface.CallError) error                       //perm:admin retry:true
+	ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storiface.Commit1Out, err *storiface.CallError) error                             //perm:admin retry:true
+	ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storiface.Proof, err *storiface.CallError) error                                //perm:admin retry:true
+	ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                    //perm:admin retry:true
+	ReturnReplicaUpdate(ctx context.Context, callID storiface.CallID, out storiface.ReplicaUpdateOut, err *storiface.CallError) error                     //perm:admin retry:true
+	ReturnProveReplicaUpdate1(ctx context.Context, callID storiface.CallID, vanillaProofs storiface.ReplicaVanillaProofs, err *storiface.CallError) error //perm:admin retry:true
+	ReturnProveReplicaUpdate2(ctx context.Context, callID storiface.CallID, proof storiface.ReplicaUpdateProof, err *storiface.CallError) error           //perm:admin retry:true
+	ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                         //perm:admin retry:true
+	ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                             //perm:admin retry:true
+	ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                   //perm:admin retry:true
+	ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                       //perm:admin retry:true
+	ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                       //perm:admin retry:true
+	ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err *storiface.CallError) error                                                //perm:admin retry:true
+	ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                             //perm:admin retry:true
 
 	// SealingSchedDiag dumps internal sealing scheduler state
 	SealingSchedDiag(ctx context.Context, doSched bool) (interface{}, error) //perm:admin
 	SealingAbort(ctx context.Context, call storiface.CallID) error           //perm:admin
+	//SealingSchedRemove removes a request from sealing pipeline
+	SealingRemoveRequest(ctx context.Context, schedId uuid.UUID) error //perm:admin
 
-	// SectorIndex
-	StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error                                                                                             //perm:admin
-	StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error)                                                                                              //perm:admin
-	StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error                                                                                       //perm:admin
-	StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error                                     //perm:admin
-	StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error                                                      //perm:admin
+	// paths.SectorIndex
+	StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error                                                         //perm:admin
+	StorageDetach(ctx context.Context, id storiface.ID, url string) error                                                              //perm:admin
+	StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error)                                                          //perm:admin
+	StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error                                                   //perm:admin
+	StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error //perm:admin
+	StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error                  //perm:admin
+	// StorageFindSector returns list of paths where the specified sector files exist.
+	//
+	// If allowFetch is set, list of paths to which the sector can be fetched will also be returned.
+	// - Paths which have sector files locally (don't require fetching) will be listed first.
+	// - Paths which have sector files locally will not be filtered based on based on AllowTypes/DenyTypes.
+	// - Paths which require fetching will be filtered based on AllowTypes/DenyTypes. If multiple
+	//   file types are specified, each type will be considered individually, and a union of all paths
+	//   which can accommodate each file type will be returned.
 	StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error) //perm:admin
-	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error)           //perm:admin
-	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                                             //perm:admin
-	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                                  //perm:admin
-	StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error)                                                                                            //perm:admin
-	StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error)                                                                                                    //perm:admin
+	// StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.
+	// Paths with more weight and more % of free space are preferred.
+	// Note: This method doesn't filter paths based on AllowTypes/DenyTypes.
+	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) //perm:admin
+	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                                   //perm:admin
+	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                        //perm:admin
+	StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error)                                                                                  //perm:admin
+	StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error)                                                                                          //perm:admin
 
 	StorageLocal(ctx context.Context) (map[storiface.ID]string, error)       //perm:admin
 	StorageStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error) //perm:admin
 
+	StorageAuthVerify(ctx context.Context, token string) ([]auth.Permission, error) //perm:read
+
+	StorageAddLocal(ctx context.Context, path string) error                              //perm:admin
+	StorageDetachLocal(ctx context.Context, path string) error                           //perm:admin
+	StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error //perm:admin
+
 	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error                                                                                                        //perm:write
-	MarketListDeals(ctx context.Context) ([]MarketDeal, error)                                                                                                                           //perm:read
+	MarketListDeals(ctx context.Context) ([]*MarketDeal, error)                                                                                                                          //perm:read
 	MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error)                                                                                           //perm:read
 	MarketGetDealUpdates(ctx context.Context) (<-chan storagemarket.MinerDeal, error)                                                                                                    //perm:read
 	MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error)                                                                                                    //perm:read
@@ -227,6 +250,9 @@ type StorageMiner interface {
 	// DagstoreGC runs garbage collection on the DAG store.
 	DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin
 
+	// DagstoreRegisterShard registers a shard manually with dagstore with given pieceCID
+	DagstoreRegisterShard(ctx context.Context, key string) error //perm:admin
+
 	// IndexerAnnounceDeal informs indexer nodes that a new deal was received,
 	// so they can download its index
 	IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin
@@ -242,7 +268,7 @@ type StorageMiner interface {
 	RuntimeSubsystems(ctx context.Context) (MinerSubsystems, error) //perm:read
 
 	DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error //perm:admin
-	DealsList(ctx context.Context) ([]MarketDeal, error)                         //perm:admin
+	DealsList(ctx context.Context) ([]*MarketDeal, error)                        //perm:admin
 	DealsConsiderOnlineStorageDeals(context.Context) (bool, error)               //perm:admin
 	DealsSetConsiderOnlineStorageDeals(context.Context, bool) error              //perm:admin
 	DealsConsiderOnlineRetrievalDeals(context.Context) (bool, error)             //perm:admin
@@ -258,8 +284,6 @@ type StorageMiner interface {
 	DealsConsiderUnverifiedStorageDeals(context.Context) (bool, error)           //perm:admin
 	DealsSetConsiderUnverifiedStorageDeals(context.Context, bool) error          //perm:admin
 
-	StorageAddLocal(ctx context.Context, path string) error //perm:admin
-
 	PiecesListPieces(ctx context.Context) ([]cid.Cid, error)                                 //perm:read
 	PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error)                               //perm:read
 	PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) //perm:read
@@ -271,7 +295,7 @@ type StorageMiner interface {
 	// the path specified when calling CreateBackup is within the base path
 	CreateBackup(ctx context.Context, fpath string) error //perm:admin
 
-	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
+	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storiface.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
 
 	ComputeProof(ctx context.Context, ssi []builtin.ExtendedSectorInfo, rand abi.PoStRandomness, poStEpoch abi.ChainEpoch, nv abinetwork.Version) ([]builtin.PoStProof, error) //perm:read
 }

api/api_worker.go

@@ -7,11 +7,10 @@ import (
 	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/specs-actors/v5/actors/runtime/proof"
+	"github.com/filecoin-project/go-state-types/proof"
 
-	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-	"github.com/filecoin-project/specs-storage/storage"
+	"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
+	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
 //                       MODIFYING THE API INTERFACE
@@ -34,22 +33,22 @@ type Worker interface {
 	Info(context.Context) (storiface.WorkerInfo, error)                 //perm:admin
 
 	// storiface.WorkerCalls
-	DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (storiface.CallID, error)                                                                                      //perm:admin
-	AddPiece(ctx context.Context, sector storage.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (storiface.CallID, error)                    //perm:admin
-	SealPreCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error)                                                           //perm:admin
-	SealPreCommit2(ctx context.Context, sector storage.SectorRef, pc1o storage.PreCommit1Out) (storiface.CallID, error)                                                                                  //perm:admin
-	SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error) //perm:admin
-	SealCommit2(ctx context.Context, sector storage.SectorRef, c1o storage.Commit1Out) (storiface.CallID, error)                                                                                         //perm:admin
-	FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                                //perm:admin
-	FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                         //perm:admin
-	ReplicaUpdate(ctx context.Context, sector storage.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error)                                                                                       //perm:admin
-	ProveReplicaUpdate1(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error)                                                              //perm:admin
-	ProveReplicaUpdate2(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storage.ReplicaVanillaProofs) (storiface.CallID, error)                  //perm:admin
-	GenerateSectorKeyFromData(ctx context.Context, sector storage.SectorRef, commD cid.Cid) (storiface.CallID, error)                                                                                    //perm:admin
-	ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) (storiface.CallID, error)                                                                                 //perm:admin
-	MoveStorage(ctx context.Context, sector storage.SectorRef, types storiface.SectorFileType) (storiface.CallID, error)                                                                                 //perm:admin
-	UnsealPiece(context.Context, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error)                                           //perm:admin
-	Fetch(context.Context, storage.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error)                                                             //perm:admin
+	DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (storiface.CallID, error)                                                                                        //perm:admin
+	AddPiece(ctx context.Context, sector storiface.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (storiface.CallID, error)                    //perm:admin
+	SealPreCommit1(ctx context.Context, sector storiface.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error)                                                             //perm:admin
+	SealPreCommit2(ctx context.Context, sector storiface.SectorRef, pc1o storiface.PreCommit1Out) (storiface.CallID, error)                                                                                  //perm:admin
+	SealCommit1(ctx context.Context, sector storiface.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storiface.SectorCids) (storiface.CallID, error) //perm:admin
+	SealCommit2(ctx context.Context, sector storiface.SectorRef, c1o storiface.Commit1Out) (storiface.CallID, error)                                                                                         //perm:admin
+	FinalizeSector(ctx context.Context, sector storiface.SectorRef, keepUnsealed []storiface.Range) (storiface.CallID, error)                                                                                //perm:admin
+	FinalizeReplicaUpdate(ctx context.Context, sector storiface.SectorRef, keepUnsealed []storiface.Range) (storiface.CallID, error)                                                                         //perm:admin
+	ReplicaUpdate(ctx context.Context, sector storiface.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error)                                                                                         //perm:admin
+	ProveReplicaUpdate1(ctx context.Context, sector storiface.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error)                                                                //perm:admin
+	ProveReplicaUpdate2(ctx context.Context, sector storiface.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storiface.ReplicaVanillaProofs) (storiface.CallID, error)                  //perm:admin
+	GenerateSectorKeyFromData(ctx context.Context, sector storiface.SectorRef, commD cid.Cid) (storiface.CallID, error)                                                                                      //perm:admin
+	ReleaseUnsealed(ctx context.Context, sector storiface.SectorRef, safeToFree []storiface.Range) (storiface.CallID, error)                                                                                 //perm:admin
+	MoveStorage(ctx context.Context, sector storiface.SectorRef, types storiface.SectorFileType) (storiface.CallID, error)                                                                                   //perm:admin
+	UnsealPiece(context.Context, storiface.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error)                                             //perm:admin
+	Fetch(context.Context, storiface.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error)                                                               //perm:admin
 
 	GenerateWinningPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, randomness abi.PoStRandomness) ([]proof.PoStProof, error)                           //perm:admin
 	GenerateWindowPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, partitionIdx int, randomness abi.PoStRandomness) (storiface.WindowPoStResult, error) //perm:admin
@@ -60,7 +59,11 @@ type Worker interface {
 	// Storage / Other
 	Remove(ctx context.Context, sector abi.SectorID) error //perm:admin
 
-	StorageAddLocal(ctx context.Context, path string) error //perm:admin
+	StorageLocal(ctx context.Context) (map[storiface.ID]string, error)                   //perm:admin
+	StorageAddLocal(ctx context.Context, path string) error                              //perm:admin
+	StorageDetachLocal(ctx context.Context, path string) error                           //perm:admin
+	StorageDetachAll(ctx context.Context) error                                          //perm:admin
+	StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error //perm:admin
 
 	// SetEnabled marks the worker as enabled/disabled. Not that this setting
 	// may take a few seconds to propagate to task scheduler
@@ -77,6 +80,10 @@ type Worker interface {
 
 	// Like ProcessSession, but returns an error when worker is disabled
 	Session(context.Context) (uuid.UUID, error) //perm:admin
+
+	// Trigger shutdown
+	Shutdown(context.Context) error //perm:admin
+
 }
 
 var _ storiface.WorkerCalls = *new(Worker)

api/cbor_gen.go

@@ -8,12 +8,13 @@ import (
 	"math"
 	"sort"
 
-	abi "github.com/filecoin-project/go-state-types/abi"
-	market "github.com/filecoin-project/specs-actors/actors/builtin/market"
-	paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
 	cid "github.com/ipfs/go-cid"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	xerrors "golang.org/x/xerrors"
+
+	abi "github.com/filecoin-project/go-state-types/abi"
+	market "github.com/filecoin-project/go-state-types/builtin/v8/market"
+	paych "github.com/filecoin-project/go-state-types/builtin/v8/paych"
 )
 
 var _ = xerrors.Errorf
@@ -26,25 +27,26 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{163}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{163}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Channel (address.Address) (struct)
 	if len("Channel") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"Channel\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Channel"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Channel"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Channel")); err != nil {
 		return err
 	}
 
-	if err := t.Channel.MarshalCBOR(w); err != nil {
+	if err := t.Channel.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -53,14 +55,14 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"WaitSentinel\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("WaitSentinel"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("WaitSentinel"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("WaitSentinel")); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteCidBuf(scratch, w, t.WaitSentinel); err != nil {
+	if err := cbg.WriteCid(cw, t.WaitSentinel); err != nil {
 		return xerrors.Errorf("failed to write cid field t.WaitSentinel: %w", err)
 	}
 
@@ -69,7 +71,7 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Vouchers\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Vouchers"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Vouchers"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Vouchers")); err != nil {
@@ -80,27 +82,32 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Slice value in field t.Vouchers was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Vouchers))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Vouchers))); err != nil {
 		return err
 	}
 	for _, v := range t.Vouchers {
-		if err := v.MarshalCBOR(w); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
+func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = PaymentInfo{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -115,7 +122,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -129,7 +136,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				if err := t.Channel.UnmarshalCBOR(br); err != nil {
+				if err := t.Channel.UnmarshalCBOR(cr); err != nil {
 					return xerrors.Errorf("unmarshaling t.Channel: %w", err)
 				}
 
@@ -139,7 +146,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				c, err := cbg.ReadCid(br)
+				c, err := cbg.ReadCid(cr)
 				if err != nil {
 					return xerrors.Errorf("failed to read cid field t.WaitSentinel: %w", err)
 				}
@@ -150,7 +157,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 			// t.Vouchers ([]*paych.SignedVoucher) (slice)
 		case "Vouchers":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -170,7 +177,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 			for i := 0; i < int(extra); i++ {
 
 				var v paych.SignedVoucher
-				if err := v.UnmarshalCBOR(br); err != nil {
+				if err := v.UnmarshalCBOR(cr); err != nil {
 					return err
 				}
 
@@ -190,25 +197,26 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{163}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{163}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.SectorID (abi.SectorNumber) (uint64)
 	if len("SectorID") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"SectorID\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SectorID"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("SectorID"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("SectorID")); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.SectorID)); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.SectorID)); err != nil {
 		return err
 	}
 
@@ -217,14 +225,14 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Offset\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Offset"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Offset"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Offset")); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Offset)); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Offset)); err != nil {
 		return err
 	}
 
@@ -233,30 +241,35 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Size\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Size"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Size"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Size")); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Size)); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Size)); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
+func (t *SealedRef) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = SealedRef{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -271,7 +284,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -285,7 +298,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err = cr.ReadHeader()
 				if err != nil {
 					return err
 				}
@@ -300,7 +313,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err = cr.ReadHeader()
 				if err != nil {
 					return err
 				}
@@ -315,7 +328,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err = cr.ReadHeader()
 				if err != nil {
 					return err
 				}
@@ -339,18 +352,19 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{161}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{161}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Refs ([]api.SealedRef) (slice)
 	if len("Refs") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"Refs\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Refs"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Refs"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Refs")); err != nil {
@@ -361,27 +375,32 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Slice value in field t.Refs was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Refs))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Refs))); err != nil {
 		return err
 	}
 	for _, v := range t.Refs {
-		if err := v.MarshalCBOR(w); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
+func (t *SealedRefs) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = SealedRefs{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -396,7 +415,7 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -408,7 +427,7 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
 		// t.Refs ([]api.SealedRef) (slice)
 		case "Refs":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -428,7 +447,7 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
 			for i := 0; i < int(extra); i++ {
 
 				var v SealedRef
-				if err := v.UnmarshalCBOR(br); err != nil {
+				if err := v.UnmarshalCBOR(cr); err != nil {
 					return err
 				}
 
@@ -448,18 +467,19 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{162}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{162}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Value (abi.SealRandomness) (slice)
 	if len("Value") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"Value\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Value"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Value")); err != nil {
@@ -470,11 +490,11 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Byte array in field t.Value was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Value))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Value))); err != nil {
 		return err
 	}
 
-	if _, err := w.Write(t.Value[:]); err != nil {
+	if _, err := cw.Write(t.Value[:]); err != nil {
 		return err
 	}
 
@@ -483,7 +503,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Epoch\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Epoch"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Epoch")); err != nil {
@@ -491,27 +511,32 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 	}
 
 	if t.Epoch >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Epoch)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Epoch)); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Epoch-1)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Epoch-1)); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
+func (t *SealTicket) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = SealTicket{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -526,7 +551,7 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -538,7 +563,7 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
 		// t.Value (abi.SealRandomness) (slice)
 		case "Value":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -554,13 +579,13 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
 				t.Value = make([]uint8, extra)
 			}
 
-			if _, err := io.ReadFull(br, t.Value[:]); err != nil {
+			if _, err := io.ReadFull(cr, t.Value[:]); err != nil {
 				return err
 			}
 			// t.Epoch (abi.ChainEpoch) (int64)
 		case "Epoch":
 			{
-				maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err := cr.ReadHeader()
 				var extraI int64
 				if err != nil {
 					return err
@@ -597,18 +622,19 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{162}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{162}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Value (abi.InteractiveSealRandomness) (slice)
 	if len("Value") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"Value\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Value"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Value")); err != nil {
@@ -619,11 +645,11 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Byte array in field t.Value was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Value))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Value))); err != nil {
 		return err
 	}
 
-	if _, err := w.Write(t.Value[:]); err != nil {
+	if _, err := cw.Write(t.Value[:]); err != nil {
 		return err
 	}
 
@@ -632,7 +658,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Epoch\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Epoch"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Epoch")); err != nil {
@@ -640,27 +666,32 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 	}
 
 	if t.Epoch >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Epoch)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Epoch)); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Epoch-1)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Epoch-1)); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
+func (t *SealSeed) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = SealSeed{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -675,7 +706,7 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -687,7 +718,7 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
 		// t.Value (abi.InteractiveSealRandomness) (slice)
 		case "Value":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -703,13 +734,13 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
 				t.Value = make([]uint8, extra)
 			}
 
-			if _, err := io.ReadFull(br, t.Value[:]); err != nil {
+			if _, err := io.ReadFull(cr, t.Value[:]); err != nil {
 				return err
 			}
 			// t.Epoch (abi.ChainEpoch) (int64)
 		case "Epoch":
 			{
-				maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err := cr.ReadHeader()
 				var extraI int64
 				if err != nil {
 					return err
@@ -746,18 +777,19 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{165}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{165}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.PublishCid (cid.Cid) (struct)
 	if len("PublishCid") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"PublishCid\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PublishCid"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PublishCid"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("PublishCid")); err != nil {
@@ -765,11 +797,11 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
 	}
 
 	if t.PublishCid == nil {
-		if _, err := w.Write(cbg.CborNull); err != nil {
+		if _, err := cw.Write(cbg.CborNull); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteCidBuf(scratch, w, *t.PublishCid); err != nil {
+		if err := cbg.WriteCid(cw, *t.PublishCid); err != nil {
 			return xerrors.Errorf("failed to write cid field t.PublishCid: %w", err)
 		}
 	}
@@ -779,14 +811,14 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"DealID\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealID"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("DealID"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("DealID")); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.DealID)); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.DealID)); err != nil {
 		return err
 	}
 
@@ -795,14 +827,14 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"DealProposal\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealProposal"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("DealProposal"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("DealProposal")); err != nil {
 		return err
 	}
 
-	if err := t.DealProposal.MarshalCBOR(w); err != nil {
+	if err := t.DealProposal.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -811,14 +843,14 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"DealSchedule\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealSchedule"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("DealSchedule"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("DealSchedule")); err != nil {
 		return err
 	}
 
-	if err := t.DealSchedule.MarshalCBOR(w); err != nil {
+	if err := t.DealSchedule.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -827,7 +859,7 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"KeepUnsealed\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("KeepUnsealed"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("KeepUnsealed"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("KeepUnsealed")); err != nil {
@@ -840,16 +872,21 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
 	return nil
 }
 
-func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
+func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = PieceDealInfo{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -864,7 +901,7 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -878,16 +915,16 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				b, err := br.ReadByte()
+				b, err := cr.ReadByte()
 				if err != nil {
 					return err
 				}
 				if b != cbg.CborNull[0] {
-					if err := br.UnreadByte(); err != nil {
+					if err := cr.UnreadByte(); err != nil {
 						return err
 					}
 
-					c, err := cbg.ReadCid(br)
+					c, err := cbg.ReadCid(cr)
 					if err != nil {
 						return xerrors.Errorf("failed to read cid field t.PublishCid: %w", err)
 					}
@@ -901,7 +938,7 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err = cr.ReadHeader()
 				if err != nil {
 					return err
 				}
@@ -916,16 +953,16 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				b, err := br.ReadByte()
+				b, err := cr.ReadByte()
 				if err != nil {
 					return err
 				}
 				if b != cbg.CborNull[0] {
-					if err := br.UnreadByte(); err != nil {
+					if err := cr.UnreadByte(); err != nil {
 						return err
 					}
 					t.DealProposal = new(market.DealProposal)
-					if err := t.DealProposal.UnmarshalCBOR(br); err != nil {
+					if err := t.DealProposal.UnmarshalCBOR(cr); err != nil {
 						return xerrors.Errorf("unmarshaling t.DealProposal pointer: %w", err)
 					}
 				}
@@ -936,7 +973,7 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				if err := t.DealSchedule.UnmarshalCBOR(br); err != nil {
+				if err := t.DealSchedule.UnmarshalCBOR(cr); err != nil {
 					return xerrors.Errorf("unmarshaling t.DealSchedule: %w", err)
 				}
 
@@ -944,7 +981,7 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
 			// t.KeepUnsealed (bool) (bool)
 		case "KeepUnsealed":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -973,18 +1010,19 @@ func (t *DealSchedule) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{162}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{162}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.StartEpoch (abi.ChainEpoch) (int64)
 	if len("StartEpoch") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"StartEpoch\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("StartEpoch"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("StartEpoch"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("StartEpoch")); err != nil {
@@ -992,11 +1030,11 @@ func (t *DealSchedule) MarshalCBOR(w io.Writer) error {
 	}
 
 	if t.StartEpoch >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.StartEpoch)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.StartEpoch)); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.StartEpoch-1)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.StartEpoch-1)); err != nil {
 			return err
 		}
 	}
@@ -1006,7 +1044,7 @@ func (t *DealSchedule) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"EndEpoch\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("EndEpoch"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("EndEpoch"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("EndEpoch")); err != nil {
@@ -1014,27 +1052,32 @@ func (t *DealSchedule) MarshalCBOR(w io.Writer) error {
 	}
 
 	if t.EndEpoch >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.EndEpoch)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.EndEpoch)); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.EndEpoch-1)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.EndEpoch-1)); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error {
+func (t *DealSchedule) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = DealSchedule{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -1049,7 +1092,7 @@ func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -1061,7 +1104,7 @@ func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error {
 		// t.StartEpoch (abi.ChainEpoch) (int64)
 		case "StartEpoch":
 			{
-				maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err := cr.ReadHeader()
 				var extraI int64
 				if err != nil {
 					return err
@@ -1087,7 +1130,7 @@ func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error {
 			// t.EndEpoch (abi.ChainEpoch) (int64)
 		case "EndEpoch":
 			{
-				maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err := cr.ReadHeader()
 				var extraI int64
 				if err != nil {
 					return err

api/docgen-openrpc/cmd/docgen_openrpc.go

@@ -8,7 +8,6 @@ import (
 	"os"
 
 	"github.com/filecoin-project/lotus/api/docgen"
-
 	docgen_openrpc "github.com/filecoin-project/lotus/api/docgen-openrpc"
 )
 

api/docgen-openrpc/openrpc.go

@@ -8,10 +8,11 @@ import (
 
 	"github.com/alecthomas/jsonschema"
 	go_openrpc_reflect "github.com/etclabscore/go-openrpc-reflect"
-	"github.com/filecoin-project/lotus/api/docgen"
-	"github.com/filecoin-project/lotus/build"
 	"github.com/ipfs/go-cid"
 	meta_schema "github.com/open-rpc/meta-schema"
+
+	"github.com/filecoin-project/lotus/api/docgen"
+	"github.com/filecoin-project/lotus/build"
 )
 
 // schemaDictEntry represents a type association passed to the jsonschema reflector.

api/docgen/docgen.go

@@ -12,11 +12,11 @@ import (
 	"time"
 	"unicode"
 
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-bitfield"
 	"github.com/google/uuid"
+	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-graphsync"
+	textselector "github.com/ipld/go-ipld-selector-text-lite"
 	"github.com/libp2p/go-libp2p-core/metrics"
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
@@ -24,12 +24,12 @@ import (
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/multiformats/go-multiaddr"
 
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-bitfield"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
-	filestore "github.com/filecoin-project/go-fil-markets/filestore"
+	"github.com/filecoin-project/go-fil-markets/filestore"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-jsonrpc/auth"
-	textselector "github.com/ipld/go-ipld-selector-text-lite"
-
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/exitcode"
@@ -39,11 +39,11 @@ import (
 	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/repo/imports"
+	sealing "github.com/filecoin-project/lotus/storage/pipeline"
+	"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
+	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
 var ExampleValues = map[reflect.Type]interface{}{
@@ -95,6 +95,9 @@ func init() {
 	apiSelExample := api.Selector("Links/21/Hash/Links/42/Hash")
 	clientEvent := retrievalmarket.ClientEventDealAccepted
 
+	block := blocks.Block(&blocks.BasicBlock{})
+	ExampleValues[reflect.TypeOf(&block).Elem()] = block
+
 	addExample(bitfield.NewFromSet([]uint64{5}))
 	addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1)
 	addExample(abi.RegisteredPoStProof_StackedDrgWindow32GiBV1)
@@ -146,6 +149,10 @@ func init() {
 	addExample(map[string]api.MarketDeal{
 		"t026363": ExampleValue("init", reflect.TypeOf(api.MarketDeal{}), nil).(api.MarketDeal),
 	})
+	addExample(map[string]*api.MarketDeal{
+		"t026363": ExampleValue("init", reflect.TypeOf(&api.MarketDeal{}), nil).(*api.MarketDeal),
+	})
+
 	addExample(map[string]api.MarketBalance{
 		"t026363": ExampleValue("init", reflect.TypeOf(api.MarketBalance{}), nil).(api.MarketBalance),
 	})
@@ -188,6 +195,7 @@ func init() {
 	addExample(&si)
 	addExample(retrievalmarket.DealID(5))
 	addExample(abi.ActorID(1000))
+	addExample(map[string]cid.Cid{})
 	addExample(map[string][]api.SealedRef{
 		"98000": {
 			api.SealedRef{
@@ -264,6 +272,8 @@ func init() {
 			Read:   [storiface.FileTypes]uint{2, 3, 0},
 		},
 	})
+	storifaceid := storiface.ID("1399aa04-2625-44b1-bad4-bd07b59b22c4")
+	addExample(&storifaceid)
 
 	// worker specific
 	addExample(storiface.AcquireMove)
@@ -350,6 +360,10 @@ func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []r
 			i = &api.WorkerStruct{}
 			t = reflect.TypeOf(new(struct{ api.Worker })).Elem()
 			permStruct = append(permStruct, reflect.TypeOf(api.WorkerStruct{}.Internal))
+		case "Gateway":
+			i = &api.GatewayStruct{}
+			t = reflect.TypeOf(new(struct{ api.Gateway })).Elem()
+			permStruct = append(permStruct, reflect.TypeOf(api.GatewayStruct{}.Internal))
 		default:
 			panic("unknown type")
 		}

api/mocks/mock_full.go

@@ -10,6 +10,15 @@ import (
 	reflect "reflect"
 	time "time"
 
+	gomock "github.com/golang/mock/gomock"
+	uuid "github.com/google/uuid"
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
+	metrics "github.com/libp2p/go-libp2p-core/metrics"
+	network0 "github.com/libp2p/go-libp2p-core/network"
+	peer "github.com/libp2p/go-libp2p-core/peer"
+	protocol "github.com/libp2p/go-libp2p-core/protocol"
+
 	address "github.com/filecoin-project/go-address"
 	bitfield "github.com/filecoin-project/go-bitfield"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
@@ -17,25 +26,19 @@ import (
 	auth "github.com/filecoin-project/go-jsonrpc/auth"
 	abi "github.com/filecoin-project/go-state-types/abi"
 	big "github.com/filecoin-project/go-state-types/big"
+	miner "github.com/filecoin-project/go-state-types/builtin/v8/miner"
+	paych "github.com/filecoin-project/go-state-types/builtin/v8/paych"
 	crypto "github.com/filecoin-project/go-state-types/crypto"
 	dline "github.com/filecoin-project/go-state-types/dline"
 	network "github.com/filecoin-project/go-state-types/network"
+
 	api "github.com/filecoin-project/lotus/api"
 	apitypes "github.com/filecoin-project/lotus/api/types"
-	miner "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	miner0 "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	types "github.com/filecoin-project/lotus/chain/types"
 	alerting "github.com/filecoin-project/lotus/journal/alerting"
 	dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
 	imports "github.com/filecoin-project/lotus/node/repo/imports"
-	miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
-	paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
-	gomock "github.com/golang/mock/gomock"
-	uuid "github.com/google/uuid"
-	cid "github.com/ipfs/go-cid"
-	metrics "github.com/libp2p/go-libp2p-core/metrics"
-	network0 "github.com/libp2p/go-libp2p-core/network"
-	peer "github.com/libp2p/go-libp2p-core/peer"
-	protocol "github.com/libp2p/go-libp2p-core/protocol"
 )
 
 // MockFullNode is a mock of FullNode interface.
@@ -91,21 +94,6 @@ func (mr *MockFullNodeMockRecorder) AuthVerify(arg0, arg1 interface{}) *gomock.C
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AuthVerify", reflect.TypeOf((*MockFullNode)(nil).AuthVerify), arg0, arg1)
 }
 
-// BeaconGetEntry mocks base method.
-func (m *MockFullNode) BeaconGetEntry(arg0 context.Context, arg1 abi.ChainEpoch) (*types.BeaconEntry, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "BeaconGetEntry", arg0, arg1)
-	ret0, _ := ret[0].(*types.BeaconEntry)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// BeaconGetEntry indicates an expected call of BeaconGetEntry.
-func (mr *MockFullNodeMockRecorder) BeaconGetEntry(arg0, arg1 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BeaconGetEntry", reflect.TypeOf((*MockFullNode)(nil).BeaconGetEntry), arg0, arg1)
-}
-
 // ChainBlockstoreInfo mocks base method.
 func (m *MockFullNode) ChainBlockstoreInfo(arg0 context.Context) (map[string]interface{}, error) {
 	m.ctrl.T.Helper()
@@ -389,6 +377,34 @@ func (mr *MockFullNodeMockRecorder) ChainNotify(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainNotify", reflect.TypeOf((*MockFullNode)(nil).ChainNotify), arg0)
 }
 
+// ChainPrune mocks base method.
+func (m *MockFullNode) ChainPrune(arg0 context.Context, arg1 api.PruneOpts) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ChainPrune", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ChainPrune indicates an expected call of ChainPrune.
+func (mr *MockFullNodeMockRecorder) ChainPrune(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainPrune", reflect.TypeOf((*MockFullNode)(nil).ChainPrune), arg0, arg1)
+}
+
+// ChainPutObj mocks base method.
+func (m *MockFullNode) ChainPutObj(arg0 context.Context, arg1 blocks.Block) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ChainPutObj", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ChainPutObj indicates an expected call of ChainPutObj.
+func (mr *MockFullNodeMockRecorder) ChainPutObj(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainPutObj", reflect.TypeOf((*MockFullNode)(nil).ChainPutObj), arg0, arg1)
+}
+
 // ChainReadObj mocks base method.
 func (m *MockFullNode) ChainReadObj(arg0 context.Context, arg1 cid.Cid) ([]byte, error) {
 	m.ctrl.T.Helper()
@@ -2271,6 +2287,21 @@ func (mr *MockFullNodeMockRecorder) StateAccountKey(arg0, arg1, arg2 interface{}
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateAccountKey", reflect.TypeOf((*MockFullNode)(nil).StateAccountKey), arg0, arg1, arg2)
 }
 
+// StateActorCodeCIDs mocks base method.
+func (m *MockFullNode) StateActorCodeCIDs(arg0 context.Context, arg1 network.Version) (map[string]cid.Cid, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "StateActorCodeCIDs", arg0, arg1)
+	ret0, _ := ret[0].(map[string]cid.Cid)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// StateActorCodeCIDs indicates an expected call of StateActorCodeCIDs.
+func (mr *MockFullNodeMockRecorder) StateActorCodeCIDs(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateActorCodeCIDs", reflect.TypeOf((*MockFullNode)(nil).StateActorCodeCIDs), arg0, arg1)
+}
+
 // StateAllMinerFaults mocks base method.
 func (m *MockFullNode) StateAllMinerFaults(arg0 context.Context, arg1 abi.ChainEpoch, arg2 types.TipSetKey) ([]*api.Fault, error) {
 	m.ctrl.T.Helper()
@@ -2346,6 +2377,21 @@ func (mr *MockFullNodeMockRecorder) StateCompute(arg0, arg1, arg2, arg3 interfac
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateCompute", reflect.TypeOf((*MockFullNode)(nil).StateCompute), arg0, arg1, arg2, arg3)
 }
 
+// StateComputeDataCID mocks base method.
+func (m *MockFullNode) StateComputeDataCID(arg0 context.Context, arg1 address.Address, arg2 abi.RegisteredSealProof, arg3 []abi.DealID, arg4 types.TipSetKey) (cid.Cid, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "StateComputeDataCID", arg0, arg1, arg2, arg3, arg4)
+	ret0, _ := ret[0].(cid.Cid)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// StateComputeDataCID indicates an expected call of StateComputeDataCID.
+func (mr *MockFullNodeMockRecorder) StateComputeDataCID(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateComputeDataCID", reflect.TypeOf((*MockFullNode)(nil).StateComputeDataCID), arg0, arg1, arg2, arg3, arg4)
+}
+
 // StateDealProviderCollateralBounds mocks base method.
 func (m *MockFullNode) StateDealProviderCollateralBounds(arg0 context.Context, arg1 abi.PaddedPieceSize, arg2 bool, arg3 types.TipSetKey) (api.DealCollateralBounds, error) {
 	m.ctrl.T.Helper()
@@ -2406,6 +2452,21 @@ func (mr *MockFullNodeMockRecorder) StateGetActor(arg0, arg1, arg2 interface{})
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetActor", reflect.TypeOf((*MockFullNode)(nil).StateGetActor), arg0, arg1, arg2)
 }
 
+// StateGetBeaconEntry mocks base method.
+func (m *MockFullNode) StateGetBeaconEntry(arg0 context.Context, arg1 abi.ChainEpoch) (*types.BeaconEntry, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "StateGetBeaconEntry", arg0, arg1)
+	ret0, _ := ret[0].(*types.BeaconEntry)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// StateGetBeaconEntry indicates an expected call of StateGetBeaconEntry.
+func (mr *MockFullNodeMockRecorder) StateGetBeaconEntry(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetBeaconEntry", reflect.TypeOf((*MockFullNode)(nil).StateGetBeaconEntry), arg0, arg1)
+}
+
 // StateGetNetworkParams mocks base method.
 func (m *MockFullNode) StateGetNetworkParams(arg0 context.Context) (*api.NetworkParams, error) {
 	m.ctrl.T.Helper()
@@ -2542,10 +2603,10 @@ func (mr *MockFullNodeMockRecorder) StateMarketBalance(arg0, arg1, arg2 interfac
 }
 
 // StateMarketDeals mocks base method.
-func (m *MockFullNode) StateMarketDeals(arg0 context.Context, arg1 types.TipSetKey) (map[string]api.MarketDeal, error) {
+func (m *MockFullNode) StateMarketDeals(arg0 context.Context, arg1 types.TipSetKey) (map[string]*api.MarketDeal, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateMarketDeals", arg0, arg1)
-	ret0, _ := ret[0].(map[string]api.MarketDeal)
+	ret0, _ := ret[0].(map[string]*api.MarketDeal)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
@@ -2647,10 +2708,10 @@ func (mr *MockFullNodeMockRecorder) StateMinerFaults(arg0, arg1, arg2 interface{
 }
 
 // StateMinerInfo mocks base method.
-func (m *MockFullNode) StateMinerInfo(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (miner.MinerInfo, error) {
+func (m *MockFullNode) StateMinerInfo(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (api.MinerInfo, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateMinerInfo", arg0, arg1, arg2)
-	ret0, _ := ret[0].(miner.MinerInfo)
+	ret0, _ := ret[0].(api.MinerInfo)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
@@ -2662,7 +2723,7 @@ func (mr *MockFullNodeMockRecorder) StateMinerInfo(arg0, arg1, arg2 interface{})
 }
 
 // StateMinerInitialPledgeCollateral mocks base method.
-func (m *MockFullNode) StateMinerInitialPledgeCollateral(arg0 context.Context, arg1 address.Address, arg2 miner0.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
+func (m *MockFullNode) StateMinerInitialPledgeCollateral(arg0 context.Context, arg1 address.Address, arg2 miner.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateMinerInitialPledgeCollateral", arg0, arg1, arg2, arg3)
 	ret0, _ := ret[0].(big.Int)
@@ -2707,7 +2768,7 @@ func (mr *MockFullNodeMockRecorder) StateMinerPower(arg0, arg1, arg2 interface{}
 }
 
 // StateMinerPreCommitDepositForPower mocks base method.
-func (m *MockFullNode) StateMinerPreCommitDepositForPower(arg0 context.Context, arg1 address.Address, arg2 miner0.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
+func (m *MockFullNode) StateMinerPreCommitDepositForPower(arg0 context.Context, arg1 address.Address, arg2 miner.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateMinerPreCommitDepositForPower", arg0, arg1, arg2, arg3)
 	ret0, _ := ret[0].(big.Int)
@@ -2872,10 +2933,10 @@ func (mr *MockFullNodeMockRecorder) StateSearchMsg(arg0, arg1, arg2, arg3, arg4
 }
 
 // StateSectorExpiration mocks base method.
-func (m *MockFullNode) StateSectorExpiration(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner.SectorExpiration, error) {
+func (m *MockFullNode) StateSectorExpiration(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner0.SectorExpiration, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateSectorExpiration", arg0, arg1, arg2, arg3)
-	ret0, _ := ret[0].(*miner.SectorExpiration)
+	ret0, _ := ret[0].(*miner0.SectorExpiration)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
@@ -2902,10 +2963,10 @@ func (mr *MockFullNodeMockRecorder) StateSectorGetInfo(arg0, arg1, arg2, arg3 in
 }
 
 // StateSectorPartition mocks base method.
-func (m *MockFullNode) StateSectorPartition(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner.SectorLocation, error) {
+func (m *MockFullNode) StateSectorPartition(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner0.SectorLocation, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateSectorPartition", arg0, arg1, arg2, arg3)
-	ret0, _ := ret[0].(*miner.SectorLocation)
+	ret0, _ := ret[0].(*miner0.SectorLocation)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
@@ -2917,10 +2978,10 @@ func (mr *MockFullNodeMockRecorder) StateSectorPartition(arg0, arg1, arg2, arg3
 }
 
 // StateSectorPreCommitInfo mocks base method.
-func (m *MockFullNode) StateSectorPreCommitInfo(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) {
+func (m *MockFullNode) StateSectorPreCommitInfo(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner.SectorPreCommitOnChainInfo, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateSectorPreCommitInfo", arg0, arg1, arg2, arg3)
-	ret0, _ := ret[0].(miner.SectorPreCommitOnChainInfo)
+	ret0, _ := ret[0].(*miner.SectorPreCommitOnChainInfo)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }

api/types.go

@@ -5,19 +5,20 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/libp2p/go-libp2p-core/network"
-
-	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-graphsync"
-
+	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	ma "github.com/multiformats/go-multiaddr"
+
+	"github.com/filecoin-project/go-address"
+	datatransfer "github.com/filecoin-project/go-data-transfer"
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+	"github.com/filecoin-project/go-state-types/abi"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
 )
 
 type MultiaddrSlice []ma.Multiaddr
@@ -286,6 +287,20 @@ type ExportRef struct {
 	DealID       retrievalmarket.DealID
 }
 
+type MinerInfo struct {
+	Owner                      address.Address   // Must be an ID-address.
+	Worker                     address.Address   // Must be an ID-address.
+	NewWorker                  address.Address   // Must be an ID-address.
+	ControlAddresses           []address.Address // Must be an ID-addresses.
+	WorkerChangeEpoch          abi.ChainEpoch
+	PeerId                     *peer.ID
+	Multiaddrs                 []abi.Multiaddrs
+	WindowPoStProofType        abi.RegisteredPoStProof
+	SectorSize                 abi.SectorSize
+	WindowPoStPartitionSectors uint64
+	ConsensusFaultElapsed      abi.ChainEpoch
+}
+
 type NetworkParams struct {
 	NetworkName             dtypes.NetworkName
 	BlockDelaySecs          uint64

api/v0api/full.go

@@ -3,22 +3,26 @@ package v0api
 import (
 	"context"
 
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	textselector "github.com/ipld/go-ipld-selector-text-lite"
+	"github.com/libp2p/go-libp2p-core/peer"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/builtin/v8/miner"
+	"github.com/filecoin-project/go-state-types/builtin/v8/paych"
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/dline"
-	"github.com/ipfs/go-cid"
-	textselector "github.com/ipld/go-ipld-selector-text-lite"
-	"github.com/libp2p/go-libp2p-core/peer"
+	abinetwork "github.com/filecoin-project/go-state-types/network"
 
 	"github.com/filecoin-project/lotus/api"
 	apitypes "github.com/filecoin-project/lotus/api/types"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
+	lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
 	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
@@ -109,6 +113,9 @@ type FullNode interface {
 	// ChainDeleteObj deletes node referenced by the given CID
 	ChainDeleteObj(context.Context, cid.Cid) error //perm:admin
 
+	// ChainPutObj puts and object into the blockstore
+	ChainPutObj(context.Context, blocks.Block) error
+
 	// ChainHasObj checks if a given CID exists in the chain blockstore.
 	ChainHasObj(context.Context, cid.Cid) (bool, error) //perm:read
 
@@ -379,7 +386,7 @@ type FullNode interface {
 	StateCall(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) //perm:read
 	// StateReplay replays a given message, assuming it was included in a block in the specified tipset.
 	//
-	// If a tipset key is provided, and a replacing message is found on chain,
+	// If a tipset key is provided, and a replacing message is not found on chain,
 	// the method will return an error saying that the message wasn't found
 	//
 	// If no tipset key is provided, the appropriate tipset is looked up, and if
@@ -416,7 +423,7 @@ type FullNode interface {
 	// StateMinerPower returns the power of the indicated miner
 	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) //perm:read
 	// StateMinerInfo returns info about the indicated miner
-	StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) //perm:read
+	StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) //perm:read
 	// StateMinerDeadlines returns all the proving deadlines for the given miner
 	StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]api.Deadline, error) //perm:read
 	// StateMinerPartitions returns all partitions in the specified deadline
@@ -442,9 +449,9 @@ type FullNode interface {
 	// expiration epoch
 	StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) //perm:read
 	// StateSectorExpiration returns epoch at which given sector will expire
-	StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error) //perm:read
+	StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*lminer.SectorExpiration, error) //perm:read
 	// StateSectorPartition finds deadline/partition with the specified sector
-	StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) //perm:read
+	StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*lminer.SectorLocation, error) //perm:read
 	// StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed
 	//
 	// NOTE: If a replacing message is found on chain, this method will return
@@ -521,7 +528,7 @@ type FullNode interface {
 	// StateMarketParticipants returns the Escrow and Locked balances of every participant in the Storage Market
 	StateMarketParticipants(context.Context, types.TipSetKey) (map[string]api.MarketBalance, error) //perm:read
 	// StateMarketDeals returns information about every deal in the Storage Market
-	StateMarketDeals(context.Context, types.TipSetKey) (map[string]api.MarketDeal, error) //perm:read
+	StateMarketDeals(context.Context, types.TipSetKey) (map[string]*api.MarketDeal, error) //perm:read
 	// StateMarketStorageDeal returns information about the indicated deal
 	StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error) //perm:read
 	// StateLookupID retrieves the ID address of the given address
@@ -598,6 +605,8 @@ type FullNode interface {
 	StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error) //perm:read
 	// StateNetworkVersion returns the network version at the given tipset
 	StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) //perm:read
+	// StateActorCodeCIDs returns the CIDs of all the builtin actors for the given network version
+	StateActorCodeCIDs(context.Context, abinetwork.Version) (map[string]cid.Cid, error) //perm:read
 
 	// StateGetRandomnessFromTickets is used to sample the chain for randomness.
 	StateGetRandomnessFromTickets(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read

api/v0api/gateway.go

@@ -3,15 +3,16 @@ package v0api
 import (
 	"context"
 
+	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/builtin/v8/miner"
 	"github.com/filecoin-project/go-state-types/dline"
 	abinetwork "github.com/filecoin-project/go-state-types/network"
 
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
@@ -34,6 +35,7 @@ import (
 
 type Gateway interface {
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
+	ChainPutObj(context.Context, blocks.Block) error
 	ChainHead(ctx context.Context) (*types.TipSet, error)
 	ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error)
 	ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
@@ -54,7 +56,7 @@ type Gateway interface {
 	StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
 	StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error)
 	StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error)
-	StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error)
+	StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (api.MinerInfo, error)
 	StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
 	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)
 	StateNetworkVersion(context.Context, types.TipSetKey) (abinetwork.Version, error)

api/v0api/permissioned.go

@@ -2,6 +2,7 @@ package v0api
 
 import (
 	"github.com/filecoin-project/go-jsonrpc/auth"
+
 	"github.com/filecoin-project/lotus/api"
 )
 

api/v0api/proxy_gen.go

@@ -5,26 +5,30 @@ package v0api
 import (
 	"context"
 
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	"github.com/libp2p/go-libp2p-core/peer"
+	"golang.org/x/xerrors"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/builtin/v8/miner"
+	"github.com/filecoin-project/go-state-types/builtin/v8/paych"
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/dline"
 	abinetwork "github.com/filecoin-project/go-state-types/network"
+
 	"github.com/filecoin-project/lotus/api"
 	apitypes "github.com/filecoin-project/lotus/api/types"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
+	lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
 	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/repo/imports"
-	"github.com/ipfs/go-cid"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"golang.org/x/xerrors"
 )
 
 var ErrNotSupported = xerrors.New("method not supported")
@@ -73,6 +77,8 @@ type FullNodeStruct struct {
 
 		ChainNotify func(p0 context.Context) (<-chan []*api.HeadChange, error) `perm:"read"`
 
+		ChainPutObj func(p0 context.Context, p1 blocks.Block) error ``
+
 		ChainReadObj func(p0 context.Context, p1 cid.Cid) ([]byte, error) `perm:"read"`
 
 		ChainSetHead func(p0 context.Context, p1 types.TipSetKey) error `perm:"admin"`
@@ -251,6 +257,8 @@ type FullNodeStruct struct {
 
 		StateAccountKey func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) `perm:"read"`
 
+		StateActorCodeCIDs func(p0 context.Context, p1 abinetwork.Version) (map[string]cid.Cid, error) `perm:"read"`
+
 		StateAllMinerFaults func(p0 context.Context, p1 abi.ChainEpoch, p2 types.TipSetKey) ([]*api.Fault, error) `perm:"read"`
 
 		StateCall func(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*api.InvocResult, error) `perm:"read"`
@@ -285,7 +293,7 @@ type FullNodeStruct struct {
 
 		StateMarketBalance func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (api.MarketBalance, error) `perm:"read"`
 
-		StateMarketDeals func(p0 context.Context, p1 types.TipSetKey) (map[string]api.MarketDeal, error) `perm:"read"`
+		StateMarketDeals func(p0 context.Context, p1 types.TipSetKey) (map[string]*api.MarketDeal, error) `perm:"read"`
 
 		StateMarketParticipants func(p0 context.Context, p1 types.TipSetKey) (map[string]api.MarketBalance, error) `perm:"read"`
 
@@ -299,7 +307,7 @@ type FullNodeStruct struct {
 
 		StateMinerFaults func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (bitfield.BitField, error) `perm:"read"`
 
-		StateMinerInfo func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (miner.MinerInfo, error) `perm:"read"`
+		StateMinerInfo func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (api.MinerInfo, error) `perm:"read"`
 
 		StateMinerInitialPledgeCollateral func(p0 context.Context, p1 address.Address, p2 miner.SectorPreCommitInfo, p3 types.TipSetKey) (types.BigInt, error) `perm:"read"`
 
@@ -331,11 +339,11 @@ type FullNodeStruct struct {
 
 		StateSearchMsgLimited func(p0 context.Context, p1 cid.Cid, p2 abi.ChainEpoch) (*api.MsgLookup, error) `perm:"read"`
 
-		StateSectorExpiration func(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*miner.SectorExpiration, error) `perm:"read"`
+		StateSectorExpiration func(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*lminer.SectorExpiration, error) `perm:"read"`
 
 		StateSectorGetInfo func(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*miner.SectorOnChainInfo, error) `perm:"read"`
 
-		StateSectorPartition func(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*miner.SectorLocation, error) `perm:"read"`
+		StateSectorPartition func(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*lminer.SectorLocation, error) `perm:"read"`
 
 		StateSectorPreCommitInfo func(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) `perm:"read"`
 
@@ -419,6 +427,8 @@ type GatewayStruct struct {
 
 		ChainNotify func(p0 context.Context) (<-chan []*api.HeadChange, error) ``
 
+		ChainPutObj func(p0 context.Context, p1 blocks.Block) error ``
+
 		ChainReadObj func(p0 context.Context, p1 cid.Cid) ([]byte, error) ``
 
 		GasEstimateMessageGas func(p0 context.Context, p1 *types.Message, p2 *api.MessageSendSpec, p3 types.TipSetKey) (*types.Message, error) ``
@@ -447,7 +457,7 @@ type GatewayStruct struct {
 
 		StateMarketStorageDeal func(p0 context.Context, p1 abi.DealID, p2 types.TipSetKey) (*api.MarketDeal, error) ``
 
-		StateMinerInfo func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (miner.MinerInfo, error) ``
+		StateMinerInfo func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (api.MinerInfo, error) ``
 
 		StateMinerPower func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*api.MinerPower, error) ``
 
@@ -681,6 +691,17 @@ func (s *FullNodeStub) ChainNotify(p0 context.Context) (<-chan []*api.HeadChange
 	return nil, ErrNotSupported
 }
 
+func (s *FullNodeStruct) ChainPutObj(p0 context.Context, p1 blocks.Block) error {
+	if s.Internal.ChainPutObj == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.ChainPutObj(p0, p1)
+}
+
+func (s *FullNodeStub) ChainPutObj(p0 context.Context, p1 blocks.Block) error {
+	return ErrNotSupported
+}
+
 func (s *FullNodeStruct) ChainReadObj(p0 context.Context, p1 cid.Cid) ([]byte, error) {
 	if s.Internal.ChainReadObj == nil {
 		return *new([]byte), ErrNotSupported
@@ -1660,6 +1681,17 @@ func (s *FullNodeStub) StateAccountKey(p0 context.Context, p1 address.Address, p
 	return *new(address.Address), ErrNotSupported
 }
 
+func (s *FullNodeStruct) StateActorCodeCIDs(p0 context.Context, p1 abinetwork.Version) (map[string]cid.Cid, error) {
+	if s.Internal.StateActorCodeCIDs == nil {
+		return *new(map[string]cid.Cid), ErrNotSupported
+	}
+	return s.Internal.StateActorCodeCIDs(p0, p1)
+}
+
+func (s *FullNodeStub) StateActorCodeCIDs(p0 context.Context, p1 abinetwork.Version) (map[string]cid.Cid, error) {
+	return *new(map[string]cid.Cid), ErrNotSupported
+}
+
 func (s *FullNodeStruct) StateAllMinerFaults(p0 context.Context, p1 abi.ChainEpoch, p2 types.TipSetKey) ([]*api.Fault, error) {
 	if s.Internal.StateAllMinerFaults == nil {
 		return *new([]*api.Fault), ErrNotSupported
@@ -1847,15 +1879,15 @@ func (s *FullNodeStub) StateMarketBalance(p0 context.Context, p1 address.Address
 	return *new(api.MarketBalance), ErrNotSupported
 }
 
-func (s *FullNodeStruct) StateMarketDeals(p0 context.Context, p1 types.TipSetKey) (map[string]api.MarketDeal, error) {
+func (s *FullNodeStruct) StateMarketDeals(p0 context.Context, p1 types.TipSetKey) (map[string]*api.MarketDeal, error) {
 	if s.Internal.StateMarketDeals == nil {
-		return *new(map[string]api.MarketDeal), ErrNotSupported
+		return *new(map[string]*api.MarketDeal), ErrNotSupported
 	}
 	return s.Internal.StateMarketDeals(p0, p1)
 }
 
-func (s *FullNodeStub) StateMarketDeals(p0 context.Context, p1 types.TipSetKey) (map[string]api.MarketDeal, error) {
-	return *new(map[string]api.MarketDeal), ErrNotSupported
+func (s *FullNodeStub) StateMarketDeals(p0 context.Context, p1 types.TipSetKey) (map[string]*api.MarketDeal, error) {
+	return *new(map[string]*api.MarketDeal), ErrNotSupported
 }
 
 func (s *FullNodeStruct) StateMarketParticipants(p0 context.Context, p1 types.TipSetKey) (map[string]api.MarketBalance, error) {
@@ -1924,15 +1956,15 @@ func (s *FullNodeStub) StateMinerFaults(p0 context.Context, p1 address.Address,
 	return *new(bitfield.BitField), ErrNotSupported
 }
 
-func (s *FullNodeStruct) StateMinerInfo(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (miner.MinerInfo, error) {
+func (s *FullNodeStruct) StateMinerInfo(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (api.MinerInfo, error) {
 	if s.Internal.StateMinerInfo == nil {
-		return *new(miner.MinerInfo), ErrNotSupported
+		return *new(api.MinerInfo), ErrNotSupported
 	}
 	return s.Internal.StateMinerInfo(p0, p1, p2)
 }
 
-func (s *FullNodeStub) StateMinerInfo(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (miner.MinerInfo, error) {
-	return *new(miner.MinerInfo), ErrNotSupported
+func (s *FullNodeStub) StateMinerInfo(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (api.MinerInfo, error) {
+	return *new(api.MinerInfo), ErrNotSupported
 }
 
 func (s *FullNodeStruct) StateMinerInitialPledgeCollateral(p0 context.Context, p1 address.Address, p2 miner.SectorPreCommitInfo, p3 types.TipSetKey) (types.BigInt, error) {
@@ -2100,14 +2132,14 @@ func (s *FullNodeStub) StateSearchMsgLimited(p0 context.Context, p1 cid.Cid, p2
 	return nil, ErrNotSupported
 }
 
-func (s *FullNodeStruct) StateSectorExpiration(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*miner.SectorExpiration, error) {
+func (s *FullNodeStruct) StateSectorExpiration(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*lminer.SectorExpiration, error) {
 	if s.Internal.StateSectorExpiration == nil {
 		return nil, ErrNotSupported
 	}
 	return s.Internal.StateSectorExpiration(p0, p1, p2, p3)
 }
 
-func (s *FullNodeStub) StateSectorExpiration(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*miner.SectorExpiration, error) {
+func (s *FullNodeStub) StateSectorExpiration(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*lminer.SectorExpiration, error) {
 	return nil, ErrNotSupported
 }
 
@@ -2122,14 +2154,14 @@ func (s *FullNodeStub) StateSectorGetInfo(p0 context.Context, p1 address.Address
 	return nil, ErrNotSupported
 }
 
-func (s *FullNodeStruct) StateSectorPartition(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*miner.SectorLocation, error) {
+func (s *FullNodeStruct) StateSectorPartition(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*lminer.SectorLocation, error) {
 	if s.Internal.StateSectorPartition == nil {
 		return nil, ErrNotSupported
 	}
 	return s.Internal.StateSectorPartition(p0, p1, p2, p3)
 }
 
-func (s *FullNodeStub) StateSectorPartition(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*miner.SectorLocation, error) {
+func (s *FullNodeStub) StateSectorPartition(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*lminer.SectorLocation, error) {
 	return nil, ErrNotSupported
 }
 
@@ -2529,6 +2561,17 @@ func (s *GatewayStub) ChainNotify(p0 context.Context) (<-chan []*api.HeadChange,
 	return nil, ErrNotSupported
 }
 
+func (s *GatewayStruct) ChainPutObj(p0 context.Context, p1 blocks.Block) error {
+	if s.Internal.ChainPutObj == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.ChainPutObj(p0, p1)
+}
+
+func (s *GatewayStub) ChainPutObj(p0 context.Context, p1 blocks.Block) error {
+	return ErrNotSupported
+}
+
 func (s *GatewayStruct) ChainReadObj(p0 context.Context, p1 cid.Cid) ([]byte, error) {
 	if s.Internal.ChainReadObj == nil {
 		return *new([]byte), ErrNotSupported
@@ -2683,15 +2726,15 @@ func (s *GatewayStub) StateMarketStorageDeal(p0 context.Context, p1 abi.DealID,
 	return nil, ErrNotSupported
 }
 
-func (s *GatewayStruct) StateMinerInfo(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (miner.MinerInfo, error) {
+func (s *GatewayStruct) StateMinerInfo(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (api.MinerInfo, error) {
 	if s.Internal.StateMinerInfo == nil {
-		return *new(miner.MinerInfo), ErrNotSupported
+		return *new(api.MinerInfo), ErrNotSupported
 	}
 	return s.Internal.StateMinerInfo(p0, p1, p2)
 }
 
-func (s *GatewayStub) StateMinerInfo(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (miner.MinerInfo, error) {
-	return *new(miner.MinerInfo), ErrNotSupported
+func (s *GatewayStub) StateMinerInfo(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (api.MinerInfo, error) {
+	return *new(api.MinerInfo), ErrNotSupported
 }
 
 func (s *GatewayStruct) StateMinerPower(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*api.MinerPower, error) {

api/v0api/v0mocks/mock_full.go

@@ -9,6 +9,15 @@ import (
 	reflect "reflect"
 	time "time"
 
+	gomock "github.com/golang/mock/gomock"
+	uuid "github.com/google/uuid"
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
+	metrics "github.com/libp2p/go-libp2p-core/metrics"
+	network0 "github.com/libp2p/go-libp2p-core/network"
+	peer "github.com/libp2p/go-libp2p-core/peer"
+	protocol "github.com/libp2p/go-libp2p-core/protocol"
+
 	address "github.com/filecoin-project/go-address"
 	bitfield "github.com/filecoin-project/go-bitfield"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
@@ -17,27 +26,21 @@ import (
 	auth "github.com/filecoin-project/go-jsonrpc/auth"
 	abi "github.com/filecoin-project/go-state-types/abi"
 	big "github.com/filecoin-project/go-state-types/big"
+	miner "github.com/filecoin-project/go-state-types/builtin/v8/miner"
+	paych "github.com/filecoin-project/go-state-types/builtin/v8/paych"
 	crypto "github.com/filecoin-project/go-state-types/crypto"
 	dline "github.com/filecoin-project/go-state-types/dline"
 	network "github.com/filecoin-project/go-state-types/network"
+
 	api "github.com/filecoin-project/lotus/api"
 	apitypes "github.com/filecoin-project/lotus/api/types"
 	v0api "github.com/filecoin-project/lotus/api/v0api"
-	miner "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	miner0 "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	types "github.com/filecoin-project/lotus/chain/types"
 	alerting "github.com/filecoin-project/lotus/journal/alerting"
 	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 	dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
 	imports "github.com/filecoin-project/lotus/node/repo/imports"
-	miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
-	paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
-	gomock "github.com/golang/mock/gomock"
-	uuid "github.com/google/uuid"
-	cid "github.com/ipfs/go-cid"
-	metrics "github.com/libp2p/go-libp2p-core/metrics"
-	network0 "github.com/libp2p/go-libp2p-core/network"
-	peer "github.com/libp2p/go-libp2p-core/peer"
-	protocol "github.com/libp2p/go-libp2p-core/protocol"
 )
 
 // MockFullNode is a mock of FullNode interface.
@@ -377,6 +380,20 @@ func (mr *MockFullNodeMockRecorder) ChainNotify(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainNotify", reflect.TypeOf((*MockFullNode)(nil).ChainNotify), arg0)
 }
 
+// ChainPutObj mocks base method.
+func (m *MockFullNode) ChainPutObj(arg0 context.Context, arg1 blocks.Block) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ChainPutObj", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ChainPutObj indicates an expected call of ChainPutObj.
+func (mr *MockFullNodeMockRecorder) ChainPutObj(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainPutObj", reflect.TypeOf((*MockFullNode)(nil).ChainPutObj), arg0, arg1)
+}
+
 // ChainReadObj mocks base method.
 func (m *MockFullNode) ChainReadObj(arg0 context.Context, arg1 cid.Cid) ([]byte, error) {
 	m.ctrl.T.Helper()
@@ -2155,6 +2172,21 @@ func (mr *MockFullNodeMockRecorder) StateAccountKey(arg0, arg1, arg2 interface{}
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateAccountKey", reflect.TypeOf((*MockFullNode)(nil).StateAccountKey), arg0, arg1, arg2)
 }
 
+// StateActorCodeCIDs mocks base method.
+func (m *MockFullNode) StateActorCodeCIDs(arg0 context.Context, arg1 network.Version) (map[string]cid.Cid, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "StateActorCodeCIDs", arg0, arg1)
+	ret0, _ := ret[0].(map[string]cid.Cid)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// StateActorCodeCIDs indicates an expected call of StateActorCodeCIDs.
+func (mr *MockFullNodeMockRecorder) StateActorCodeCIDs(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateActorCodeCIDs", reflect.TypeOf((*MockFullNode)(nil).StateActorCodeCIDs), arg0, arg1)
+}
+
 // StateAllMinerFaults mocks base method.
 func (m *MockFullNode) StateAllMinerFaults(arg0 context.Context, arg1 abi.ChainEpoch, arg2 types.TipSetKey) ([]*api.Fault, error) {
 	m.ctrl.T.Helper()
@@ -2411,10 +2443,10 @@ func (mr *MockFullNodeMockRecorder) StateMarketBalance(arg0, arg1, arg2 interfac
 }
 
 // StateMarketDeals mocks base method.
-func (m *MockFullNode) StateMarketDeals(arg0 context.Context, arg1 types.TipSetKey) (map[string]api.MarketDeal, error) {
+func (m *MockFullNode) StateMarketDeals(arg0 context.Context, arg1 types.TipSetKey) (map[string]*api.MarketDeal, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateMarketDeals", arg0, arg1)
-	ret0, _ := ret[0].(map[string]api.MarketDeal)
+	ret0, _ := ret[0].(map[string]*api.MarketDeal)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
@@ -2516,10 +2548,10 @@ func (mr *MockFullNodeMockRecorder) StateMinerFaults(arg0, arg1, arg2 interface{
 }
 
 // StateMinerInfo mocks base method.
-func (m *MockFullNode) StateMinerInfo(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (miner.MinerInfo, error) {
+func (m *MockFullNode) StateMinerInfo(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (api.MinerInfo, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateMinerInfo", arg0, arg1, arg2)
-	ret0, _ := ret[0].(miner.MinerInfo)
+	ret0, _ := ret[0].(api.MinerInfo)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
@@ -2531,7 +2563,7 @@ func (mr *MockFullNodeMockRecorder) StateMinerInfo(arg0, arg1, arg2 interface{})
 }
 
 // StateMinerInitialPledgeCollateral mocks base method.
-func (m *MockFullNode) StateMinerInitialPledgeCollateral(arg0 context.Context, arg1 address.Address, arg2 miner0.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
+func (m *MockFullNode) StateMinerInitialPledgeCollateral(arg0 context.Context, arg1 address.Address, arg2 miner.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateMinerInitialPledgeCollateral", arg0, arg1, arg2, arg3)
 	ret0, _ := ret[0].(big.Int)
@@ -2576,7 +2608,7 @@ func (mr *MockFullNodeMockRecorder) StateMinerPower(arg0, arg1, arg2 interface{}
 }
 
 // StateMinerPreCommitDepositForPower mocks base method.
-func (m *MockFullNode) StateMinerPreCommitDepositForPower(arg0 context.Context, arg1 address.Address, arg2 miner0.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
+func (m *MockFullNode) StateMinerPreCommitDepositForPower(arg0 context.Context, arg1 address.Address, arg2 miner.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateMinerPreCommitDepositForPower", arg0, arg1, arg2, arg3)
 	ret0, _ := ret[0].(big.Int)
@@ -2756,10 +2788,10 @@ func (mr *MockFullNodeMockRecorder) StateSearchMsgLimited(arg0, arg1, arg2 inter
 }
 
 // StateSectorExpiration mocks base method.
-func (m *MockFullNode) StateSectorExpiration(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner.SectorExpiration, error) {
+func (m *MockFullNode) StateSectorExpiration(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner0.SectorExpiration, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateSectorExpiration", arg0, arg1, arg2, arg3)
-	ret0, _ := ret[0].(*miner.SectorExpiration)
+	ret0, _ := ret[0].(*miner0.SectorExpiration)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
@@ -2786,10 +2818,10 @@ func (mr *MockFullNodeMockRecorder) StateSectorGetInfo(arg0, arg1, arg2, arg3 in
 }
 
 // StateSectorPartition mocks base method.
-func (m *MockFullNode) StateSectorPartition(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner.SectorLocation, error) {
+func (m *MockFullNode) StateSectorPartition(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner0.SectorLocation, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateSectorPartition", arg0, arg1, arg2, arg3)
-	ret0, _ := ret[0].(*miner.SectorLocation)
+	ret0, _ := ret[0].(*miner0.SectorLocation)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }

api/v0api/v1_wrapper.go

@@ -3,30 +3,40 @@ package v0api
 import (
 	"context"
 
-	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/ipfs/go-cid"
 	"github.com/libp2p/go-libp2p-core/peer"
-
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/go-state-types/crypto"
-	marketevents "github.com/filecoin-project/lotus/markets/loggers"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/lotus/chain/types"
 	"golang.org/x/xerrors"
 
-	"github.com/ipfs/go-cid"
-
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/builtin/v8/miner"
+	"github.com/filecoin-project/go-state-types/crypto"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/v1api"
+	"github.com/filecoin-project/lotus/chain/types"
+	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 )
 
 type WrapperV1Full struct {
 	v1api.FullNode
 }
 
+func (w *WrapperV1Full) StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, s abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) {
+	pi, err := w.FullNode.StateSectorPreCommitInfo(ctx, maddr, s, tsk)
+	if err != nil {
+		return miner.SectorPreCommitOnChainInfo{}, err
+	}
+	if pi == nil {
+		return miner.SectorPreCommitOnChainInfo{}, xerrors.Errorf("precommit info does not exist")
+	}
+
+	return *pi, nil
+}
+
 func (w *WrapperV1Full) StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error) {
 	return w.FullNode.StateSearchMsg(ctx, types.EmptyTSK, msg, api.LookbackNoLimit, true)
 }
@@ -352,4 +362,8 @@ func (w *WrapperV1Full) ClientQueryAsk(ctx context.Context, p peer.ID, miner add
 	return a.Response, nil
 }
 
+func (w *WrapperV1Full) BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) {
+	return w.StateGetBeaconEntry(ctx, epoch)
+}
+
 var _ FullNode = &WrapperV1Full{}

api/version.go

@@ -3,7 +3,7 @@ package api
 import (
 	"fmt"
 
-	xerrors "golang.org/x/xerrors"
+	"golang.org/x/xerrors"
 )
 
 type Version uint32
@@ -55,7 +55,7 @@ func VersionForType(nodeType NodeType) (Version, error) {
 // semver versions of the rpc api exposed
 var (
 	FullAPIVersion0 = newVer(1, 5, 0)
-	FullAPIVersion1 = newVer(2, 2, 0)
+	FullAPIVersion1 = newVer(2, 3, 0)
 
 	MinerAPIVersion0  = newVer(1, 5, 0)
 	WorkerAPIVersion0 = newVer(1, 6, 0)

blockstore/api.go

@@ -11,6 +11,7 @@ import (
 type ChainIO interface {
 	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
+	ChainPutObj(context.Context, blocks.Block) error
 }
 
 type apiBlockstore struct {
@@ -49,12 +50,18 @@ func (a *apiBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
 	return len(bb), nil
 }
 
-func (a *apiBlockstore) Put(context.Context, blocks.Block) error {
-	return xerrors.New("not supported")
+func (a *apiBlockstore) Put(ctx context.Context, block blocks.Block) error {
+	return a.api.ChainPutObj(ctx, block)
 }
 
-func (a *apiBlockstore) PutMany(context.Context, []blocks.Block) error {
-	return xerrors.New("not supported")
+func (a *apiBlockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
+	for _, block := range blocks {
+		err := a.api.ChainPutObj(ctx, block)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 func (a *apiBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {

blockstore/autobatch.go

@@ -5,10 +5,10 @@ import (
 	"sync"
 	"time"
 
-	"golang.org/x/xerrors"
-
 	block "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
+	"golang.org/x/xerrors"
 )
 
 // autolog is a logger for the autobatching blockstore. It is subscoped from the
@@ -176,7 +176,7 @@ func (bs *AutobatchBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block,
 		return blk, nil
 	}
 
-	if err != ErrNotFound {
+	if !ipld.IsNotFound(err) {
 		return blk, err
 	}
 
@@ -214,7 +214,7 @@ func (bs *AutobatchBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error)
 	if err == nil {
 		return true, nil
 	}
-	if err == ErrNotFound {
+	if ipld.IsNotFound(err) {
 		return false, nil
 	}
 

blockstore/badger/blockstore.go

@@ -13,13 +13,13 @@ import (
 	"github.com/dgraph-io/badger/v2"
 	"github.com/dgraph-io/badger/v2/options"
 	"github.com/dgraph-io/badger/v2/pb"
-	"github.com/multiformats/go-base32"
-	"go.uber.org/zap"
-
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
 	logger "github.com/ipfs/go-log/v2"
 	pool "github.com/libp2p/go-buffer-pool"
+	"github.com/multiformats/go-base32"
+	"go.uber.org/zap"
 
 	"github.com/filecoin-project/lotus/blockstore"
 )
@@ -544,7 +544,7 @@ func (b *Blockstore) View(ctx context.Context, cid cid.Cid, fn func([]byte) erro
 		case nil:
 			return item.Value(fn)
 		case badger.ErrKeyNotFound:
-			return blockstore.ErrNotFound
+			return ipld.ErrNotFound{Cid: cid}
 		default:
 			return fmt.Errorf("failed to view block from badger blockstore: %w", err)
 		}
@@ -584,7 +584,7 @@ func (b *Blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 // Get implements Blockstore.Get.
 func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
 	if !cid.Defined() {
-		return nil, blockstore.ErrNotFound
+		return nil, ipld.ErrNotFound{Cid: cid}
 	}
 
 	if err := b.access(); err != nil {
@@ -607,7 +607,7 @@ func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
 			val, err = item.ValueCopy(nil)
 			return err
 		case badger.ErrKeyNotFound:
-			return blockstore.ErrNotFound
+			return ipld.ErrNotFound{Cid: cid}
 		default:
 			return fmt.Errorf("failed to get block from badger blockstore: %w", err)
 		}
@@ -639,7 +639,7 @@ func (b *Blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
 		case nil:
 			size = int(item.ValueSize())
 		case badger.ErrKeyNotFound:
-			return blockstore.ErrNotFound
+			return ipld.ErrNotFound{Cid: cid}
 		default:
 			return fmt.Errorf("failed to get block size from badger blockstore: %w", err)
 		}

blockstore/badger/blockstore_test.go

@@ -10,12 +10,11 @@ import (
 	"strings"
 	"testing"
 
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
 
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-
 	"github.com/filecoin-project/lotus/blockstore"
 )
 

blockstore/badger/blockstore_test_suite.go

@@ -12,10 +12,10 @@ import (
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	u "github.com/ipfs/go-ipfs-util"
+	ipld "github.com/ipfs/go-ipld-format"
+	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/lotus/blockstore"
-
-	"github.com/stretchr/testify/require"
 )
 
 // TODO: move this to go-ipfs-blockstore.
@@ -56,7 +56,7 @@ func (s *Suite) TestGetWhenKeyNotPresent(t *testing.T) {
 	c := cid.NewCidV0(u.Hash([]byte("stuff")))
 	bl, err := bs.Get(ctx, c)
 	require.Nil(t, bl)
-	require.Equal(t, blockstore.ErrNotFound, err)
+	require.Equal(t, ipld.ErrNotFound{Cid: c}, err)
 }
 
 func (s *Suite) TestGetWhenKeyIsNil(t *testing.T) {
@@ -69,7 +69,7 @@ func (s *Suite) TestGetWhenKeyIsNil(t *testing.T) {
 	}
 
 	_, err := bs.Get(ctx, cid.Undef)
-	require.Equal(t, blockstore.ErrNotFound, err)
+	require.Equal(t, ipld.ErrNotFound{Cid: cid.Undef}, err)
 }
 
 func (s *Suite) TestPutThenGetBlock(t *testing.T) {
@@ -164,8 +164,9 @@ func (s *Suite) TestPutThenGetSizeBlock(t *testing.T) {
 	require.NoError(t, err)
 	require.Zero(t, emptySize)
 
-	missingSize, err := bs.GetSize(ctx, missingBlock.Cid())
-	require.Equal(t, blockstore.ErrNotFound, err)
+	missingCid := missingBlock.Cid()
+	missingSize, err := bs.GetSize(ctx, missingCid)
+	require.Equal(t, ipld.ErrNotFound{Cid: missingCid}, err)
 	require.Equal(t, -1, missingSize)
 }
 

blockstore/blockstore.go

@@ -3,17 +3,14 @@ package blockstore
 import (
 	"context"
 
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 	ds "github.com/ipfs/go-datastore"
-	logging "github.com/ipfs/go-log/v2"
-
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	logging "github.com/ipfs/go-log/v2"
 )
 
 var log = logging.Logger("blockstore")
 
-var ErrNotFound = blockstore.ErrNotFound
-
 // Blockstore is the blockstore interface used by Lotus. It is the union
 // of the basic go-ipfs blockstore, with other capabilities required by Lotus,
 // e.g. View or Sync.

blockstore/buffered.go

@@ -6,6 +6,7 @@ import (
 
 	block "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
 )
 
 // buflog is a logger for the buffered blockstore. It is subscoped from the
@@ -106,7 +107,7 @@ func (bs *BufferedBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) er
 
 func (bs *BufferedBlockstore) View(ctx context.Context, c cid.Cid, callback func([]byte) error) error {
 	// both stores are viewable.
-	if err := bs.write.View(ctx, c, callback); err == ErrNotFound {
+	if err := bs.write.View(ctx, c, callback); ipld.IsNotFound(err) {
 		// not found in write blockstore; fall through.
 	} else {
 		return err // propagate errors, or nil, i.e. found.
@@ -116,7 +117,7 @@ func (bs *BufferedBlockstore) View(ctx context.Context, c cid.Cid, callback func
 
 func (bs *BufferedBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block, error) {
 	if out, err := bs.write.Get(ctx, c); err != nil {
-		if err != ErrNotFound {
+		if !ipld.IsNotFound(err) {
 			return nil, err
 		}
 	} else {
@@ -128,7 +129,7 @@ func (bs *BufferedBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block,
 
 func (bs *BufferedBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
 	s, err := bs.read.GetSize(ctx, c)
-	if err == ErrNotFound || s == 0 {
+	if ipld.IsNotFound(err) || s == 0 {
 		return bs.write.GetSize(ctx, c)
 	}
 

blockstore/discard.go

@@ -5,7 +5,7 @@ import (
 	"io"
 
 	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 )
 
 var _ Blockstore = (*discardstore)(nil)

blockstore/fallback.go

@@ -5,10 +5,10 @@ import (
 	"sync"
 	"time"
 
-	"golang.org/x/xerrors"
-
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
+	"golang.org/x/xerrors"
 )
 
 // UnwrapFallbackStore takes a blockstore, and returns the underlying blockstore
@@ -56,7 +56,7 @@ func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
 
 		if fbs.missFn == nil {
 			log.Errorw("fallbackstore: missFn not configured yet")
-			return nil, ErrNotFound
+			return nil, ipld.ErrNotFound{Cid: c}
 		}
 	}
 
@@ -79,10 +79,10 @@ func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
 
 func (fbs *FallbackStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
 	b, err := fbs.Blockstore.Get(ctx, c)
-	switch err {
-	case nil:
+	switch {
+	case err == nil:
 		return b, nil
-	case ErrNotFound:
+	case ipld.IsNotFound(err):
 		return fbs.getFallback(c)
 	default:
 		return b, err
@@ -91,10 +91,10 @@ func (fbs *FallbackStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, err
 
 func (fbs *FallbackStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
 	sz, err := fbs.Blockstore.GetSize(ctx, c)
-	switch err {
-	case nil:
+	switch {
+	case err == nil:
 		return sz, nil
-	case ErrNotFound:
+	case ipld.IsNotFound(err):
 		b, err := fbs.getFallback(c)
 		if err != nil {
 			return 0, err

blockstore/idstore.go

@@ -4,11 +4,10 @@ import (
 	"context"
 	"io"
 
-	"golang.org/x/xerrors"
-
 	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 	mh "github.com/multiformats/go-multihash"
+	"golang.org/x/xerrors"
 )
 
 var _ Blockstore = (*idstore)(nil)
@@ -103,6 +102,14 @@ func (b *idstore) Put(ctx context.Context, blk blocks.Block) error {
 	return b.bs.Put(ctx, blk)
 }
 
+func (b *idstore) ForEachKey(f func(cid.Cid) error) error {
+	iterBstore, ok := b.bs.(BlockstoreIterator)
+	if !ok {
+		return xerrors.Errorf("underlying blockstore (type %T) doesn't support fast iteration", b.bs)
+	}
+	return iterBstore.ForEachKey(f)
+}
+
 func (b *idstore) PutMany(ctx context.Context, blks []blocks.Block) error {
 	toPut := make([]blocks.Block, 0, len(blks))
 	for _, blk := range blks {

blockstore/ipfs.go

@@ -5,17 +5,15 @@ import (
 	"context"
 	"io/ioutil"
 
-	"golang.org/x/xerrors"
-
-	"github.com/multiformats/go-multiaddr"
-	"github.com/multiformats/go-multihash"
-
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	httpapi "github.com/ipfs/go-ipfs-http-client"
 	iface "github.com/ipfs/interface-go-ipfs-core"
 	"github.com/ipfs/interface-go-ipfs-core/options"
 	"github.com/ipfs/interface-go-ipfs-core/path"
+	"github.com/multiformats/go-multiaddr"
+	"github.com/multiformats/go-multihash"
+	"golang.org/x/xerrors"
 )
 
 type IPFSBlockstore struct {
@@ -130,7 +128,7 @@ func (i *IPFSBlockstore) Put(ctx context.Context, block blocks.Block) error {
 
 	_, err = i.api.Block().Put(ctx, bytes.NewReader(block.RawData()),
 		options.Block.Hash(mhd.Code, mhd.Length),
-		options.Block.Format(cid.CodecToStr[block.Cid().Type()]))
+		options.Block.Format(multihash.Codes[block.Cid().Type()]))
 	return err
 }
 

blockstore/mem.go

@@ -5,6 +5,7 @@ import (
 
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
 )
 
 // NewMemory returns a temporary memory-backed blockstore.
@@ -13,46 +14,47 @@ func NewMemory() MemBlockstore {
 }
 
 // MemBlockstore is a terminal blockstore that keeps blocks in memory.
-type MemBlockstore map[cid.Cid]blocks.Block
+// To match behavior of badger blockstore we index by multihash only.
+type MemBlockstore map[string]blocks.Block
 
 func (m MemBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
-	delete(m, k)
+	delete(m, string(k.Hash()))
 	return nil
 }
 
 func (m MemBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error {
 	for _, k := range ks {
-		delete(m, k)
+		delete(m, string(k.Hash()))
 	}
 	return nil
 }
 
 func (m MemBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error) {
-	_, ok := m[k]
+	_, ok := m[string(k.Hash())]
 	return ok, nil
 }
 
 func (m MemBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error {
-	b, ok := m[k]
+	b, ok := m[string(k.Hash())]
 	if !ok {
-		return ErrNotFound
+		return ipld.ErrNotFound{Cid: k}
 	}
 	return callback(b.RawData())
 }
 
 func (m MemBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) {
-	b, ok := m[k]
+	b, ok := m[string(k.Hash())]
 	if !ok {
-		return nil, ErrNotFound
+		return nil, ipld.ErrNotFound{Cid: k}
 	}
 	return b, nil
 }
 
 // GetSize returns the CIDs mapped BlockSize
 func (m MemBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
-	b, ok := m[k]
+	b, ok := m[string(k.Hash())]
 	if !ok {
-		return 0, ErrNotFound
+		return 0, ipld.ErrNotFound{Cid: k}
 	}
 	return len(b.RawData()), nil
 }
@@ -61,7 +63,7 @@ func (m MemBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
 func (m MemBlockstore) Put(ctx context.Context, b blocks.Block) error {
 	// Convert to a basic block for safety, but try to reuse the existing
 	// block if it's already a basic block.
-	k := b.Cid()
+	k := string(b.Cid().Hash())
 	if _, ok := b.(*blocks.BasicBlock); !ok {
 		// If we already have the block, abort.
 		if _, ok := m[k]; ok {
@@ -70,7 +72,7 @@ func (m MemBlockstore) Put(ctx context.Context, b blocks.Block) error {
 		// the error is only for debugging.
 		b, _ = blocks.NewBlockWithCid(b.RawData(), b.Cid())
 	}
-	m[b.Cid()] = b
+	m[k] = b
 	return nil
 }
 
@@ -88,8 +90,8 @@ func (m MemBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
 // the given context, closing the channel if it becomes Done.
 func (m MemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
 	ch := make(chan cid.Cid, len(m))
-	for k := range m {
-		ch <- k
+	for _, b := range m {
+		ch <- b.Cid()
 	}
 	close(ch)
 	return ch, nil

blockstore/splitstore/checkpoint.go

@@ -5,10 +5,9 @@ import (
 	"io"
 	"os"
 
-	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 	mh "github.com/multiformats/go-multihash"
+	"golang.org/x/xerrors"
 )
 
 type Checkpoint struct {

blockstore/splitstore/coldset.go

@@ -5,9 +5,8 @@ import (
 	"io"
 	"os"
 
+	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
 )
 
 type ColdSetWriter struct {

blockstore/splitstore/debug.go

@@ -12,11 +12,10 @@ import (
 	"sync"
 	"time"
 
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
 	"go.uber.org/multierr"
 	"golang.org/x/xerrors"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
 )
 
 type debugLog struct {

blockstore/splitstore/markset.go

@@ -3,9 +3,8 @@ package splitstore
 import (
 	"errors"
 
+	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
 )
 
 var errMarkSetClosed = errors.New("markset closed")

blockstore/splitstore/markset_badger.go

@@ -6,13 +6,11 @@ import (
 	"runtime"
 	"sync"
 
-	"golang.org/x/xerrors"
-
 	"github.com/dgraph-io/badger/v2"
 	"github.com/dgraph-io/badger/v2/options"
+	"github.com/ipfs/go-cid"
 	"go.uber.org/zap"
-
-	cid "github.com/ipfs/go-cid"
+	"golang.org/x/xerrors"
 )
 
 type BadgerMarkSetEnv struct {

blockstore/splitstore/markset_map.go

@@ -7,9 +7,8 @@ import (
 	"path/filepath"
 	"sync"
 
+	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
 )
 
 type MapMarkSetEnv struct {

blockstore/splitstore/markset_test.go

@@ -4,7 +4,7 @@ package splitstore
 import (
 	"testing"
 
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 	"github.com/multiformats/go-multihash"
 )
 

blockstore/splitstore/splitstore.go

@@ -8,22 +8,22 @@ import (
 	"sync/atomic"
 	"time"
 
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	dstore "github.com/ipfs/go-datastore"
+	ipld "github.com/ipfs/go-ipld-format"
+	logging "github.com/ipfs/go-log/v2"
+	"go.opencensus.io/stats"
 	"go.uber.org/multierr"
 	"golang.org/x/xerrors"
 
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-	dstore "github.com/ipfs/go-datastore"
-	logging "github.com/ipfs/go-log/v2"
-
 	"github.com/filecoin-project/go-state-types/abi"
+
 	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/metrics"
-
-	"go.opencensus.io/stats"
 )
 
 var (
@@ -43,8 +43,16 @@ var (
 	// compactionIndexKey stores the compaction index (serial number)
 	compactionIndexKey = dstore.NewKey("/splitstore/compactionIndex")
 
+	// stores the prune index (serial number)
+	pruneIndexKey = dstore.NewKey("/splitstore/pruneIndex")
+
+	// stores the base epoch of last prune in the metadata store
+	pruneEpochKey = dstore.NewKey("/splitstore/pruneEpoch")
+
 	log = logging.Logger("splitstore")
 
+	errClosing = errors.New("splitstore is closing")
+
 	// set this to true if you are debugging the splitstore to enable debug logging
 	enableDebugLog = false
 	// set this to true if you want to track origin stack traces in the write log
@@ -54,6 +62,16 @@ var (
 	upgradeBoundary = build.Finality
 )
 
+type CompactType int
+
+const (
+	none CompactType = iota
+	warmup
+	hot
+	cold
+	check
+)
+
 func init() {
 	if os.Getenv("LOTUS_SPLITSTORE_DEBUG_LOG") == "1" {
 		enableDebugLog = true
@@ -93,6 +111,21 @@ type Config struct {
 	// A positive value is the number of compactions before a full GC is performed;
 	// a value of 1 will perform full GC in every compaction.
 	HotStoreFullGCFrequency uint64
+
+	// EnableColdStoreAutoPrune turns on compaction of the cold store i.e. pruning
+	// where hotstore compaction occurs every finality epochs pruning happens every 3 finalities
+	// Default is false
+	EnableColdStoreAutoPrune bool
+
+	// ColdStoreFullGCFrequency specifies how often to performa a full (moving) GC on the coldstore.
+	// Only applies if auto prune is enabled.  A value of 0 disables while a value of 1 will do
+	// full GC in every prune.
+	// Default is 7 (about once every a week)
+	ColdStoreFullGCFrequency uint64
+
+	// ColdStoreRetention specifies the retention policy for data reachable from the chain, in
+	// finalities beyond the compaction boundary, default is 0, -1 retains everything
+	ColdStoreRetention int64
 }
 
 // ChainAccessor allows the Splitstore to access the chain. It will most likely
@@ -117,8 +150,9 @@ type hotstore interface {
 }
 
 type SplitStore struct {
-	compacting int32 // compaction/prune/warmup in progress
-	closing    int32 // the splitstore is closing
+	compacting  int32       // flag for when compaction is in progress
+	compactType CompactType // compaction type, protected by compacting atomic, only meaningful when compacting == 1
+	closing     int32       // the splitstore is closing
 
 	cfg  *Config
 	path string
@@ -126,6 +160,7 @@ type SplitStore struct {
 	mx          sync.Mutex
 	warmupEpoch abi.ChainEpoch // protected by mx
 	baseEpoch   abi.ChainEpoch // protected by compaction lock
+	pruneEpoch  abi.ChainEpoch // protected by compaction lock
 
 	headChangeMx sync.Mutex
 
@@ -140,6 +175,7 @@ type SplitStore struct {
 	markSetSize int64
 
 	compactionIndex int64
+	pruneIndex      int64
 
 	ctx    context.Context
 	cancel func()
@@ -227,6 +263,13 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 			return nil, xerrors.Errorf("error resuming compaction: %w", err)
 		}
 	}
+	if ss.pruneCheckpointExists() {
+		log.Info("found prune checkpoint; resuming prune")
+		if err := ss.completePrune(); err != nil {
+			markSetEnv.Close() //nolint:errcheck
+			return nil, xerrors.Errorf("error resuming prune: %w", err)
+		}
+	}
 
 	return ss, nil
 }
@@ -260,8 +303,14 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 		if has {
 			return s.has(cid)
 		}
-
-		return s.cold.Has(ctx, cid)
+		switch s.compactType {
+		case hot:
+			return s.cold.Has(ctx, cid)
+		case cold:
+			return s.hot.Has(ctx, cid)
+		default:
+			return false, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
+		}
 	}
 
 	has, err := s.hot.Has(ctx, cid)
@@ -276,8 +325,11 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 	}
 
 	has, err = s.cold.Has(ctx, cid)
-	if has && bstore.IsHotView(ctx) {
-		s.reifyColdObject(cid)
+	if has {
+		s.trackTxnRef(cid)
+		if bstore.IsHotView(ctx) {
+			s.reifyColdObject(cid)
+		}
 	}
 
 	return has, err
@@ -307,24 +359,31 @@ func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
 		if has {
 			return s.get(cid)
 		}
-
-		return s.cold.Get(ctx, cid)
+		switch s.compactType {
+		case hot:
+			return s.cold.Get(ctx, cid)
+		case cold:
+			return s.hot.Get(ctx, cid)
+		default:
+			return nil, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
+		}
 	}
 
 	blk, err := s.hot.Get(ctx, cid)
 
-	switch err {
-	case nil:
+	switch {
+	case err == nil:
 		s.trackTxnRef(cid)
 		return blk, nil
 
-	case bstore.ErrNotFound:
+	case ipld.IsNotFound(err):
 		if s.isWarm() {
 			s.debug.LogReadMiss(cid)
 		}
 
 		blk, err = s.cold.Get(ctx, cid)
 		if err == nil {
+			s.trackTxnRef(cid)
 			if bstore.IsHotView(ctx) {
 				s.reifyColdObject(cid)
 			}
@@ -361,24 +420,31 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
 		if has {
 			return s.getSize(cid)
 		}
-
-		return s.cold.GetSize(ctx, cid)
+		switch s.compactType {
+		case hot:
+			return s.cold.GetSize(ctx, cid)
+		case cold:
+			return s.hot.GetSize(ctx, cid)
+		default:
+			return 0, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
+		}
 	}
 
 	size, err := s.hot.GetSize(ctx, cid)
 
-	switch err {
-	case nil:
+	switch {
+	case err == nil:
 		s.trackTxnRef(cid)
 		return size, nil
 
-	case bstore.ErrNotFound:
+	case ipld.IsNotFound(err):
 		if s.isWarm() {
 			s.debug.LogReadMiss(cid)
 		}
 
 		size, err = s.cold.GetSize(ctx, cid)
 		if err == nil {
+			s.trackTxnRef(cid)
 			if bstore.IsHotView(ctx) {
 				s.reifyColdObject(cid)
 			}
@@ -408,12 +474,12 @@ func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error {
 	s.debug.LogWrite(blk)
 
 	// critical section
-	if s.txnMarkSet != nil {
+	if s.txnMarkSet != nil && s.compactType == hot { // puts only touch hot store
 		s.markLiveRefs([]cid.Cid{blk.Cid()})
 		return nil
 	}
-
 	s.trackTxnRef(blk.Cid())
+
 	return nil
 }
 
@@ -459,12 +525,12 @@ func (s *SplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {
 	s.debug.LogWriteMany(blks)
 
 	// critical section
-	if s.txnMarkSet != nil {
+	if s.txnMarkSet != nil && s.compactType == hot { // puts only touch hot store
 		s.markLiveRefs(batch)
 		return nil
 	}
-
 	s.trackTxnRefMany(batch)
+
 	return nil
 }
 
@@ -536,8 +602,14 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
 		if has {
 			return s.view(cid, cb)
 		}
-
-		return s.cold.View(ctx, cid, cb)
+		switch s.compactType {
+		case hot:
+			return s.cold.View(ctx, cid, cb)
+		case cold:
+			return s.hot.View(ctx, cid, cb)
+		default:
+			return xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
+		}
 	}
 
 	// views are (optimistically) protected two-fold:
@@ -552,8 +624,7 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
 	defer s.viewDone()
 
 	err := s.hot.View(ctx, cid, cb)
-	switch err {
-	case bstore.ErrNotFound:
+	if ipld.IsNotFound(err) {
 		if s.isWarm() {
 			s.debug.LogReadMiss(cid)
 		}
@@ -567,10 +638,8 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
 			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return err
-
-	default:
-		return err
 	}
+	return err
 }
 
 func (s *SplitStore) isWarm() bool {
@@ -626,6 +695,23 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
 		return xerrors.Errorf("error loading base epoch: %w", err)
 	}
 
+	// load prune epoch from metadata ds
+	bs, err = s.ds.Get(s.ctx, pruneEpochKey)
+	switch err {
+	case nil:
+		s.pruneEpoch = bytesToEpoch(bs)
+	case dstore.ErrNotFound:
+		if curTs == nil {
+			//this can happen in some tests
+			break
+		}
+		if err := s.setPruneEpoch(curTs.Height()); err != nil {
+			return xerrors.Errorf("error saving prune epoch: %w", err)
+		}
+	default:
+		return xerrors.Errorf("error loading prune epoch: %w", err)
+	}
+
 	// load warmup epoch from metadata ds
 	bs, err = s.ds.Get(s.ctx, warmupEpochKey)
 	switch err {
@@ -725,3 +811,8 @@ func (s *SplitStore) setBaseEpoch(epoch abi.ChainEpoch) error {
 	s.baseEpoch = epoch
 	return s.ds.Put(s.ctx, baseEpochKey, epochToBytes(epoch))
 }
+
+func (s *SplitStore) setPruneEpoch(epoch abi.ChainEpoch) error {
+	s.pruneEpoch = epoch
+	return s.ds.Put(s.ctx, pruneEpochKey, epochToBytes(epoch))
+}

blockstore/splitstore/splitstore_check.go

@@ -8,10 +8,9 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
 
-	cid "github.com/ipfs/go-cid"
-
 	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
 )
@@ -26,6 +25,7 @@ func (s *SplitStore) Check() error {
 	if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
 		return xerrors.Errorf("can't acquire compaction lock; compacting operation in progress")
 	}
+	s.compactType = check
 
 	if s.compactionIndex == 0 {
 		atomic.StoreInt32(&s.compacting, 0)
@@ -147,6 +147,8 @@ func (s *SplitStore) Info() map[string]interface{} {
 	info["base epoch"] = s.baseEpoch
 	info["warmup epoch"] = s.warmupEpoch
 	info["compactions"] = s.compactionIndex
+	info["prunes"] = s.pruneIndex
+	info["compacting"] = s.compacting == 1
 
 	sizer, ok := s.hot.(bstore.BlockstoreSize)
 	if ok {

blockstore/splitstore/splitstore_compact.go

@@ -10,20 +10,20 @@ import (
 	"sync/atomic"
 	"time"
 
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	"go.opencensus.io/stats"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/xerrors"
 
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-	cbg "github.com/whyrusleeping/cbor-gen"
-
 	"github.com/filecoin-project/go-state-types/abi"
+
 	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/metrics"
-
-	"go.opencensus.io/stats"
 )
 
 var (
@@ -53,6 +53,12 @@ var (
 	// SyncWaitTime is the time delay from a tipset's min timestamp before we decide
 	// we have synced.
 	SyncWaitTime = 30 * time.Second
+
+	// This is a testing flag that should always be true when running a node. itests rely on the rough hack
+	// of starting genesis so far in the past that they exercise catchup mining to mine
+	// blocks quickly and so disabling syncgap checking is necessary to test compaction
+	// without a deep structural improvement of itests.
+	CheckSyncGap = true
 )
 
 var (
@@ -82,7 +88,7 @@ func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
 	//       this is guaranteed by the chainstore, and it is pervasive in all lotus
 	//       -- if that ever changes then all hell will break loose in general and
 	//       we will have a rance to protectTipSets here.
-	//      Reagrdless, we put a mutex in HeadChange just to be safe
+	//      Regardless, we put a mutex in HeadChange just to be safe
 
 	if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
 		// we are currently compacting -- protect the new tipset(s)
@@ -97,7 +103,8 @@ func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
 	}
 
 	timestamp := time.Unix(int64(curTs.MinTimestamp()), 0)
-	if time.Since(timestamp) > SyncGapTime {
+
+	if CheckSyncGap && time.Since(timestamp) > SyncGapTime {
 		// don't attempt compaction before we have caught up syncing
 		atomic.StoreInt32(&s.compacting, 0)
 		return nil
@@ -109,9 +116,12 @@ func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
 		return nil
 	}
 
+	// Prioritize hot store compaction over cold store prune
+
 	if epoch-s.baseEpoch > CompactionThreshold {
 		// it's time to compact -- prepare the transaction and go!
 		s.beginTxnProtect()
+		s.compactType = hot
 		go func() {
 			defer atomic.StoreInt32(&s.compacting, 0)
 			defer s.endTxnProtect()
@@ -123,6 +133,40 @@ func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
 
 			log.Infow("compaction done", "took", time.Since(start))
 		}()
+		// only prune if auto prune is enabled and after at least one compaction
+	} else if s.cfg.EnableColdStoreAutoPrune && epoch-s.pruneEpoch > PruneThreshold && s.compactionIndex > 0 {
+		s.beginTxnProtect()
+		s.compactType = cold
+		go func() {
+			defer atomic.StoreInt32(&s.compacting, 0)
+			defer s.endTxnProtect()
+
+			log.Info("pruning splitstore")
+			start := time.Now()
+
+			var retainP func(int64) bool
+			switch {
+			case s.cfg.ColdStoreRetention > int64(0):
+				retainP = func(depth int64) bool {
+					return depth <= int64(CompactionBoundary)+s.cfg.ColdStoreRetention*int64(build.Finality)
+				}
+			case s.cfg.ColdStoreRetention < 0:
+				retainP = func(_ int64) bool { return true }
+			default:
+				retainP = func(depth int64) bool {
+					return depth <= int64(CompactionBoundary)
+				}
+			}
+			movingGC := s.cfg.ColdStoreFullGCFrequency > 0 && s.pruneIndex%int64(s.cfg.ColdStoreFullGCFrequency) == 0
+			var gcOpts []bstore.BlockstoreGCOption
+			if movingGC {
+				gcOpts = append(gcOpts, bstore.WithFullGC(true))
+			}
+			doGC := func() error { return s.gcBlockstore(s.cold, gcOpts) }
+
+			s.prune(curTs, retainP, doGC)
+			log.Infow("prune done", "took", time.Since(start))
+		}()
 	} else {
 		// no compaction necessary
 		atomic.StoreInt32(&s.compacting, 0)
@@ -588,7 +632,6 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 
 	// some stats for logging
 	var hotCnt, coldCnt int
-
 	err = s.hot.ForEachKey(func(c cid.Cid) error {
 		// was it marked?
 		mark, err := markSet.Has(c)
@@ -609,7 +652,6 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 
 		return nil
 	})
-
 	if err != nil {
 		return xerrors.Errorf("error collecting cold objects: %w", err)
 	}
@@ -778,6 +820,10 @@ func (s *SplitStore) beginCriticalSection(markSet MarkSet) error {
 
 func (s *SplitStore) waitForSync() {
 	log.Info("waiting for sync")
+	if !CheckSyncGap {
+		log.Warnf("If you see this outside of test it is a serious splitstore issue")
+		return
+	}
 	startWait := time.Now()
 	defer func() {
 		log.Infow("waiting for sync done", "took", time.Since(startWait))
@@ -1065,13 +1111,10 @@ func (s *SplitStore) view(c cid.Cid, cb func([]byte) error) error {
 	}
 
 	err := s.hot.View(s.ctx, c, cb)
-	switch err {
-	case bstore.ErrNotFound:
+	if ipld.IsNotFound(err) {
 		return s.cold.View(s.ctx, c, cb)
-
-	default:
-		return err
 	}
+	return err
 }
 
 func (s *SplitStore) has(c cid.Cid) (bool, error) {
@@ -1090,10 +1133,10 @@ func (s *SplitStore) has(c cid.Cid) (bool, error) {
 
 func (s *SplitStore) get(c cid.Cid) (blocks.Block, error) {
 	blk, err := s.hot.Get(s.ctx, c)
-	switch err {
-	case nil:
+	switch {
+	case err == nil:
 		return blk, nil
-	case bstore.ErrNotFound:
+	case ipld.IsNotFound(err):
 		return s.cold.Get(s.ctx, c)
 	default:
 		return nil, err
@@ -1102,10 +1145,10 @@ func (s *SplitStore) get(c cid.Cid) (blocks.Block, error) {
 
 func (s *SplitStore) getSize(c cid.Cid) (int, error) {
 	sz, err := s.hot.GetSize(s.ctx, c)
-	switch err {
-	case nil:
+	switch {
+	case err == nil:
 		return sz, nil
-	case bstore.ErrNotFound:
+	case ipld.IsNotFound(err):
 		return s.cold.GetSize(s.ctx, c)
 	default:
 		return 0, err
@@ -1119,10 +1162,9 @@ func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
 		if err := s.checkClosing(); err != nil {
 			return err
 		}
-
 		blk, err := s.hot.Get(s.ctx, c)
 		if err != nil {
-			if err == bstore.ErrNotFound {
+			if ipld.IsNotFound(err) {
 				log.Warnf("hotstore missing block %s", c)
 				return nil
 			}
@@ -1137,6 +1179,7 @@ func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
 				return xerrors.Errorf("error putting batch to coldstore: %w", err)
 			}
 			batch = batch[:0]
+
 		}
 
 		return nil
@@ -1225,8 +1268,17 @@ func (s *SplitStore) purgeBatch(batch, deadCids []cid.Cid, checkpoint *Checkpoin
 		return 0, liveCnt, nil
 	}
 
-	if err := s.hot.DeleteMany(s.ctx, deadCids); err != nil {
-		return 0, liveCnt, xerrors.Errorf("error purging cold objects: %w", err)
+	switch s.compactType {
+	case hot:
+		if err := s.hot.DeleteMany(s.ctx, deadCids); err != nil {
+			return 0, liveCnt, xerrors.Errorf("error purging cold objects: %w", err)
+		}
+	case cold:
+		if err := s.cold.DeleteMany(s.ctx, deadCids); err != nil {
+			return 0, liveCnt, xerrors.Errorf("error purging dead objects: %w", err)
+		}
+	default:
+		return 0, liveCnt, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
 	}
 
 	s.debug.LogDelete(deadCids)
@@ -1243,15 +1295,28 @@ func (s *SplitStore) coldSetPath() string {
 	return filepath.Join(s.path, "coldset")
 }
 
+func (s *SplitStore) deadSetPath() string {
+	return filepath.Join(s.path, "deadset")
+}
+
 func (s *SplitStore) checkpointPath() string {
 	return filepath.Join(s.path, "checkpoint")
 }
 
+func (s *SplitStore) pruneCheckpointPath() string {
+	return filepath.Join(s.path, "prune-checkpoint")
+}
+
 func (s *SplitStore) checkpointExists() bool {
 	_, err := os.Stat(s.checkpointPath())
 	return err == nil
 }
 
+func (s *SplitStore) pruneCheckpointExists() bool {
+	_, err := os.Stat(s.pruneCheckpointPath())
+	return err == nil
+}
+
 func (s *SplitStore) completeCompaction() error {
 	checkpoint, last, err := OpenCheckpoint(s.checkpointPath())
 	if err != nil {
@@ -1272,6 +1337,7 @@ func (s *SplitStore) completeCompaction() error {
 	defer markSet.Close() //nolint:errcheck
 
 	// PURGE
+	s.compactType = hot
 	log.Info("purging cold objects from the hotstore")
 	startPurge := time.Now()
 	err = s.completePurge(coldr, checkpoint, last, markSet)
@@ -1294,6 +1360,7 @@ func (s *SplitStore) completeCompaction() error {
 	if err := os.Remove(s.coldSetPath()); err != nil {
 		log.Warnf("error removing coldset: %s", err)
 	}
+	s.compactType = none
 
 	// Note: at this point we can start the splitstore; a compaction should run on
 	//       the first head change, which will trigger gc on the hotstore.

blockstore/splitstore/splitstore_expose.go

@@ -5,7 +5,8 @@ import (
 	"errors"
 
 	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
 
 	bstore "github.com/filecoin-project/lotus/blockstore"
 )
@@ -42,6 +43,7 @@ func (es *exposedSplitStore) Has(ctx context.Context, c cid.Cid) (bool, error) {
 }
 
 func (es *exposedSplitStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
+
 	if isIdentiyCid(c) {
 		data, err := decodeIdentityCid(c)
 		if err != nil {
@@ -52,12 +54,10 @@ func (es *exposedSplitStore) Get(ctx context.Context, c cid.Cid) (blocks.Block,
 	}
 
 	blk, err := es.s.hot.Get(ctx, c)
-	switch err {
-	case bstore.ErrNotFound:
+	if ipld.IsNotFound(err) {
 		return es.s.cold.Get(ctx, c)
-	default:
-		return blk, err
 	}
+	return blk, err
 }
 
 func (es *exposedSplitStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
@@ -71,12 +71,10 @@ func (es *exposedSplitStore) GetSize(ctx context.Context, c cid.Cid) (int, error
 	}
 
 	size, err := es.s.hot.GetSize(ctx, c)
-	switch err {
-	case bstore.ErrNotFound:
+	if ipld.IsNotFound(err) {
 		return es.s.cold.GetSize(ctx, c)
-	default:
-		return size, err
 	}
+	return size, err
 }
 
 func (es *exposedSplitStore) Put(ctx context.Context, blk blocks.Block) error {
@@ -104,11 +102,9 @@ func (es *exposedSplitStore) View(ctx context.Context, c cid.Cid, f func([]byte)
 	}
 
 	err := es.s.hot.View(ctx, c, f)
-	switch err {
-	case bstore.ErrNotFound:
+	if ipld.IsNotFound(err) {
 		return es.s.cold.View(ctx, c, f)
-
-	default:
-		return err
 	}
+
+	return err
 }

blockstore/splitstore/splitstore_gc.go

@@ -27,7 +27,7 @@ func (s *SplitStore) gcBlockstore(b bstore.Blockstore, opts []bstore.BlockstoreG
 			return err
 		}
 
-		log.Infow("garbage collecting hotstore done", "took", time.Since(startGC))
+		log.Infow("garbage collecting blockstore done", "took", time.Since(startGC))
 		return nil
 	}
 

blockstore/splitstore/splitstore_prune.go

@@ -0,0 +1,574 @@
+package splitstore
+
+import (
+	"bytes"
+	"os"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	cid "github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	"go.opencensus.io/stats"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/api"
+	bstore "github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/metrics"
+)
+
+var (
+	// PruneOnline is a prune option that instructs PruneChain to use online gc for reclaiming space;
+	// there is no value associated with this option.
+	PruneOnlineGC = "splitstore.PruneOnlineGC"
+
+	// PruneMoving is a prune option that instructs PruneChain to use moving gc for reclaiming space;
+	// the value associated with this option is the path of the new coldstore.
+	PruneMovingGC = "splitstore.PruneMovingGC"
+
+	// PruneRetainState is a prune option that instructs PruneChain as to how many finalities worth
+	// of state to retain in the coldstore.
+	// The value is an integer:
+	// - if it is -1 then all state objects reachable from the chain will be retained in the coldstore.
+	//   this is useful for garbage collecting side-chains and other garbage in archival nodes.
+	//   This is the (safe) default.
+	// - if it is 0 then no state objects that are unreachable within the compaction boundary will
+	//   be retained in the coldstore.
+	// - if it is a positive integer, then it's the number of finalities past the compaction boundary
+	//   for which chain-reachable state objects are retained.
+	PruneRetainState = "splitstore.PruneRetainState"
+
+	// PruneThreshold is the number of epochs that need to have elapsed
+	// from the previously pruned epoch to trigger a new prune
+	PruneThreshold = 7 * build.Finality
+)
+
+// PruneChain instructs the SplitStore to prune chain state in the coldstore, according to the
+// options specified.
+func (s *SplitStore) PruneChain(opts api.PruneOpts) error {
+	retainState := opts.RetainState
+
+	var gcOpts []bstore.BlockstoreGCOption
+	if opts.MovingGC {
+		gcOpts = append(gcOpts, bstore.WithFullGC(true))
+	}
+	doGC := func() error { return s.gcBlockstore(s.cold, gcOpts) }
+
+	var retainStateP func(int64) bool
+	switch {
+	case retainState > 0:
+		retainStateP = func(depth int64) bool {
+			return depth <= int64(CompactionBoundary)+retainState*int64(build.Finality)
+		}
+	case retainState < 0:
+		retainStateP = func(_ int64) bool { return true }
+	default:
+		retainStateP = func(depth int64) bool {
+			return depth <= int64(CompactionBoundary)
+		}
+	}
+
+	if _, ok := s.cold.(bstore.BlockstoreIterator); !ok {
+		return xerrors.Errorf("coldstore does not support efficient iteration")
+	}
+
+	return s.pruneChain(retainStateP, doGC)
+}
+
+func (s *SplitStore) pruneChain(retainStateP func(int64) bool, doGC func() error) error {
+	// inhibit compaction while we are setting up
+	s.headChangeMx.Lock()
+	defer s.headChangeMx.Unlock()
+
+	// take the compaction lock; fail if there is a compaction in progress
+	if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
+		return xerrors.Errorf("compaction, prune or warmup in progress")
+	}
+
+	// check if we are actually closing first
+	if atomic.LoadInt32(&s.closing) == 1 {
+		atomic.StoreInt32(&s.compacting, 0)
+		return errClosing
+	}
+
+	// ensure that we have compacted at least once
+	if s.compactionIndex == 0 {
+		atomic.StoreInt32(&s.compacting, 0)
+		return xerrors.Errorf("splitstore has not compacted yet")
+	}
+
+	// get the current tipset
+	curTs := s.chain.GetHeaviestTipSet()
+
+	// begin the transaction and go
+	s.beginTxnProtect()
+	s.compactType = cold
+	go func() {
+		defer atomic.StoreInt32(&s.compacting, 0)
+		defer s.endTxnProtect()
+
+		log.Info("pruning splitstore")
+		start := time.Now()
+
+		s.prune(curTs, retainStateP, doGC)
+
+		log.Infow("prune done", "took", time.Since(start))
+	}()
+
+	return nil
+}
+
+func (s *SplitStore) prune(curTs *types.TipSet, retainStateP func(int64) bool, doGC func() error) {
+	log.Debug("waiting for active views to complete")
+	start := time.Now()
+	s.viewWait()
+	log.Debugw("waiting for active views done", "took", time.Since(start))
+
+	err := s.doPrune(curTs, retainStateP, doGC)
+	if err != nil {
+		log.Errorf("PRUNE ERROR: %s", err)
+	}
+}
+
+func (s *SplitStore) doPrune(curTs *types.TipSet, retainStateP func(int64) bool, doGC func() error) error {
+	currentEpoch := curTs.Height()
+	boundaryEpoch := currentEpoch - CompactionBoundary
+
+	log.Infow("running prune", "currentEpoch", currentEpoch, "pruneEpoch", s.pruneEpoch)
+
+	markSet, err := s.markSetEnv.New("live", s.markSetSize)
+	if err != nil {
+		return xerrors.Errorf("error creating mark set: %w", err)
+	}
+	defer markSet.Close() //nolint:errcheck
+	defer s.debug.Flush()
+
+	if err := s.checkClosing(); err != nil {
+		return err
+	}
+
+	// 0. track all protected references at beginning of compaction; anything added later should
+	//    be transactionally protected by the write
+	log.Info("protecting references with registered protectors")
+	err = s.applyProtectors()
+	if err != nil {
+		return err
+	}
+
+	// 1. mark reachable objects by walking the chain from the current epoch; we keep all messages
+	//    and chain headers; state and reciepts are retained only if it is within retention policy scope
+	log.Info("marking reachable objects")
+	startMark := time.Now()
+
+	count := new(int64)
+	err = s.walkChainDeep(curTs, retainStateP,
+		func(c cid.Cid) error {
+			if isUnitaryObject(c) {
+				return errStopWalk
+			}
+
+			mark, err := markSet.Has(c)
+			if err != nil {
+				return xerrors.Errorf("error checking markset: %w", err)
+			}
+
+			if mark {
+				return errStopWalk
+			}
+
+			atomic.AddInt64(count, 1)
+			return markSet.Mark(c)
+		})
+
+	if err != nil {
+		return xerrors.Errorf("error marking: %w", err)
+	}
+
+	log.Infow("marking done", "took", time.Since(startMark), "marked", count)
+
+	if err := s.checkClosing(); err != nil {
+		return err
+	}
+
+	// 1.1 protect transactional refs
+	err = s.protectTxnRefs(markSet)
+	if err != nil {
+		return xerrors.Errorf("error protecting transactional refs: %w", err)
+	}
+
+	if err := s.checkClosing(); err != nil {
+		return err
+	}
+
+	// 2. iterate through the coldstore to collect dead objects
+	log.Info("collecting dead objects")
+	startCollect := time.Now()
+
+	deadw, err := NewColdSetWriter(s.deadSetPath())
+	if err != nil {
+		return xerrors.Errorf("error creating coldset: %w", err)
+	}
+	defer deadw.Close() //nolint:errcheck
+
+	// some stats for logging
+	var liveCnt, deadCnt int
+
+	err = s.cold.(bstore.BlockstoreIterator).ForEachKey(func(c cid.Cid) error {
+		// was it marked?
+		mark, err := markSet.Has(c)
+		if err != nil {
+			return xerrors.Errorf("error checking mark set for %s: %w", c, err)
+		}
+
+		if mark {
+			liveCnt++
+			return nil
+		}
+
+		// Note: a possible optimization here is to also purge objects that are in the hotstore
+		//       but this needs special care not to purge genesis state, so we don't bother (yet)
+
+		// it's dead in the coldstore, mark it as candidate for purge
+
+		if err := deadw.Write(c); err != nil {
+			return xerrors.Errorf("error writing cid to coldstore: %w", err)
+		}
+		deadCnt++
+
+		return nil
+	})
+
+	if err != nil {
+		return xerrors.Errorf("error dead objects: %w", err)
+	}
+
+	if err := deadw.Close(); err != nil {
+		return xerrors.Errorf("error closing deadset: %w", err)
+	}
+
+	stats.Record(s.ctx, metrics.SplitstoreCompactionDead.M(int64(deadCnt)))
+
+	log.Infow("dead collection done", "took", time.Since(startCollect))
+	log.Infow("prune stats", "live", liveCnt, "dead", deadCnt)
+
+	if err := s.checkClosing(); err != nil {
+		return err
+	}
+
+	// now that we have collected dead objects, check for missing references from transactional i/o
+	// this is carried over from hot compaction for completeness
+	s.waitForMissingRefs(markSet)
+
+	if err := s.checkClosing(); err != nil {
+		return err
+	}
+
+	deadr, err := NewColdSetReader(s.deadSetPath())
+	if err != nil {
+		return xerrors.Errorf("error opening deadset: %w", err)
+	}
+	defer deadr.Close() //nolint:errcheck
+
+	// 3. Purge dead objects with checkpointing for recovery.
+	// This is the critical section of prune, whereby any dead object not in the markSet is
+	// considered already deleted.
+	// We delete dead objects in batches, holding the transaction lock, where we check the markSet
+	// again for new references created by the caller.
+	// After each batch we write a checkpoint to disk; if the process is interrupted before completion
+	// the process will continue from the checkpoint in the next recovery.
+	if err := s.beginCriticalSection(markSet); err != nil {
+		return xerrors.Errorf("error beginning critical section: %w", err)
+	}
+
+	if err := s.checkClosing(); err != nil {
+		return err
+	}
+
+	checkpoint, err := NewCheckpoint(s.pruneCheckpointPath())
+	if err != nil {
+		return xerrors.Errorf("error creating checkpoint: %w", err)
+	}
+	defer checkpoint.Close() //nolint:errcheck
+
+	log.Info("purging dead objects from the coldstore")
+	startPurge := time.Now()
+	err = s.purge(deadr, checkpoint, markSet)
+	if err != nil {
+		return xerrors.Errorf("error purging dead objects: %w", err)
+	}
+	log.Infow("purging dead objects from coldstore done", "took", time.Since(startPurge))
+
+	s.endCriticalSection()
+
+	if err := checkpoint.Close(); err != nil {
+		log.Warnf("error closing checkpoint: %s", err)
+	}
+	if err := os.Remove(s.pruneCheckpointPath()); err != nil {
+		log.Warnf("error removing checkpoint: %s", err)
+	}
+	if err := deadr.Close(); err != nil {
+		log.Warnf("error closing deadset: %s", err)
+	}
+	if err := os.Remove(s.deadSetPath()); err != nil {
+		log.Warnf("error removing deadset: %s", err)
+	}
+
+	// we are done; do some housekeeping
+	s.endTxnProtect()
+	err = doGC()
+	if err != nil {
+		log.Warnf("error garbage collecting cold store: %s", err)
+	}
+
+	if err := s.setPruneEpoch(boundaryEpoch); err != nil {
+		return xerrors.Errorf("error saving prune base epoch: %w", err)
+	}
+
+	s.pruneIndex++
+	err = s.ds.Put(s.ctx, pruneIndexKey, int64ToBytes(s.compactionIndex))
+	if err != nil {
+		return xerrors.Errorf("error saving compaction index: %w", err)
+	}
+
+	return nil
+}
+
+func (s *SplitStore) completePrune() error {
+	checkpoint, last, err := OpenCheckpoint(s.pruneCheckpointPath())
+	if err != nil {
+		return xerrors.Errorf("error opening checkpoint: %w", err)
+	}
+	defer checkpoint.Close() //nolint:errcheck
+
+	deadr, err := NewColdSetReader(s.deadSetPath())
+	if err != nil {
+		return xerrors.Errorf("error opening deadset: %w", err)
+	}
+	defer deadr.Close() //nolint:errcheck
+
+	markSet, err := s.markSetEnv.Recover("live")
+	if err != nil {
+		return xerrors.Errorf("error recovering markset: %w", err)
+	}
+	defer markSet.Close() //nolint:errcheck
+
+	// PURGE!
+	s.compactType = cold
+	log.Info("purging dead objects from the coldstore")
+	startPurge := time.Now()
+	err = s.completePurge(deadr, checkpoint, last, markSet)
+	if err != nil {
+		return xerrors.Errorf("error purgin dead objects: %w", err)
+	}
+	log.Infow("purging dead objects from the coldstore done", "took", time.Since(startPurge))
+
+	markSet.EndCriticalSection()
+	s.compactType = none
+
+	if err := checkpoint.Close(); err != nil {
+		log.Warnf("error closing checkpoint: %s", err)
+	}
+	if err := os.Remove(s.pruneCheckpointPath()); err != nil {
+		log.Warnf("error removing checkpoint: %s", err)
+	}
+	if err := deadr.Close(); err != nil {
+		log.Warnf("error closing deadset: %s", err)
+	}
+	if err := os.Remove(s.deadSetPath()); err != nil {
+		log.Warnf("error removing deadset: %s", err)
+	}
+
+	return nil
+}
+
+// like walkChain but peforms a deep walk, using parallel walking with walkObjectLax,
+// whereby all extant messages are retained and state roots are retained if they satisfy
+// the given predicate.
+// missing references are ignored, as we expect to have plenty for snapshot syncs.
+func (s *SplitStore) walkChainDeep(ts *types.TipSet, retainStateP func(int64) bool,
+	f func(cid.Cid) error) error {
+	visited := cid.NewSet()
+	toWalk := ts.Cids()
+	walkCnt := 0
+
+	workers := runtime.NumCPU() / 2
+	if workers < 2 {
+		workers = 2
+	}
+
+	var wg sync.WaitGroup
+	workch := make(chan cid.Cid, 16*workers)
+	errch := make(chan error, workers)
+
+	var once sync.Once
+	defer once.Do(func() { close(workch) })
+
+	push := func(c cid.Cid) error {
+		if !visited.Visit(c) {
+			return nil
+		}
+
+		select {
+		case workch <- c:
+			return nil
+		case err := <-errch:
+			return err
+		}
+	}
+
+	worker := func() {
+		defer wg.Done()
+		for c := range workch {
+			err := s.walkObjectLax(c, f)
+			if err != nil {
+				errch <- xerrors.Errorf("error walking object (cid: %s): %w", c, err)
+				return
+			}
+		}
+	}
+
+	for i := 0; i < workers; i++ {
+		wg.Add(1)
+		go worker()
+	}
+
+	baseEpoch := ts.Height()
+	minEpoch := baseEpoch // for progress report
+	log.Infof("walking at epoch %d", minEpoch)
+
+	walkBlock := func(c cid.Cid) error {
+		if !visited.Visit(c) {
+			return nil
+		}
+
+		walkCnt++
+
+		if err := f(c); err != nil {
+			return err
+		}
+
+		var hdr types.BlockHeader
+		err := s.view(c, func(data []byte) error {
+			return hdr.UnmarshalCBOR(bytes.NewBuffer(data))
+		})
+
+		if err != nil {
+			return xerrors.Errorf("error unmarshaling block header (cid: %s): %w", c, err)
+		}
+
+		if hdr.Height < minEpoch {
+			minEpoch = hdr.Height
+			if minEpoch%10_000 == 0 {
+				log.Infof("walking at epoch %d (walked: %d)", minEpoch, walkCnt)
+			}
+		}
+
+		depth := int64(baseEpoch - hdr.Height)
+		retainState := retainStateP(depth)
+
+		if hdr.Height > 0 {
+			if err := push(hdr.Messages); err != nil {
+				return err
+			}
+			if retainState {
+				if err := push(hdr.ParentMessageReceipts); err != nil {
+					return err
+				}
+			}
+		}
+
+		if retainState || hdr.Height == 0 {
+			if err := push(hdr.ParentStateRoot); err != nil {
+				return err
+			}
+		}
+
+		if hdr.Height > 0 {
+			toWalk = append(toWalk, hdr.Parents...)
+		}
+
+		return nil
+	}
+
+	for len(toWalk) > 0 {
+		// walking can take a while, so check this with every opportunity
+		if err := s.checkClosing(); err != nil {
+			return err
+		}
+
+		select {
+		case err := <-errch:
+			return err
+		default:
+		}
+
+		walking := toWalk
+		toWalk = nil
+		for _, c := range walking {
+			if err := walkBlock(c); err != nil {
+				return xerrors.Errorf("error walking block (cid: %s): %w", c, err)
+			}
+		}
+	}
+
+	once.Do(func() { close(workch) })
+	wg.Wait()
+	select {
+	case err := <-errch:
+		return err
+	default:
+	}
+
+	log.Infow("chain walk done", "walked", walkCnt)
+
+	return nil
+}
+
+// like walkObject but treats missing references laxly; faster version of walkObjectIncomplete
+// without an occurs check.
+func (s *SplitStore) walkObjectLax(c cid.Cid, f func(cid.Cid) error) error {
+	if err := f(c); err != nil {
+		if err == errStopWalk {
+			return nil
+		}
+
+		return err
+	}
+
+	if c.Prefix().Codec != cid.DagCBOR {
+		return nil
+	}
+
+	// check this before recursing
+	if err := s.checkClosing(); err != nil {
+		return err
+	}
+
+	var links []cid.Cid
+	err := s.view(c, func(data []byte) error {
+		return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
+			links = append(links, c)
+		})
+	})
+
+	if err != nil {
+		if ipld.IsNotFound(err) { // not a problem for deep walks
+			return nil
+		}
+
+		return xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err)
+	}
+
+	for _, c := range links {
+		err := s.walkObjectLax(c, f)
+		if err != nil {
+			return xerrors.Errorf("error walking link (cid: %s): %w", c, err)
+		}
+	}
+
+	return nil
+}

blockstore/splitstore/splitstore_reify.go

@@ -5,10 +5,9 @@ import (
 	"runtime"
 	"sync/atomic"
 
-	"golang.org/x/xerrors"
-
 	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
+	"golang.org/x/xerrors"
 )
 
 var (
@@ -84,7 +83,7 @@ func (s *SplitStore) reifyWorker(workch chan cid.Cid) {
 }
 
 func (s *SplitStore) doReify(c cid.Cid) {
-	var toreify, totrack, toforget []cid.Cid
+	var toreify, toforget []cid.Cid
 
 	defer func() {
 		s.reifyMx.Lock()
@@ -93,9 +92,6 @@ func (s *SplitStore) doReify(c cid.Cid) {
 		for _, c := range toreify {
 			delete(s.reifyInProgress, c)
 		}
-		for _, c := range totrack {
-			delete(s.reifyInProgress, c)
-		}
 		for _, c := range toforget {
 			delete(s.reifyInProgress, c)
 		}
@@ -132,19 +128,10 @@ func (s *SplitStore) doReify(c cid.Cid) {
 				return xerrors.Errorf("error checking hotstore: %w", err)
 			}
 
+			// All reified blocks are tracked at reification start
 			if has {
-				if s.txnMarkSet != nil {
-					hasMark, err := s.txnMarkSet.Has(c)
-					if err != nil {
-						log.Warnf("error checking markset: %s", err)
-					} else if hasMark {
-						toforget = append(toforget, c)
-						return errStopWalk
-					}
-				} else {
-					totrack = append(totrack, c)
-					return errStopWalk
-				}
+				toforget = append(toforget, c)
+				return errStopWalk
 			}
 
 			toreify = append(toreify, c)
@@ -156,7 +143,7 @@ func (s *SplitStore) doReify(c cid.Cid) {
 		})
 
 	if err != nil {
-		if xerrors.Is(err, errReifyLimit) {
+		if errors.Is(err, errReifyLimit) {
 			log.Debug("reification aborted; reify limit reached")
 			return
 		}
@@ -191,24 +178,4 @@ func (s *SplitStore) doReify(c cid.Cid) {
 		}
 	}
 
-	if s.txnMarkSet != nil {
-		if len(toreify) > 0 {
-			if err := s.txnMarkSet.MarkMany(toreify); err != nil {
-				log.Warnf("error marking reified objects: %s", err)
-			}
-		}
-		if len(totrack) > 0 {
-			if err := s.txnMarkSet.MarkMany(totrack); err != nil {
-				log.Warnf("error marking tracked objects: %s", err)
-			}
-		}
-	} else {
-		// if txnActive is false these are noops
-		if len(toreify) > 0 {
-			s.trackTxnRefMany(toreify)
-		}
-		if len(totrack) > 0 {
-			s.trackTxnRefMany(totrack)
-		}
-	}
 }

blockstore/splitstore/splitstore_test.go

@@ -11,18 +11,20 @@ import (
 	"testing"
 	"time"
 
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-datastore"
+	dssync "github.com/ipfs/go-datastore/sync"
+	ipld "github.com/ipfs/go-ipld-format"
+	logging "github.com/ipfs/go-log/v2"
+	mh "github.com/multiformats/go-multihash"
+
 	"github.com/filecoin-project/go-state-types/abi"
+
 	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/types/mock"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-	datastore "github.com/ipfs/go-datastore"
-	dssync "github.com/ipfs/go-datastore/sync"
-	logging "github.com/ipfs/go-log/v2"
-	mh "github.com/multiformats/go-multihash"
 )
 
 func init() {
@@ -696,7 +698,7 @@ func (b *mockStore) Get(_ context.Context, cid cid.Cid) (blocks.Block, error) {
 
 	blk, ok := b.set[b.keyOf(cid)]
 	if !ok {
-		return nil, blockstore.ErrNotFound
+		return nil, ipld.ErrNotFound{Cid: cid}
 	}
 	return blk, nil
 }

blockstore/splitstore/splitstore_util.go

@@ -3,10 +3,9 @@ package splitstore
 import (
 	"encoding/binary"
 
-	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 	mh "github.com/multiformats/go-multihash"
+	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-state-types/abi"
 )

blockstore/splitstore/splitstore_warmup.go

@@ -5,13 +5,13 @@ import (
 	"sync/atomic"
 	"time"
 
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
 	"golang.org/x/xerrors"
 
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-
 	"github.com/filecoin-project/go-state-types/abi"
-	bstore "github.com/filecoin-project/lotus/blockstore"
+
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 )
@@ -21,14 +21,14 @@ var (
 	WarmupBoundary = build.Finality
 )
 
-// warmup acuiqres the compaction lock and spawns a goroutine to warm up the hotstore;
+// warmup acquires the compaction lock and spawns a goroutine to warm up the hotstore;
 // this is necessary when we sync from a snapshot or when we enable the splitstore
 // on top of an existing blockstore (which becomes the coldstore).
 func (s *SplitStore) warmup(curTs *types.TipSet) error {
 	if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
 		return xerrors.Errorf("error locking compaction")
 	}
-
+	s.compactType = warmup
 	go func() {
 		defer atomic.StoreInt32(&s.compacting, 0)
 
@@ -88,7 +88,7 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 
 			blk, err := s.cold.Get(s.ctx, c)
 			if err != nil {
-				if err == bstore.ErrNotFound {
+				if ipld.IsNotFound(err) {
 					atomic.AddInt64(missing, 1)
 					return errStopWalk
 				}

blockstore/splitstore/visitor.go

@@ -3,7 +3,7 @@ package splitstore
 import (
 	"sync"
 
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 )
 
 // ObjectVisitor is an interface for deduplicating objects during walks

blockstore/timed.go

@@ -8,6 +8,7 @@ import (
 
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
 	"github.com/raulk/clock"
 	"go.uber.org/multierr"
 )
@@ -112,7 +113,7 @@ func (t *TimedCacheBlockstore) View(ctx context.Context, k cid.Cid, callback fun
 	// calling an arbitrary callback while holding a lock.
 	t.mu.RLock()
 	block, err := t.active.Get(ctx, k)
-	if err == ErrNotFound {
+	if ipld.IsNotFound(err) {
 		block, err = t.inactive.Get(ctx, k)
 	}
 	t.mu.RUnlock()
@@ -127,7 +128,7 @@ func (t *TimedCacheBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 	b, err := t.active.Get(ctx, k)
-	if err == ErrNotFound {
+	if ipld.IsNotFound(err) {
 		b, err = t.inactive.Get(ctx, k)
 	}
 	return b, err
@@ -137,7 +138,7 @@ func (t *TimedCacheBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, err
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 	size, err := t.active.GetSize(ctx, k)
-	if err == ErrNotFound {
+	if ipld.IsNotFound(err) {
 		size, err = t.inactive.GetSize(ctx, k)
 	}
 	return size, err
@@ -175,11 +176,12 @@ func (t *TimedCacheBlockstore) AllKeysChan(_ context.Context) (<-chan cid.Cid, e
 	defer t.mu.RUnlock()
 
 	ch := make(chan cid.Cid, len(t.active)+len(t.inactive))
-	for c := range t.active {
-		ch <- c
+	for _, b := range t.active {
+		ch <- b.Cid()
 	}
-	for c := range t.inactive {
-		if _, ok := t.active[c]; ok {
+	for _, b := range t.inactive {
+		c := b.Cid()
+		if _, ok := t.active[string(c.Hash())]; ok {
 			continue
 		}
 		ch <- c

blockstore/timed_test.go

@@ -6,11 +6,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/raulk/clock"
-	"github.com/stretchr/testify/require"
-
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	"github.com/raulk/clock"
+	"github.com/stretchr/testify/require"
 )
 
 func TestTimedCacheBlockstoreSimple(t *testing.T) {

blockstore/union.go

@@ -5,6 +5,7 @@ import (
 
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
 )
 
 type unionBlockstore []Blockstore
@@ -13,7 +14,7 @@ type unionBlockstore []Blockstore
 //
 // * Reads return from the first blockstore that has the value, querying in the
 //   supplied order.
-// * Writes (puts and deltes) are broadcast to all stores.
+// * Writes (puts and deletes) are broadcast to all stores.
 //
 func Union(stores ...Blockstore) Blockstore {
 	return unionBlockstore(stores)
@@ -30,7 +31,7 @@ func (m unionBlockstore) Has(ctx context.Context, cid cid.Cid) (has bool, err er
 
 func (m unionBlockstore) Get(ctx context.Context, cid cid.Cid) (blk blocks.Block, err error) {
 	for _, bs := range m {
-		if blk, err = bs.Get(ctx, cid); err == nil || err != ErrNotFound {
+		if blk, err = bs.Get(ctx, cid); err == nil || !ipld.IsNotFound(err) {
 			break
 		}
 	}
@@ -39,7 +40,7 @@ func (m unionBlockstore) Get(ctx context.Context, cid cid.Cid) (blk blocks.Block
 
 func (m unionBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) (err error) {
 	for _, bs := range m {
-		if err = bs.View(ctx, cid, callback); err == nil || err != ErrNotFound {
+		if err = bs.View(ctx, cid, callback); err == nil || !ipld.IsNotFound(err) {
 			break
 		}
 	}
@@ -48,7 +49,7 @@ func (m unionBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]
 
 func (m unionBlockstore) GetSize(ctx context.Context, cid cid.Cid) (size int, err error) {
 	for _, bs := range m {
-		if size, err = bs.GetSize(ctx, cid); err == nil || err != ErrNotFound {
+		if size, err = bs.GetSize(ctx, cid); err == nil || !ipld.IsNotFound(err) {
 			break
 		}
 	}

build/actors/README.md

@@ -0,0 +1,26 @@
+# Bundles
+
+This directory includes the actors bundles for each release. Each actor bundle is a zstd compressed
+tarfile containing one bundle per network type. These tarfiles are subsequently embedded in the
+lotus binary.
+
+## Updating
+
+To update, run the `./pack.sh` script. For example, the following will pack the [builtin actors release](https://github.com/filecoin-project/builtin-actors/releases) `dev/20220602` into the `v8` tarfile.
+
+```bash
+./pack.sh v8 dev/20220602
+```
+
+This will:
+
+1. Download the actors bundles and pack them into the appropriate tarfile (`$VERSION.tar.zst`).
+2. Run `make bundle-gen` in the top-level directory to regenerate the bundle metadata file for _all_ network versions (all `*.tar.zst` files in this directory).
+
+## Overriding
+
+To build a bundle, but specify a different release/tag for a specific network, append `$network=$alternative_release` on the command line. For example:
+
+```bash
+./pack.sh v8 dev/20220602 mainnet=v8.0.0 calibrationnet=v8.0.0-rc.1
+```

build/actors/pack.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+NETWORKS=(devnet mainnet caterpillarnet butterflynet testing testing-fake-proofs calibrationnet)
+
+set -e
+
+if [[ $# -lt 2 ]]; then
+    echo "Usage: $0 VERSION RELEASE [NETWORK=RELEASE_OVERRIDE]..." >&2
+    echo "expected at least two arguments, an actors version (e.g., v8), an actors release, and any number of release overrides." >&2
+    exit 1
+fi
+
+VERSION="$1" # actors version
+RELEASE="$2" # actors release name
+RELEASE_OVERRIDES=("${@:3}")
+
+echo "Downloading bundles for actors version ${VERSION} release ${RELEASE}"
+echo "With release overrides ${RELEASE_OVERRIDES[*]}"
+
+TARGET_FILE="$(pwd)/${VERSION}.tar.zst"
+WORKDIR=$(mktemp -d -t "actor-bundles-${VERSION}.XXXXXXXXXX")
+trap 'rm -rf -- "$WORKDIR"' EXIT
+
+encode_release() {
+    jq -rn --arg release "$1" '$release | @uri'
+}
+
+pushd "${WORKDIR}"
+for network in "${NETWORKS[@]}"; do
+    release="$RELEASE"
+    # Ideally, we'd use an associative array (map). But that's not supported on macos.
+    for override in "${RELEASE_OVERRIDES[@]}"; do
+        if [[ "${network}" = "${override%%=*}" ]]; then
+            release="${override#*=}"
+            break
+        fi
+    done
+    encoded_release="$(encode_release "$release")"
+    echo "Downloading $release for network $network."
+    wget "https://github.com/filecoin-project/builtin-actors/releases/download/${encoded_release}/builtin-actors-${network}"{.car,.sha256}
+done
+
+echo "Checking the checksums..."
+
+sha256sum -c -- *.sha256
+
+echo "Packing..."
+
+rm -f -- "$TARGET_FILE"
+tar -cf "$TARGET_FILE" --use-compress-program "zstd -19" -- *.car
+popd
+
+echo "Generating metadata..."
+
+make -C ../../ bundle-gen

build/bootstrap.go

@@ -6,9 +6,9 @@ import (
 	"path"
 	"strings"
 
-	"github.com/filecoin-project/lotus/lib/addrutil"
-
 	"github.com/libp2p/go-libp2p-core/peer"
+
+	"github.com/filecoin-project/lotus/lib/addrutil"
 )
 
 //go:embed bootstrap

build/bootstrap/butterflynet.pi

@@ -1,2 +1,2 @@
-/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWFHDtFx7CVTy4xoCDutVo1cScvSnQjDeaM8UzwVS1qwkh
-/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWKt8cwpkiumkT8x32c3YFxsPRwhV5J8hCYPn9mhUmcAXt
+/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWSUZhAY3eyoPUboJ1ZWe4dNPFWTr1EPoDjbTDSAN15uhY
+/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWDfvNrSRVGWAGbn3sm9C8z98W2x25qCZjaXGHXmGiH24e

build/bootstrap/interopnet.pi

@@ -1,2 +1,2 @@
-/dns4/bootstrap-0.interop.fildev.network/tcp/1347/p2p/12D3KooWLGPq9JL1xwL6gHok7HSNxtK1Q5kyfg4Hk69ifRPghn4i
-/dns4/bootstrap-1.interop.fildev.network/tcp/1347/p2p/12D3KooWFYS1f31zafv8mqqYu8U3hEqYvaZ6avWzYU3BmZdpyH3h
+/dns4/bootstrap-0.interop.fildev.network/tcp/1347/p2p/12D3KooWDpppr8csCNvEPnD2Z83KTPdBTM7iJhL66qK8LK3bB5NU
+/dns4/bootstrap-1.interop.fildev.network/tcp/1347/p2p/12D3KooWR3K1sXWoDYcXWqDF26mFEM1o1g7e7fcVR3NYE7rn24Gs

build/bootstrap/mainnet.pi

@@ -13,4 +13,4 @@
 /dns4/node.glif.io/tcp/1235/p2p/12D3KooWBF8cpp65hp2u9LK5mh19x67ftAam84z9LsfaquTDSBpt
 /dns4/bootstrap-0.ipfsmain.cn/tcp/34721/p2p/12D3KooWQnwEGNqcM2nAcPtRR9rAX8Hrg4k9kJLCHoTR5chJfz6d
 /dns4/bootstrap-1.ipfsmain.cn/tcp/34723/p2p/12D3KooWMKxMkD5DMpSWsW7dBddKxKT7L2GgbNuckz9otxvkvByP
-/dns4/bootstarp-0.1475.io/tcp/61256/p2p/12D3KooWQjaNmbz9b1XmheQB3RWsRjKSzuRLfjeiDZHyX7Y5RcBr
+/dns4/bootstarp-0.1475.io/tcp/61256/p2p/12D3KooWRzCVDwHUkgdK7eRgnoXbjDAELhxPErjHzbRLguSV1aRt

build/builtin_actors.go

@@ -0,0 +1,263 @@
+package build
+
+import (
+	"archive/tar"
+	"context"
+	"embed"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/DataDog/zstd"
+	"github.com/ipfs/go-cid"
+	cbor "github.com/ipfs/go-ipld-cbor"
+	"github.com/ipld/go-car"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/adt"
+)
+
+//go:embed actors/*.tar.zst
+var embeddedBuiltinActorReleases embed.FS
+
+func init() {
+	if BundleOverrides == nil {
+		BundleOverrides = make(map[actors.Version]string)
+	}
+	for _, av := range actors.Versions {
+		path := os.Getenv(fmt.Sprintf("LOTUS_BUILTIN_ACTORS_V%d_BUNDLE", av))
+		if path == "" {
+			continue
+		}
+		BundleOverrides[actors.Version(av)] = path
+	}
+	if err := loadManifests(NetworkBundle); err != nil {
+		panic(err)
+	}
+}
+
+// UseNetworkBundle switches to a different network bundle, by name.
+func UseNetworkBundle(netw string) error {
+	if NetworkBundle == netw {
+		return nil
+	}
+	if err := loadManifests(netw); err != nil {
+		return err
+	}
+	NetworkBundle = netw
+	return nil
+}
+
+func loadManifests(netw string) error {
+	overridden := make(map[actors.Version]struct{})
+	var newMetadata []*BuiltinActorsMetadata
+	// First, prefer overrides.
+	for av, path := range BundleOverrides {
+		root, actorCids, err := readBundleManifestFromFile(path)
+		if err != nil {
+			return err
+		}
+		newMetadata = append(newMetadata, &BuiltinActorsMetadata{
+			Network:     netw,
+			Version:     av,
+			ManifestCid: root,
+			Actors:      actorCids,
+		})
+		overridden[av] = struct{}{}
+	}
+
+	// Then load embedded bundle metadata.
+	for _, meta := range EmbeddedBuiltinActorsMetadata {
+		if meta.Network != netw {
+			continue
+		}
+		if _, ok := overridden[meta.Version]; ok {
+			continue
+		}
+		newMetadata = append(newMetadata, meta)
+	}
+
+	actors.ClearManifests()
+
+	for _, meta := range newMetadata {
+		actors.RegisterManifest(meta.Version, meta.ManifestCid, meta.Actors)
+	}
+
+	return nil
+}
+
+type BuiltinActorsMetadata struct {
+	Network     string
+	Version     actors.Version
+	ManifestCid cid.Cid
+	Actors      map[string]cid.Cid
+}
+
+// ReadEmbeddedBuiltinActorsMetadata reads the metadata from the embedded built-in actor bundles.
+// There should be no need to call this method as the result is cached in the
+// `EmbeddedBuiltinActorsMetadata` variable on `make gen`.
+func ReadEmbeddedBuiltinActorsMetadata() ([]*BuiltinActorsMetadata, error) {
+	files, err := embeddedBuiltinActorReleases.ReadDir("actors")
+	if err != nil {
+		return nil, xerrors.Errorf("failed to read embedded bundle directory: %s", err)
+	}
+	var bundles []*BuiltinActorsMetadata
+	for _, dirent := range files {
+		name := dirent.Name()
+		b, err := readEmbeddedBuiltinActorsMetadata(name)
+		if err != nil {
+			return nil, err
+		}
+		bundles = append(bundles, b...)
+	}
+	// Sort by network, then by bundle.
+	sort.Slice(bundles, func(i, j int) bool {
+		if bundles[i].Network == bundles[j].Network {
+			return bundles[i].Version < bundles[j].Version
+		}
+		return bundles[i].Network < bundles[j].Network
+	})
+	return bundles, nil
+}
+
+func readEmbeddedBuiltinActorsMetadata(bundle string) ([]*BuiltinActorsMetadata, error) {
+	const (
+		archiveExt   = ".tar.zst"
+		bundleExt    = ".car"
+		bundlePrefix = "builtin-actors-"
+	)
+
+	if !strings.HasPrefix(bundle, "v") {
+		return nil, xerrors.Errorf("bundle bundle '%q' doesn't start with a 'v'", bundle)
+	}
+	if !strings.HasSuffix(bundle, archiveExt) {
+		return nil, xerrors.Errorf("bundle bundle '%q' doesn't end with '%s'", bundle, archiveExt)
+	}
+	version, err := strconv.ParseInt(bundle[1:len(bundle)-len(archiveExt)], 10, 0)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to parse actors version from bundle '%q': %s", bundle, err)
+	}
+	fi, err := embeddedBuiltinActorReleases.Open(fmt.Sprintf("actors/%s", bundle))
+	if err != nil {
+		return nil, err
+	}
+	defer fi.Close() //nolint
+
+	uncompressed := zstd.NewReader(fi)
+	defer uncompressed.Close() //nolint
+
+	var bundles []*BuiltinActorsMetadata
+
+	tarReader := tar.NewReader(uncompressed)
+	for {
+		header, err := tarReader.Next()
+		switch err {
+		case io.EOF:
+			return bundles, nil
+		case nil:
+		default:
+			return nil, err
+		}
+
+		// Read the network name from the bundle name.
+		name := path.Base(header.Name)
+		if !strings.HasSuffix(name, bundleExt) {
+			return nil, xerrors.Errorf("expected bundle to end with .car: %s", name)
+		}
+		if !strings.HasPrefix(name, bundlePrefix) {
+			return nil, xerrors.Errorf("expected bundle to end with .car: %s", name)
+		}
+		name = name[len(bundlePrefix) : len(name)-len(bundleExt)]
+
+		// Load the bundle.
+		root, actorCids, err := readBundleManifest(tarReader)
+		if err != nil {
+			return nil, xerrors.Errorf("error loading builtin actors bundle: %w", err)
+		}
+		bundles = append(bundles, &BuiltinActorsMetadata{
+			Network:     name,
+			Version:     actors.Version(version),
+			ManifestCid: root,
+			Actors:      actorCids,
+		})
+	}
+}
+
+func readBundleManifestFromFile(path string) (cid.Cid, map[string]cid.Cid, error) {
+	fi, err := os.Open(path)
+	if err != nil {
+		return cid.Undef, nil, err
+	}
+	defer fi.Close() //nolint
+
+	return readBundleManifest(fi)
+}
+
+func readBundleManifest(r io.Reader) (cid.Cid, map[string]cid.Cid, error) {
+	// Load the bundle.
+	bs := blockstore.NewMemory()
+	hdr, err := car.LoadCar(context.Background(), bs, r)
+	if err != nil {
+		return cid.Undef, nil, xerrors.Errorf("error loading builtin actors bundle: %w", err)
+	}
+
+	if len(hdr.Roots) != 1 {
+		return cid.Undef, nil, xerrors.Errorf("expected one root when loading actors bundle, got %d", len(hdr.Roots))
+	}
+	root := hdr.Roots[0]
+	actorCids, err := actors.ReadManifest(context.Background(), adt.WrapStore(context.Background(), cbor.NewCborStore(bs)), root)
+	if err != nil {
+		return cid.Undef, nil, err
+	}
+
+	// Make sure we have all the actors.
+	for name, c := range actorCids {
+		if has, err := bs.Has(context.Background(), c); err != nil {
+			return cid.Undef, nil, xerrors.Errorf("got an error when checking that the bundle has the actor %q: %w", name, err)
+		} else if !has {
+			return cid.Undef, nil, xerrors.Errorf("actor %q missing from bundle", name)
+		}
+	}
+
+	return root, actorCids, nil
+}
+
+// GetEmbeddedBuiltinActorsBundle returns the builtin-actors bundle for the given actors version.
+func GetEmbeddedBuiltinActorsBundle(version actors.Version) ([]byte, bool) {
+	fi, err := embeddedBuiltinActorReleases.Open(fmt.Sprintf("actors/v%d.tar.zst", version))
+	if err != nil {
+		return nil, false
+	}
+	defer fi.Close() //nolint
+
+	uncompressed := zstd.NewReader(fi)
+	defer uncompressed.Close() //nolint
+
+	tarReader := tar.NewReader(uncompressed)
+	targetFileName := fmt.Sprintf("builtin-actors-%s.car", NetworkBundle)
+	for {
+		header, err := tarReader.Next()
+		switch err {
+		case io.EOF:
+			return nil, false
+		case nil:
+		default:
+			panic(err)
+		}
+		if header.Name != targetFileName {
+			continue
+		}
+
+		car, err := io.ReadAll(tarReader)
+		if err != nil {
+			panic(err)
+		}
+		return car, true
+	}
+}

build/builtin_actors_gen.go

@@ -0,0 +1,128 @@
+// WARNING: This file has automatically been generated
+
+package build
+
+import (
+	"github.com/ipfs/go-cid"
+)
+
+var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMetadata{{
+	Network:     "butterflynet",
+	Version:     8,
+	ManifestCid: MustParseCid("bafy2bzacedvaarfyh6q3bk4dyzux46ednlace2ckxp5nbyn6mb3da2apqn6sk"),
+	Actors: map[string]cid.Cid{
+		"account":          MustParseCid("bafk2bzaceavzeu4gqte7o33vr4htiaapiwpfq6p26tgdkqla2baqhmiqswfso"),
+		"cron":             MustParseCid("bafk2bzacech35onpqxep4yox36k7sr4mj4bch54s3i4b3yyaustrbo5xwfbfs"),
+		"init":             MustParseCid("bafk2bzaceahxin3sf5f6ude5j6we4yeqlg66s5qe4tu7lwp26jcg7yp2ns6hi"),
+		"multisig":         MustParseCid("bafk2bzacectfmzjtniypgl4whm42sws5aupihqgfikwsr7p5yoq3bmqaogldi"),
+		"paymentchannel":   MustParseCid("bafk2bzacecbwu54ce5mjgp2pqxyj6kpn2vlgiu5wv2lj2byjiegxnn3infd5i"),
+		"reward":           MustParseCid("bafk2bzacecskkbhe6c4ud5jt62wg4w7j7shj6xdwoyic74s5y6pgywxxvnw72"),
+		"storagemarket":    MustParseCid("bafk2bzacebycxcwwm7hwhuhpasaskil2kxaqb7tins7azdvvm72rorlciuysi"),
+		"storageminer":     MustParseCid("bafk2bzacecgx3etor5m6lahpmjdwqnryutqe6naiurfhgsju72rd4nqssutbg"),
+		"storagepower":     MustParseCid("bafk2bzaceayvy6xyp5cwtngm457c5hssvihidppgq3o7gy3dlmhgor3yzujoc"),
+		"system":           MustParseCid("bafk2bzacec6xctjxybp7r3kkhase56o6jsaiua7ure5ttu2xfuojt4jhlsoa6"),
+		"verifiedregistry": MustParseCid("bafk2bzacec2hcqlqcfacylfcrhhliwkisvh4y3adwt47xkf2gdvodwu6ccepc"),
+	},
+}, {
+	Network:     "calibrationnet",
+	Version:     8,
+	ManifestCid: MustParseCid("bafy2bzacedrdn6z3z7xz7lx4wll3tlgktirhllzqxb766dxpaqp3ukxsjfsba"),
+	Actors: map[string]cid.Cid{
+		"account":          MustParseCid("bafk2bzacecruossn66xqbeutqx5r4k2kjzgd43frmwd4qkw6haez44ubvvpxo"),
+		"cron":             MustParseCid("bafk2bzaceaxlezmclw5ugldhhtfgvn7yztux45scqik3ez4yhwiqhg5ssib44"),
+		"init":             MustParseCid("bafk2bzaceadyfilb22bcvzvnpzbg2lyg6npmperyq6es2brvzjdh5rmywc4ry"),
+		"multisig":         MustParseCid("bafk2bzacec66wmb4kohuzvuxsulhcgiwju7sqkldwfpmmgw7dbbwgm5l2574q"),
+		"paymentchannel":   MustParseCid("bafk2bzaceblot4pemhfgwb3lceellwrpgxaqkpselzbpqu32maffpopdunlha"),
+		"reward":           MustParseCid("bafk2bzaceayah37uvj7brl5no4gmvmqbmtndh5raywuts7h6tqbgbq2ge7dhu"),
+		"storagemarket":    MustParseCid("bafk2bzacebotg5coqnglzsdrqxtkqk2eq4krxt6zvds3i3vb2yejgxhexl2n6"),
+		"storageminer":     MustParseCid("bafk2bzacea6rabflc7kpwr6y4lzcqsnuahr4zblyq3rhzrrsfceeiw2lufrb4"),
+		"storagepower":     MustParseCid("bafk2bzacecpwr4mynn55bg5hrlns3osvg7sty3rca6zlai3vl52vbbjk7ulfa"),
+		"system":           MustParseCid("bafk2bzaceaqrkllksxv2jsfgjvmuewx5vbzrammw5mdscod6gkdr3ijih2q64"),
+		"verifiedregistry": MustParseCid("bafk2bzaceaihibfu625lbtzdp3tcftscshrmbgghgrc7kzqhxn4455pycpdkm"),
+	},
+}, {
+	Network:     "caterpillarnet",
+	Version:     8,
+	ManifestCid: MustParseCid("bafy2bzacecsmunz6fzhg53276cixadn6ybhcnzkgbw3la5hf342tfxsdoet26"),
+	Actors: map[string]cid.Cid{
+		"account":          MustParseCid("bafk2bzaced6yatl4y2nmqmx2h3btk3np6oelyw2yt57elsb2nfmm33fadzt2g"),
+		"cron":             MustParseCid("bafk2bzacebrujytq4u7g62jbz52gio5k2s6rhruty7nt4eqq7ygitzxuee5zi"),
+		"init":             MustParseCid("bafk2bzacedajw5ptnwfdidv6m4rvd4c2m7dve4lhfbawygl5idkalcxbiiudu"),
+		"multisig":         MustParseCid("bafk2bzaceb3kh5hjh6eebb5236xp7crn2owyyo7irap6sy4ns76uc7om6pxuy"),
+		"paymentchannel":   MustParseCid("bafk2bzacedl5am53e4mtxpzligcycxvmkolfkhfiuavww2dq3ukgaqwowj7vw"),
+		"reward":           MustParseCid("bafk2bzacecbswf242j43cymj3wh7nszawwlofv6z6z4qipb5d32hpxdhxywng"),
+		"storagemarket":    MustParseCid("bafk2bzaceca5ersmg3zxf2cztgktq33bmfjuiqjcjlktwj52xyrpujbdsqvek"),
+		"storageminer":     MustParseCid("bafk2bzacedg2fqaq5udfp3h6cxhywm27dgagxtselfgkyyyunqq362eaxpdm4"),
+		"storagepower":     MustParseCid("bafk2bzaceb3dm2i2q323e6iozo3r6pyded645vvlpf537kga2a3hu5x7abgl4"),
+		"system":           MustParseCid("bafk2bzacebu47th3xerlngqavlipb6cfu2utljkxxzgadc3totogto2tmx2jc"),
+		"verifiedregistry": MustParseCid("bafk2bzaceci3niq3rmbcmepgn27zvlgci6d5t4dvthx3pbmmx3wcu5elova6i"),
+	},
+}, {
+	Network:     "devnet",
+	Version:     8,
+	ManifestCid: MustParseCid("bafy2bzacedq7tuibavyqxzkq4uybjj7ly22eu42mjkoehwn5d47xfunmtjm4k"),
+	Actors: map[string]cid.Cid{
+		"account":          MustParseCid("bafk2bzacea4tlgnp7m6tlldpz3termlwxlnyq24nwd4zdzv4r6nsjuaktuuzc"),
+		"cron":             MustParseCid("bafk2bzacecgrlf3vg3mufwovddlbgclhpnpp3jftr46stssh3crd3pyljc37w"),
+		"init":             MustParseCid("bafk2bzacedarbnovmucppbjkcwsxopludrj5ttmtm7mzfqsugmxdnqevqso7o"),
+		"multisig":         MustParseCid("bafk2bzaced4gcxjwy6garxwfw6y5a2k4jewj4t5nzopjy4qwnimhjtnsgo3ss"),
+		"paymentchannel":   MustParseCid("bafk2bzaceb3isfguytt6cs4xecyoonbhhekmngfbap2msggbwyde7zch3a6w4"),
+		"reward":           MustParseCid("bafk2bzacedn3fkp27ys5dxn4pwqdq2atj2x6cyezxuekdorvjwi7zazirgvgy"),
+		"storagemarket":    MustParseCid("bafk2bzacecw57fpkqesfhi5g3nr4csy4oy7oc42wmwjuis6l7ijniolo4rt2k"),
+		"storageminer":     MustParseCid("bafk2bzacebze3elvppssc6v5457ukszzy6ndrg6xgaojfsqfbbtg3xfwo4rbs"),
+		"storagepower":     MustParseCid("bafk2bzaceb45l6zhgc34n6clz7xnvd7ek55bhw46q25umuje34t6kroix6hh6"),
+		"system":           MustParseCid("bafk2bzacecf7eta2stfd3cnuxzervd33imbvlaqq6b5tsho7pxmhifrybreru"),
+		"verifiedregistry": MustParseCid("bafk2bzaceaajgtglewgitshgdi2nzrvq7eihjtyqj5yiamesqun2hujl3xev2"),
+	},
+}, {
+	Network:     "mainnet",
+	Version:     8,
+	ManifestCid: MustParseCid("bafy2bzacebogjbpiemi7npzxchgcjjki3tfxon4ims55obfyfleqntteljsea"),
+	Actors: map[string]cid.Cid{
+		"account":          MustParseCid("bafk2bzacedudbf7fc5va57t3tmo63snmt3en4iaidv4vo3qlyacbxaa6hlx6y"),
+		"cron":             MustParseCid("bafk2bzacecqb3eolfurehny6yp7tgmapib4ocazo5ilkopjce2c7wc2bcec62"),
+		"init":             MustParseCid("bafk2bzaceaipvjhoxmtofsnv3aj6gj5ida4afdrxa4ewku2hfipdlxpaektlw"),
+		"multisig":         MustParseCid("bafk2bzacebhldfjuy4o5v7amrhp5p2gzv2qo5275jut4adnbyp56fxkwy5fag"),
+		"paymentchannel":   MustParseCid("bafk2bzacebalad3f72wyk7qyilvfjijcwubdspytnyzlrhvn73254gqis44rq"),
+		"reward":           MustParseCid("bafk2bzacecwzzxlgjiavnc3545cqqil3cmq4hgpvfp2crguxy2pl5ybusfsbe"),
+		"storagemarket":    MustParseCid("bafk2bzacediohrxkp2fbsl4yj4jlupjdkgsiwqb4zuezvinhdo2j5hrxco62q"),
+		"storageminer":     MustParseCid("bafk2bzacecgnynvd3tene3bvqoknuspit56canij5bpra6wl4mrq2mxxwriyu"),
+		"storagepower":     MustParseCid("bafk2bzacebjvqva6ppvysn5xpmiqcdfelwbbcxmghx5ww6hr37cgred6dyrpm"),
+		"system":           MustParseCid("bafk2bzacedwq5uppsw7vp55zpj7jdieizirmldceehu6wvombw3ixq2tcq57w"),
+		"verifiedregistry": MustParseCid("bafk2bzaceb3zbkjz3auizmoln2unmxep7dyfcmsre64vnqfhdyh7rkqfoxlw4"),
+	},
+}, {
+	Network:     "testing",
+	Version:     8,
+	ManifestCid: MustParseCid("bafy2bzacedkjpqx27wgsvfxzuxfvixuxtbpt2y6yo6igcasez6gqiowron776"),
+	Actors: map[string]cid.Cid{
+		"account":          MustParseCid("bafk2bzacebmfbtdj5vruje5auacrhhprcjdd6uclhukb7je7t2f6ozfcgqlu2"),
+		"cron":             MustParseCid("bafk2bzacea4gwsbeux7z4yxvpkxpco77iyxijoyqaoikofrxdewunwh3unjem"),
+		"init":             MustParseCid("bafk2bzacecqk6zlwein7tzy7yrrhtj4pzavrkofgpyxvvw5ktr3w4x4ml4lis"),
+		"multisig":         MustParseCid("bafk2bzacea5zp2g6ag5qfuro7zw6kyku2swxs57wjxncaaxbih5iqflqy4ghm"),
+		"paymentchannel":   MustParseCid("bafk2bzaced47dbtbygmfwnyfsp5iihzhhdmnkpuyc5nlnfgc4mkkvlsgvj2do"),
+		"reward":           MustParseCid("bafk2bzacecmcagk32pzdzfg7piobzqhlgla37x3g7jjzyndlz7mqdno2zulfi"),
+		"storagemarket":    MustParseCid("bafk2bzaceballmgd7puoixfwm65f5shi3kzreqdisowtsoufbvduwytydqotw"),
+		"storageminer":     MustParseCid("bafk2bzacebucngwdhxtod2gvv52adtdssafyg43znsoy4omtfkkqe2hbhvxeu"),
+		"storagepower":     MustParseCid("bafk2bzaceakxw5wx3rtqoarrdbzhmxkufg2kx7n34xotzxzacvvbe5iqggmsa"),
+		"system":           MustParseCid("bafk2bzaced6kjkbv7lrb2qwq5we2hqaxc6ztch5p52g27qtjy45zdemsk4b7m"),
+		"verifiedregistry": MustParseCid("bafk2bzacectzxvtoselhnzsair5nv6k5vokvegnht6z2lfee4p3xexo4kg4m6"),
+	},
+}, {
+	Network:     "testing-fake-proofs",
+	Version:     8,
+	ManifestCid: MustParseCid("bafy2bzacecd3lb5v6tzjylnhnrhexslssyaozy6hogzgpkhztoe76exbrgrug"),
+	Actors: map[string]cid.Cid{
+		"account":          MustParseCid("bafk2bzacebmfbtdj5vruje5auacrhhprcjdd6uclhukb7je7t2f6ozfcgqlu2"),
+		"cron":             MustParseCid("bafk2bzacea4gwsbeux7z4yxvpkxpco77iyxijoyqaoikofrxdewunwh3unjem"),
+		"init":             MustParseCid("bafk2bzacebwkqd6e7gdphfzw2kdmbokdh2bly6fvzgfopxzy7quq4l67gmkks"),
+		"multisig":         MustParseCid("bafk2bzacea5zp2g6ag5qfuro7zw6kyku2swxs57wjxncaaxbih5iqflqy4ghm"),
+		"paymentchannel":   MustParseCid("bafk2bzaced47dbtbygmfwnyfsp5iihzhhdmnkpuyc5nlnfgc4mkkvlsgvj2do"),
+		"reward":           MustParseCid("bafk2bzacecmcagk32pzdzfg7piobzqhlgla37x3g7jjzyndlz7mqdno2zulfi"),
+		"storagemarket":    MustParseCid("bafk2bzacecxqgajcaednamgolc6wc3lzbjc6tz5alfrbwqez2y3c372vts6dg"),
+		"storageminer":     MustParseCid("bafk2bzaceaqwxllfycpq6decpsnkqjdeycpysh5acubonjae7u3wciydlkvki"),
+		"storagepower":     MustParseCid("bafk2bzaceddmeolsokbxgcr25cuf2skrobtmmoof3dmqfpcfp33lmw63oikvm"),
+		"system":           MustParseCid("bafk2bzaced6kjkbv7lrb2qwq5we2hqaxc6ztch5p52g27qtjy45zdemsk4b7m"),
+		"verifiedregistry": MustParseCid("bafk2bzacectzxvtoselhnzsair5nv6k5vokvegnht6z2lfee4p3xexo4kg4m6"),
+	},
+}}

build/builtin_actors_test.go

@@ -0,0 +1,34 @@
+package build_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors"
+)
+
+// Test that the embedded metadata is correct.
+func TestEmbeddedMetadata(t *testing.T) {
+	metadata, err := build.ReadEmbeddedBuiltinActorsMetadata()
+	require.NoError(t, err)
+
+	require.Equal(t, metadata, build.EmbeddedBuiltinActorsMetadata)
+}
+
+// Test that we're registering the manifest correctly.
+func TestRegistration(t *testing.T) {
+	manifestCid, found := actors.GetManifest(actors.Version8)
+	require.True(t, found)
+	require.True(t, manifestCid.Defined())
+
+	for _, key := range actors.GetBuiltinActorsKeys() {
+		actorCid, found := actors.GetActorCodeID(actors.Version8, key)
+		require.True(t, found)
+		name, version, found := actors.GetActorMetaByCode(actorCid)
+		require.True(t, found)
+		require.Equal(t, actors.Version8, version)
+		require.Equal(t, key, name)
+	}
+}

build/openrpc.go

@@ -52,3 +52,11 @@ func OpenRPCDiscoverJSON_Worker() apitypes.OpenRPCDocument {
 	}
 	return mustReadGzippedOpenRPCDocument(data)
 }
+
+func OpenRPCDiscoverJSON_Gateway() apitypes.OpenRPCDocument {
+	data, err := openrpcfs.ReadFile("openrpc/gateway.json.gz")
+	if err != nil {
+		panic(err)
+	}
+	return mustReadGzippedOpenRPCDocument(data)
+}

build/openrpc_test.go

@@ -16,6 +16,7 @@ func TestOpenRPCDiscoverJSON_Version(t *testing.T) {
 		OpenRPCDiscoverJSON_Full,
 		OpenRPCDiscoverJSON_Miner,
 		OpenRPCDiscoverJSON_Worker,
+		OpenRPCDiscoverJSON_Gateway,
 	} {
 		doc := docFn()
 		if got, ok := doc["openrpc"]; !ok || got != openRPCDocVersion {

build/params_2k.go

@@ -11,13 +11,18 @@ import (
 
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/network"
+
+	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
 )
 
 const BootstrappersFile = ""
 const GenesisFile = ""
 
-const GenesisNetworkVersion = network.Version15
+var NetworkBundle = "devnet"
+var BundleOverrides map[actors.Version]string
+
+const GenesisNetworkVersion = network.Version16
 
 var UpgradeBreezeHeight = abi.ChainEpoch(-1)
 
@@ -49,6 +54,8 @@ var UpgradeChocolateHeight = abi.ChainEpoch(-17)
 
 var UpgradeOhSnapHeight = abi.ChainEpoch(-18)
 
+var UpgradeSkyrHeight = abi.ChainEpoch(-19)
+
 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 	0: DrandMainnet,
 }
@@ -99,6 +106,7 @@ func init() {
 	UpgradeHyperdriveHeight = getUpgradeHeight("LOTUS_HYPERDRIVE_HEIGHT", UpgradeHyperdriveHeight)
 	UpgradeChocolateHeight = getUpgradeHeight("LOTUS_CHOCOLATE_HEIGHT", UpgradeChocolateHeight)
 	UpgradeOhSnapHeight = getUpgradeHeight("LOTUS_OHSNAP_HEIGHT", UpgradeOhSnapHeight)
+	UpgradeSkyrHeight = getUpgradeHeight("LOTUS_SKYR_HEIGHT", UpgradeSkyrHeight)
 
 	BuildType |= Build2k
 

build/params_butterfly.go

@@ -4,19 +4,25 @@
 package build
 
 import (
+	"github.com/ipfs/go-cid"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/network"
-	"github.com/filecoin-project/lotus/chain/actors/policy"
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
-	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
 )
 
 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 	0: DrandMainnet,
 }
 
-const GenesisNetworkVersion = network.Version14
+const GenesisNetworkVersion = network.Version15
+
+var NetworkBundle = "butterflynet"
+var BundleOverrides map[actors.Version]string
 
 const BootstrappersFile = "butterflynet.pi"
 const GenesisFile = "butterflynet.car"
@@ -42,7 +48,9 @@ const UpgradeTurboHeight = -15
 const UpgradeHyperdriveHeight = -16
 const UpgradeChocolateHeight = -17
 
-const UpgradeOhSnapHeight = 240
+const UpgradeOhSnapHeight = -18
+
+const UpgradeSkyrHeight = abi.ChainEpoch(50)
 
 var SupportedProofTypes = []abi.RegisteredSealProof{
 	abi.RegisteredSealProof_StackedDrg512MiBV1,

build/params_calibnet.go

@@ -4,12 +4,15 @@
 package build
 
 import (
+	"github.com/ipfs/go-cid"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/network"
-	"github.com/filecoin-project/lotus/chain/actors/policy"
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
-	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
 )
 
 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
@@ -18,6 +21,9 @@ var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 
 const GenesisNetworkVersion = network.Version0
 
+var NetworkBundle = "calibrationnet"
+var BundleOverrides map[actors.Version]string
+
 const BootstrappersFile = "calibnet.pi"
 const GenesisFile = "calibnet.car"
 
@@ -57,6 +63,9 @@ const UpgradeChocolateHeight = 312746
 // 2022-02-10T19:23:00Z
 const UpgradeOhSnapHeight = 682006
 
+// 2022-06-16T17:30:00Z
+const UpgradeSkyrHeight = 1044660
+
 var SupportedProofTypes = []abi.RegisteredSealProof{
 	abi.RegisteredSealProof_StackedDrg32GiBV1,
 	abi.RegisteredSealProof_StackedDrg64GiBV1,

build/params_interop.go

@@ -12,14 +12,19 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/network"
-	"github.com/filecoin-project/lotus/chain/actors/policy"
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
+
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
 )
 
+var NetworkBundle = "caterpillarnet"
+var BundleOverrides map[actors.Version]string
+
 const BootstrappersFile = "interopnet.pi"
 const GenesisFile = "interopnet.car"
 
-const GenesisNetworkVersion = network.Version13
+const GenesisNetworkVersion = network.Version15
 
 var UpgradeBreezeHeight = abi.ChainEpoch(-1)
 
@@ -48,6 +53,7 @@ var UpgradeTurboHeight = abi.ChainEpoch(-15)
 var UpgradeHyperdriveHeight = abi.ChainEpoch(-16)
 var UpgradeChocolateHeight = abi.ChainEpoch(-17)
 var UpgradeOhSnapHeight = abi.ChainEpoch(-18)
+var UpgradeSkyrHeight = abi.ChainEpoch(100)
 
 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 	0: DrandMainnet,
@@ -98,6 +104,9 @@ func init() {
 	UpgradeNorwegianHeight = getUpgradeHeight("LOTUS_NORWEGIAN_HEIGHT", UpgradeNorwegianHeight)
 	UpgradeTurboHeight = getUpgradeHeight("LOTUS_ACTORSV4_HEIGHT", UpgradeTurboHeight)
 	UpgradeHyperdriveHeight = getUpgradeHeight("LOTUS_HYPERDRIVE_HEIGHT", UpgradeHyperdriveHeight)
+	UpgradeChocolateHeight = getUpgradeHeight("LOTUS_CHOCOLATE_HEIGHT", UpgradeChocolateHeight)
+	UpgradeOhSnapHeight = getUpgradeHeight("LOTUS_OHSNAP_HEIGHT", UpgradeOhSnapHeight)
+	UpgradeSkyrHeight = getUpgradeHeight("LOTUS_SKYR_HEIGHT", UpgradeSkyrHeight)
 
 	BuildType |= BuildInteropnet
 	SetAddressNetwork(address.Testnet)

build/params_mainnet.go

@@ -1,5 +1,5 @@
-//go:build !debug && !2k && !testground && !calibnet && !nerpanet && !butterflynet && !interopnet
-// +build !debug,!2k,!testground,!calibnet,!nerpanet,!butterflynet,!interopnet
+//go:build !debug && !2k && !testground && !calibnet && !butterflynet && !interopnet
+// +build !debug,!2k,!testground,!calibnet,!butterflynet,!interopnet
 
 package build
 
@@ -7,11 +7,12 @@ import (
 	"math"
 	"os"
 
-	"github.com/filecoin-project/go-state-types/network"
-
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/network"
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
+
+	"github.com/filecoin-project/lotus/chain/actors"
 )
 
 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
@@ -19,6 +20,11 @@ var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 	UpgradeSmokeHeight: DrandMainnet,
 }
 
+var NetworkBundle = "mainnet"
+
+// NOTE: DO NOT change this unless you REALLY know what you're doing. This is consensus critical.
+var BundleOverrides map[actors.Version]string
+
 const GenesisNetworkVersion = network.Version0
 
 const BootstrappersFile = "mainnet.pi"
@@ -68,7 +74,10 @@ const UpgradeHyperdriveHeight = 892800
 const UpgradeChocolateHeight = 1231620
 
 // 2022-03-01T15:00:00Z
-var UpgradeOhSnapHeight = abi.ChainEpoch(1594680)
+const UpgradeOhSnapHeight = 1594680
+
+// 2022-07-06T14:00:00Z
+var UpgradeSkyrHeight = abi.ChainEpoch(1960320)
 
 var SupportedProofTypes = []abi.RegisteredSealProof{
 	abi.RegisteredSealProof_StackedDrg32GiBV1,
@@ -83,8 +92,8 @@ func init() {
 		SetAddressNetwork(address.Mainnet)
 	}
 
-	if os.Getenv("LOTUS_DISABLE_SNAPDEALS") == "1" {
-		UpgradeOhSnapHeight = math.MaxInt64
+	if os.Getenv("LOTUS_DISABLE_SKYR") == "1" {
+		UpgradeSkyrHeight = math.MaxInt64
 	}
 
 	Devnet = false

build/params_nerpanet.go

@@ -1,93 +0,0 @@
-//go:build nerpanet
-// +build nerpanet
-
-package build
-
-import (
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/network"
-	"github.com/filecoin-project/lotus/chain/actors/policy"
-	"github.com/ipfs/go-cid"
-
-	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
-)
-
-var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
-	0: DrandMainnet,
-}
-
-const GenesisNetworkVersion = network.Version0
-
-const BootstrappersFile = "nerpanet.pi"
-const GenesisFile = "nerpanet.car"
-
-const UpgradeBreezeHeight = -1
-const BreezeGasTampingDuration = 0
-
-const UpgradeSmokeHeight = -1
-
-const UpgradeIgnitionHeight = -2
-const UpgradeRefuelHeight = -3
-
-const UpgradeLiftoffHeight = -5
-
-const UpgradeAssemblyHeight = 30 // critical: the network can bootstrap from v1 only
-const UpgradeTapeHeight = 60
-
-const UpgradeKumquatHeight = 90
-
-const UpgradeCalicoHeight = 100
-const UpgradePersianHeight = UpgradeCalicoHeight + (builtin2.EpochsInHour * 1)
-
-const UpgradeClausHeight = 250
-
-const UpgradeOrangeHeight = 300
-
-const UpgradeTrustHeight = 600
-const UpgradeNorwegianHeight = 201000
-const UpgradeTurboHeight = 203000
-const UpgradeHyperdriveHeight = 379178
-
-const UpgradeChocolateHeight = 999999999
-
-var SupportedProofTypes = []abi.RegisteredSealProof{
-	abi.RegisteredSealProof_StackedDrg512MiBV1,
-	abi.RegisteredSealProof_StackedDrg32GiBV1,
-	abi.RegisteredSealProof_StackedDrg64GiBV1,
-}
-
-// Minimum block production power is set to 4 TiB
-// Rationale is to discourage small-scale miners from trying to take over the network
-// One needs to invest in ~2.3x the compute to break consensus, making it not worth it
-//
-// DOWNSIDE: the fake-seals need to be kept alive/protected, otherwise network will seize
-//
-var ConsensusMinerMinPower = abi.NewStoragePower(4 << 40)
-var MinVerifiedDealSize = abi.NewStoragePower(1 << 20)
-
-// Lower the most time-consuming parts of PoRep
-var PreCommitChallengeDelay = abi.ChainEpoch(10)
-
-func init() {
-	policy.SetSupportedProofTypes(SupportedProofTypes...)
-	policy.SetConsensusMinerMinPower(ConsensusMinerMinPower)
-	policy.SetMinVerifiedDealSize(MinVerifiedDealSize)
-	policy.SetPreCommitChallengeDelay(PreCommitChallengeDelay)
-
-	// TODO - make this a variable
-	//miner.WPoStChallengeLookback = abi.ChainEpoch(2)
-
-	Devnet = false
-
-	BuildType = BuildNerpanet
-
-}
-
-const BlockDelaySecs = uint64(builtin2.EpochDurationSeconds)
-
-const PropagationDelaySecs = uint64(6)
-
-// BootstrapPeerThreshold is the minimum number peers we need to track for a sync worker to start
-const BootstrapPeerThreshold = 4
-
-var WhitelistedBlock = cid.Undef

build/params_shared_funcs.go

@@ -1,11 +1,11 @@
 package build
 
 import (
-	"github.com/filecoin-project/go-address"
 	"github.com/ipfs/go-cid"
-
 	"github.com/libp2p/go-libp2p-core/protocol"
 
+	"github.com/filecoin-project/go-address"
+
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 )
 

build/params_shared_vals.go

@@ -10,7 +10,6 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/network"
-
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
 
 	"github.com/filecoin-project/lotus/chain/actors/policy"
@@ -34,7 +33,7 @@ const NewestNetworkVersion = network.Version{{.latestNetworkVersion}}
 
 /* inline-gen start */
 
-const NewestNetworkVersion = network.Version15
+const NewestNetworkVersion = network.Version16
 
 /* inline-gen end */
 

build/params_testground.go

@@ -11,12 +11,13 @@ package build
 import (
 	"math/big"
 
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/network"
 	"github.com/ipfs/go-cid"
 
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/network"
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
 
+	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
 )
 
@@ -106,12 +107,15 @@ var (
 	UpgradeHyperdriveHeight abi.ChainEpoch = -15
 	UpgradeChocolateHeight  abi.ChainEpoch = -16
 	UpgradeOhSnapHeight     abi.ChainEpoch = -17
+	UpgradeSkyrHeight       abi.ChainEpoch = -18
 
 	DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 		0: DrandMainnet,
 	}
 
 	GenesisNetworkVersion = network.Version0
+	NetworkBundle         = "devnet"
+	BundleOverrides       map[actors.Version]string
 
 	NewestNetworkVersion       = network.Version15
 	ActorUpgradeNetworkVersion = network.Version15

build/version.go

@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }
 
 // BuildVersion is the local build version
-const BuildVersion = "1.15.3"
+const BuildVersion = "1.17.1"
 
 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

chain/actors/actor_cids.go

@@ -0,0 +1,328 @@
+package actors
+
+import (
+	"github.com/ipfs/go-cid"
+	"golang.org/x/xerrors"
+
+	builtin0 "github.com/filecoin-project/specs-actors/actors/builtin"
+	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
+	builtin3 "github.com/filecoin-project/specs-actors/v3/actors/builtin"
+	builtin4 "github.com/filecoin-project/specs-actors/v4/actors/builtin"
+	builtin5 "github.com/filecoin-project/specs-actors/v5/actors/builtin"
+	builtin6 "github.com/filecoin-project/specs-actors/v6/actors/builtin"
+	builtin7 "github.com/filecoin-project/specs-actors/v7/actors/builtin"
+)
+
+// GetActorCodeID looks up a builtin actor's code CID by actor version and canonical actor name.
+func GetActorCodeID(av Version, name string) (cid.Cid, bool) {
+
+	// Actors V8 and above
+	if av >= Version8 {
+		if cids, ok := GetActorCodeIDsFromManifest(av); ok {
+			c, ok := cids[name]
+			return c, ok
+		}
+	}
+
+	// Actors V7 and lower
+	switch name {
+
+	case AccountKey:
+		switch av {
+
+		case Version0:
+			return builtin0.AccountActorCodeID, true
+
+		case Version2:
+			return builtin2.AccountActorCodeID, true
+
+		case Version3:
+			return builtin3.AccountActorCodeID, true
+
+		case Version4:
+			return builtin4.AccountActorCodeID, true
+
+		case Version5:
+			return builtin5.AccountActorCodeID, true
+
+		case Version6:
+			return builtin6.AccountActorCodeID, true
+
+		case Version7:
+			return builtin7.AccountActorCodeID, true
+		}
+
+	case CronKey:
+		switch av {
+
+		case Version0:
+			return builtin0.CronActorCodeID, true
+
+		case Version2:
+			return builtin2.CronActorCodeID, true
+
+		case Version3:
+			return builtin3.CronActorCodeID, true
+
+		case Version4:
+			return builtin4.CronActorCodeID, true
+
+		case Version5:
+			return builtin5.CronActorCodeID, true
+
+		case Version6:
+			return builtin6.CronActorCodeID, true
+
+		case Version7:
+			return builtin7.CronActorCodeID, true
+		}
+
+	case InitKey:
+		switch av {
+
+		case Version0:
+			return builtin0.InitActorCodeID, true
+
+		case Version2:
+			return builtin2.InitActorCodeID, true
+
+		case Version3:
+			return builtin3.InitActorCodeID, true
+
+		case Version4:
+			return builtin4.InitActorCodeID, true
+
+		case Version5:
+			return builtin5.InitActorCodeID, true
+
+		case Version6:
+			return builtin6.InitActorCodeID, true
+
+		case Version7:
+			return builtin7.InitActorCodeID, true
+		}
+
+	case MarketKey:
+		switch av {
+
+		case Version0:
+			return builtin0.StorageMarketActorCodeID, true
+
+		case Version2:
+			return builtin2.StorageMarketActorCodeID, true
+
+		case Version3:
+			return builtin3.StorageMarketActorCodeID, true
+
+		case Version4:
+			return builtin4.StorageMarketActorCodeID, true
+
+		case Version5:
+			return builtin5.StorageMarketActorCodeID, true
+
+		case Version6:
+			return builtin6.StorageMarketActorCodeID, true
+
+		case Version7:
+			return builtin7.StorageMarketActorCodeID, true
+		}
+
+	case MinerKey:
+		switch av {
+
+		case Version0:
+			return builtin0.StorageMinerActorCodeID, true
+
+		case Version2:
+			return builtin2.StorageMinerActorCodeID, true
+
+		case Version3:
+			return builtin3.StorageMinerActorCodeID, true
+
+		case Version4:
+			return builtin4.StorageMinerActorCodeID, true
+
+		case Version5:
+			return builtin5.StorageMinerActorCodeID, true
+
+		case Version6:
+			return builtin6.StorageMinerActorCodeID, true
+
+		case Version7:
+			return builtin7.StorageMinerActorCodeID, true
+		}
+
+	case MultisigKey:
+		switch av {
+
+		case Version0:
+			return builtin0.MultisigActorCodeID, true
+
+		case Version2:
+			return builtin2.MultisigActorCodeID, true
+
+		case Version3:
+			return builtin3.MultisigActorCodeID, true
+
+		case Version4:
+			return builtin4.MultisigActorCodeID, true
+
+		case Version5:
+			return builtin5.MultisigActorCodeID, true
+
+		case Version6:
+			return builtin6.MultisigActorCodeID, true
+
+		case Version7:
+			return builtin7.MultisigActorCodeID, true
+		}
+
+	case PaychKey:
+		switch av {
+
+		case Version0:
+			return builtin0.PaymentChannelActorCodeID, true
+
+		case Version2:
+			return builtin2.PaymentChannelActorCodeID, true
+
+		case Version3:
+			return builtin3.PaymentChannelActorCodeID, true
+
+		case Version4:
+			return builtin4.PaymentChannelActorCodeID, true
+
+		case Version5:
+			return builtin5.PaymentChannelActorCodeID, true
+
+		case Version6:
+			return builtin6.PaymentChannelActorCodeID, true
+
+		case Version7:
+			return builtin7.PaymentChannelActorCodeID, true
+		}
+
+	case PowerKey:
+		switch av {
+
+		case Version0:
+			return builtin0.StoragePowerActorCodeID, true
+
+		case Version2:
+			return builtin2.StoragePowerActorCodeID, true
+
+		case Version3:
+			return builtin3.StoragePowerActorCodeID, true
+
+		case Version4:
+			return builtin4.StoragePowerActorCodeID, true
+
+		case Version5:
+			return builtin5.StoragePowerActorCodeID, true
+
+		case Version6:
+			return builtin6.StoragePowerActorCodeID, true
+
+		case Version7:
+			return builtin7.StoragePowerActorCodeID, true
+		}
+
+	case RewardKey:
+		switch av {
+
+		case Version0:
+			return builtin0.RewardActorCodeID, true
+
+		case Version2:
+			return builtin2.RewardActorCodeID, true
+
+		case Version3:
+			return builtin3.RewardActorCodeID, true
+
+		case Version4:
+			return builtin4.RewardActorCodeID, true
+
+		case Version5:
+			return builtin5.RewardActorCodeID, true
+
+		case Version6:
+			return builtin6.RewardActorCodeID, true
+
+		case Version7:
+			return builtin7.RewardActorCodeID, true
+		}
+
+	case SystemKey:
+		switch av {
+
+		case Version0:
+			return builtin0.SystemActorCodeID, true
+
+		case Version2:
+			return builtin2.SystemActorCodeID, true
+
+		case Version3:
+			return builtin3.SystemActorCodeID, true
+
+		case Version4:
+			return builtin4.SystemActorCodeID, true
+
+		case Version5:
+			return builtin5.SystemActorCodeID, true
+
+		case Version6:
+			return builtin6.SystemActorCodeID, true
+
+		case Version7:
+			return builtin7.SystemActorCodeID, true
+		}
+
+	case VerifregKey:
+		switch av {
+
+		case Version0:
+			return builtin0.VerifiedRegistryActorCodeID, true
+
+		case Version2:
+			return builtin2.VerifiedRegistryActorCodeID, true
+
+		case Version3:
+			return builtin3.VerifiedRegistryActorCodeID, true
+
+		case Version4:
+			return builtin4.VerifiedRegistryActorCodeID, true
+
+		case Version5:
+			return builtin5.VerifiedRegistryActorCodeID, true
+
+		case Version6:
+			return builtin6.VerifiedRegistryActorCodeID, true
+
+		case Version7:
+			return builtin7.VerifiedRegistryActorCodeID, true
+		}
+	}
+
+	return cid.Undef, false
+}
+
+// GetActorCodeIDs looks up all builtin actor's code CIDs by actor version.
+func GetActorCodeIDs(av Version) (map[string]cid.Cid, error) {
+	cids, ok := GetActorCodeIDsFromManifest(av)
+	if ok {
+		return cids, nil
+	}
+
+	actorsKeys := GetBuiltinActorsKeys()
+	synthCids := make(map[string]cid.Cid)
+
+	for _, key := range actorsKeys {
+		c, ok := GetActorCodeID(av, key)
+		if !ok {
+			return nil, xerrors.Errorf("could not find builtin actor cids for Actors version %d", av)
+		}
+		synthCids[key] = c
+	}
+
+	return synthCids, nil
+}

chain/actors/adt/diff_adt.go

@@ -3,8 +3,9 @@ package adt
 import (
 	"bytes"
 
-	"github.com/filecoin-project/go-state-types/abi"
 	typegen "github.com/whyrusleeping/cbor-gen"
+
+	"github.com/filecoin-project/go-state-types/abi"
 )
 
 // AdtArrayDiff generalizes adt.Array diffing by accepting a Deferred type that can unmarshalled to its corresponding struct

chain/actors/adt/diff_adt_test.go

@@ -6,14 +6,12 @@ import (
 	"context"
 	"testing"
 
+	cbornode "github.com/ipfs/go-ipld-cbor"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-
-	cbornode "github.com/ipfs/go-ipld-cbor"
 	typegen "github.com/whyrusleeping/cbor-gen"
 
 	"github.com/filecoin-project/go-state-types/abi"
-
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
 	adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt"
 

chain/actors/adt/store.go

@@ -3,8 +3,9 @@ package adt
 import (
 	"context"
 
-	adt "github.com/filecoin-project/specs-actors/actors/util/adt"
 	cbor "github.com/ipfs/go-ipld-cbor"
+
+	"github.com/filecoin-project/specs-actors/actors/util/adt"
 )
 
 type Store interface {

chain/actors/aerrors/error.go

@@ -3,8 +3,9 @@ package aerrors
 import (
 	"fmt"
 
-	"github.com/filecoin-project/go-state-types/exitcode"
 	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-state-types/exitcode"
 )
 
 func IsFatal(err ActorError) bool {

chain/actors/aerrors/error_test.go

@@ -4,11 +4,12 @@ package aerrors_test
 import (
 	"testing"
 
-	"github.com/filecoin-project/go-state-types/exitcode"
-	. "github.com/filecoin-project/lotus/chain/actors/aerrors"
-
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-state-types/exitcode"
+
+	. "github.com/filecoin-project/lotus/chain/actors/aerrors"
 )
 
 func TestFatalError(t *testing.T) {

chain/actors/aerrors/wrap.go

@@ -4,9 +4,10 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/filecoin-project/go-state-types/exitcode"
 	cbor "github.com/ipfs/go-ipld-cbor"
 	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-state-types/exitcode"
 )
 
 // New creates a new non-fatal error

chain/actors/agen/main.go

@@ -3,15 +3,16 @@ package main
 import (
 	"bytes"
 	"fmt"
+	"go/format"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strconv"
 	"text/template"
 
-	lotusactors "github.com/filecoin-project/lotus/chain/actors"
-
 	"golang.org/x/xerrors"
+
+	lotusactors "github.com/filecoin-project/lotus/chain/actors"
 )
 
 var actors = map[string][]int{
@@ -77,7 +78,12 @@ func generateAdapters() error {
 				return err
 			}
 
-			if err := ioutil.WriteFile(filepath.Join(actDir, fmt.Sprintf("%s.go", act)), b.Bytes(), 0666); err != nil {
+			fmted, err := format.Source(b.Bytes())
+			if err != nil {
+				return err
+			}
+
+			if err := ioutil.WriteFile(filepath.Join(actDir, fmt.Sprintf("%s.go", act)), fmted, 0666); err != nil {
 				return err
 			}
 		}

chain/actors/builtin/account/account.go

@@ -1,66 +1,40 @@
 package account
 
 import (
-	"github.com/filecoin-project/lotus/chain/actors"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
+	builtin8 "github.com/filecoin-project/go-state-types/builtin"
 	"github.com/filecoin-project/go-state-types/cbor"
-	"github.com/ipfs/go-cid"
-
-	"github.com/filecoin-project/lotus/chain/actors/adt"
-	"github.com/filecoin-project/lotus/chain/actors/builtin"
-	"github.com/filecoin-project/lotus/chain/types"
-
 	builtin0 "github.com/filecoin-project/specs-actors/actors/builtin"
-
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
-
 	builtin3 "github.com/filecoin-project/specs-actors/v3/actors/builtin"
-
 	builtin4 "github.com/filecoin-project/specs-actors/v4/actors/builtin"
-
 	builtin5 "github.com/filecoin-project/specs-actors/v5/actors/builtin"
-
 	builtin6 "github.com/filecoin-project/specs-actors/v6/actors/builtin"
-
 	builtin7 "github.com/filecoin-project/specs-actors/v7/actors/builtin"
+
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/adt"
+	"github.com/filecoin-project/lotus/chain/types"
 )
 
-func init() {
-
-	builtin.RegisterActorState(builtin0.AccountActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) {
-		return load0(store, root)
-	})
-
-	builtin.RegisterActorState(builtin2.AccountActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) {
-		return load2(store, root)
-	})
-
-	builtin.RegisterActorState(builtin3.AccountActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) {
-		return load3(store, root)
-	})
-
-	builtin.RegisterActorState(builtin4.AccountActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) {
-		return load4(store, root)
-	})
-
-	builtin.RegisterActorState(builtin5.AccountActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) {
-		return load5(store, root)
-	})
-
-	builtin.RegisterActorState(builtin6.AccountActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) {
-		return load6(store, root)
-	})
-
-	builtin.RegisterActorState(builtin7.AccountActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) {
-		return load7(store, root)
-	})
-}
-
-var Methods = builtin4.MethodsAccount
+var Methods = builtin8.MethodsAccount
 
 func Load(store adt.Store, act *types.Actor) (State, error) {
+	if name, av, ok := actors.GetActorMetaByCode(act.Code); ok {
+		if name != actors.AccountKey {
+			return nil, xerrors.Errorf("actor code is not account: %s", name)
+		}
+
+		switch av {
+
+		case actors.Version8:
+			return load8(store, act.Head)
+
+		}
+	}
+
 	switch act.Code {
 
 	case builtin0.AccountActorCodeID:
@@ -85,6 +59,7 @@ func Load(store adt.Store, act *types.Actor) (State, error) {
 		return load7(store, act.Head)
 
 	}
+
 	return nil, xerrors.Errorf("unknown actor code %s", act.Code)
 }
 
@@ -112,39 +87,13 @@ func MakeState(store adt.Store, av actors.Version, addr address.Address) (State,
 	case actors.Version7:
 		return make7(store, addr)
 
+	case actors.Version8:
+		return make8(store, addr)
+
 	}
 	return nil, xerrors.Errorf("unknown actor version %d", av)
 }
 
-func GetActorCodeID(av actors.Version) (cid.Cid, error) {
-	switch av {
-
-	case actors.Version0:
-		return builtin0.AccountActorCodeID, nil
-
-	case actors.Version2:
-		return builtin2.AccountActorCodeID, nil
-
-	case actors.Version3:
-		return builtin3.AccountActorCodeID, nil
-
-	case actors.Version4:
-		return builtin4.AccountActorCodeID, nil
-
-	case actors.Version5:
-		return builtin5.AccountActorCodeID, nil
-
-	case actors.Version6:
-		return builtin6.AccountActorCodeID, nil
-
-	case actors.Version7:
-		return builtin7.AccountActorCodeID, nil
-
-	}
-
-	return cid.Undef, xerrors.Errorf("unknown actor version %d", av)
-}
-
 type State interface {
 	cbor.Marshaler