Merge pull request #9432 from filecoin-project/release/v1.17.2

release: v1.17.2

.circleci/config.yml

@@ -5,7 +5,7 @@ orbs:
 executors:
   golang:
     docker:
-      - image: cimg/go:1.17.9
+      - image: cimg/go:1.18.1
     resource_class: 2xlarge
   ubuntu:
     docker:
@@ -46,6 +46,35 @@ commands:
           steps:
             - run: sudo apt-get update
             - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
+      - when:
+          condition: <<parameters.darwin>>
+          steps:
+            - run:
+                name: Install Go
+                command: |
+                  curl https://dl.google.com/go/go1.18.1.darwin-amd64.pkg -o /tmp/go.pkg && \
+                  sudo installer -pkg /tmp/go.pkg -target /
+            - run:
+                name: Export Go
+                command: |
+                  echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
+            - run: go version
+            - run:
+                name: Install pkg-config, goreleaser, and sha512sum
+                command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils
+            - run:
+                name: Install Rust
+                command: |
+                  curl https://sh.rustup.rs -sSf | sh -s -- -y
+            - run:
+                name: Install hwloc
+                command: |
+                  mkdir ~/hwloc
+                  curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
+                  cd ~/hwloc
+                  tar -xvzpf hwloc-2.4.1.tar.gz
+                  cd hwloc-2.4.1
+                  ./configure && make && sudo make install
       - run: git submodule sync
       - run: git submodule update --init
   download-params:
@@ -77,6 +106,16 @@ commands:
           tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz
           mv go-ipfs/ipfs /usr/local/bin/ipfs
           chmod +x /usr/local/bin/ipfs
+  install_ipfs_macos:
+    steps:
+      - run: |
+          curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz
+          tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz
+          pushd kubo
+          sudo bash install.sh
+          popd
+          rm -rf kubo/
+          rm kubo_v0.14.0_darwin-amd64.tar.gz
   git_fetch_all_tags:
     steps:
       - run:
@@ -115,7 +154,7 @@ jobs:
           command: |
             git --no-pager diff go.mod go.sum
             git --no-pager diff --quiet go.mod go.sum
-  build-all:
+  build-linux:
     executor: golang
     steps:
       - install-deps
@@ -173,16 +212,6 @@ jobs:
         type: string
         default: standard-verbose
         description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
-      coverage:
-        type: string
-        default: -coverprofile=coverage.txt -coverpkg=github.com/filecoin-project/lotus/...
-        description: Coverage flag. Set to the empty string to disable.
-      codecov-upload:
-        type: boolean
-        default: true
-        description: |
-          Upload coverage report to https://codecov.io/. Requires the codecov API token to be
-          set as an environment variable for private projects.
     executor: << parameters.executor >>
     steps:
       - install-deps
@@ -205,7 +234,6 @@ jobs:
               --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
               --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \
               -- \
-              << parameters.coverage >> \
               << parameters.go-test-flags >> \
               << parameters.target >>
           no_output_timeout: 30m
@@ -213,13 +241,6 @@ jobs:
           path: /tmp/test-reports
       - store_artifacts:
           path: /tmp/test-artifacts/<< parameters.suite >>.json
-      - when:
-          condition: << parameters.codecov-upload >>
-          steps:
-            - run:
-                shell: /bin/bash -eo pipefail
-                command: |
-                  bash <(curl -s https://codecov.io/bash)
 
   test-conformance:
     description: |
@@ -353,58 +374,42 @@ jobs:
       - run:
           name: "trigger payment channel stress testplan on taas"
           command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-
   build-macos:
     description: build darwin lotus binary
+    parameters:
+      publish:
+        default: false
+        description: publish github release and homebrew?
+        type: boolean
     macos:
-      xcode: "12.5.0"
+      xcode: "13.4.1"
     working_directory: ~/go/src/github.com/filecoin-project/lotus
     steps:
       - prepare:
           linux: false
           darwin: true
-      - run:
-          name: Install go
-          command: |
-            curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \
-            sudo installer -pkg go1.17.9.darwin-amd64.pkg -target /
-      - run:
-          name: Install pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
-      - run: go version
-      - run:
-          name: Install Rust
-          command: |
-            curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - run:
-          name: Install hwloc
-          command: |
-            mkdir ~/hwloc
-            curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-            cd ~/hwloc
-            tar -xvzpf hwloc-2.4.1.tar.gz
-            cd hwloc-2.4.1
-            ./configure && make && sudo make install
+      - install_ipfs_macos
       - restore_cache:
           name: restore cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-      - run:
-          command: make build
-          no_output_timeout: 30m
-      - run:
-          name: check tag and version output match
-          command: ./scripts/version-check.sh ./lotus
+      - when:
+          condition: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist
+            - run: ./scripts/generate-checksums.sh
+            - run: ./scripts/publish-checksums.sh
+      - when:
+          condition:
+            not: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist --snapshot
+            - run: ./scripts/generate-checksums.sh
       - store_artifacts:
-          path: lotus
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
+          path: dist
       - persist_to_workspace:
           root: "."
           paths:
-            - darwin
+            - dist
       - save_cache:
           name: save cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
@@ -419,6 +424,12 @@ jobs:
       - checkout
       - attach_workspace:
           at: "."
+      - run:
+          name: Update Go
+          command: |
+            curl -L https://golang.org/dl/go1.18.1.linux-amd64.tar.gz -o /tmp/go.tar.gz && \
+            sudo tar -C /usr/local -xvf /tmp/go.tar.gz
+      - run: go version
       - run:
           name: install appimage-builder
           command: |
@@ -536,6 +547,15 @@ jobs:
   publish:
     description: publish binary artifacts
     executor: ubuntu
+    parameters:
+      linux:
+        default: false
+        description: publish linux binaries?
+        type: boolean
+      appimage:
+        default: false
+        description: publish appimage binaries?
+        type: boolean
     steps:
       - run:
           name: Install git jq curl
@@ -546,12 +566,16 @@ jobs:
       - install_ipfs
       - attach_workspace:
           at: "."
-      - run:
-          name: Create bundles
-          command: ./scripts/build-bundle.sh
-      - run:
-          name: Publish release
-          command: ./scripts/publish-release.sh
+      - when:
+          condition: << parameters.linux >>
+          steps:
+            - run: ./scripts/build-arch-bundle.sh linux
+            - run: ./scripts/publish-arch-release.sh linux
+      - when:
+          condition: << parameters.appimage >>
+          steps:
+            - run: ./scripts/build-appimage-bundle.sh
+            - run: ./scripts/publish-arch-release.sh appimage
 
   publish-snapcraft:
     description: build and push snapcraft
@@ -563,22 +587,29 @@ jobs:
         type: string
         default: "edge"
         description: snapcraft channel
+      snap-name:
+        type: string
+        default: 'lotus-filecoin'
+        description: name of snap in snap store
     steps:
       - checkout
       - run:
-          name: install snapcraft
+          name: Install snapcraft
           command: sudo snap install snapcraft --classic
       - run:
-          name: create snapcraft config file
+          name: Build << parameters.snap-name >> snap
           command: |
-            mkdir -p ~/.config/snapcraft
-            echo "$SNAPCRAFT_LOGIN_FILE" | base64 -d > ~/.config/snapcraft/snapcraft.cfg
+            if [ "<< parameters.snap-name >>" != 'lotus-filecoin' ]; then
+              cat snap/snapcraft.yaml | sed 's/lotus-filecoin/lotus/' > edited-snapcraft.yaml
+              mv edited-snapcraft.yaml snap/snapcraft.yaml
+            fi
+
+            snapcraft --use-lxd --debug
       - run:
-          name: build snap
-          command: snapcraft --use-lxd
-      - run:
-          name: publish snap
-          command: snapcraft push *.snap --release << parameters.channel >>
+          name: Publish snap to << parameters.channel >> channel
+          shell: /bin/bash -o pipefail
+          command: |
+            snapcraft upload *.snap --release << parameters.channel >>
 
   build-and-push-image:
     description: build and push docker images to public AWS ECR registry
@@ -730,26 +761,32 @@ jobs:
           name: docker build
           command: |
             docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus .
+            docker build --target lotus-gateway -t filecoin/lotus-gateway:<< parameters.tag >> -f Dockerfile.lotus .
             docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus .
             if [[ ! -z $CIRCLE_SHA1 ]]; then
               docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus .
+              docker build --target lotus-gateway -t filecoin/lotus-gateway:$CIRCLE_SHA1 -f Dockerfile.lotus .
               docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus .
             fi
             if [[ ! -z $CIRCLE_TAG ]]; then
               docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus .
+              docker build --target lotus-gateway -t filecoin/lotus-gateway:$CIRCLE_TAG -f Dockerfile.lotus .
               docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus .
             fi
       - run:
           name: docker push
           command: |
             docker push filecoin/lotus:<< parameters.tag >>
+            docker push filecoin/lotus-gateway:<< parameters.tag >>
             docker push filecoin/lotus-all-in-one:<< parameters.tag >>
             if [[ ! -z $CIRCLE_SHA1 ]]; then
               docker push filecoin/lotus:$CIRCLE_SHA1
+              docker push filecoin/lotus-gateway:$CIRCLE_SHA1
               docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1
             fi
             if [[ ! -z $CIRCLE_TAG ]]; then
               docker push filecoin/lotus:$CIRCLE_TAG
+              docker push filecoin/lotus-gateway:$CIRCLE_TAG
               docker push filecoin/lotus-all-in-one:$CIRCLE_TAG
             fi
 
@@ -858,6 +895,16 @@ workflows:
           suite: itest-deals
           target: "./itests/deals_test.go"
       
+      - test:
+          name: test-itest-dup_mpool_messages
+          suite: itest-dup_mpool_messages
+          target: "./itests/dup_mpool_messages_test.go"
+      
+      - test:
+          name: test-itest-gas_estimation
+          suite: itest-gas_estimation
+          target: "./itests/gas_estimation_test.go"
+      
       - test:
           name: test-itest-gateway
           suite: itest-gateway
@@ -883,16 +930,36 @@ workflows:
           suite: itest-mempool
           target: "./itests/mempool_test.go"
       
+      - test:
+          name: test-itest-mpool_msg_uuid
+          suite: itest-mpool_msg_uuid
+          target: "./itests/mpool_msg_uuid_test.go"
+      
       - test:
           name: test-itest-multisig
           suite: itest-multisig
           target: "./itests/multisig_test.go"
       
+      - test:
+          name: test-itest-net
+          suite: itest-net
+          target: "./itests/net_test.go"
+      
       - test:
           name: test-itest-nonce
           suite: itest-nonce
           target: "./itests/nonce_test.go"
       
+      - test:
+          name: test-itest-path_detach_redeclare
+          suite: itest-path_detach_redeclare
+          target: "./itests/path_detach_redeclare_test.go"
+      
+      - test:
+          name: test-itest-path_type_filters
+          suite: itest-path_type_filters
+          target: "./itests/path_type_filters_test.go"
+      
       - test:
           name: test-itest-paych_api
           suite: itest-paych_api
@@ -913,6 +980,16 @@ workflows:
           suite: itest-sector_finalize_early
           target: "./itests/sector_finalize_early_test.go"
       
+      - test:
+          name: test-itest-sector_import_full
+          suite: itest-sector_import_full
+          target: "./itests/sector_import_full_test.go"
+      
+      - test:
+          name: test-itest-sector_import_simple
+          suite: itest-sector_import_simple
+          target: "./itests/sector_import_simple_test.go"
+      
       - test:
           name: test-itest-sector_make_cc_avail
           suite: itest-sector_make_cc_avail
@@ -923,6 +1000,11 @@ workflows:
           suite: itest-sector_miner_collateral
           target: "./itests/sector_miner_collateral_test.go"
       
+      - test:
+          name: test-itest-sector_numassign
+          suite: itest-sector_numassign
+          target: "./itests/sector_numassign_test.go"
+      
       - test:
           name: test-itest-sector_pledge
           suite: itest-sector_pledge
@@ -948,6 +1030,11 @@ workflows:
           suite: itest-self_sent_txn
           target: "./itests/self_sent_txn_test.go"
       
+      - test:
+          name: test-itest-splitstore
+          suite: itest-splitstore
+          target: "./itests/splitstore_test.go"
+      
       - test:
           name: test-itest-tape
           suite: itest-tape
@@ -958,16 +1045,31 @@ workflows:
           suite: itest-verifreg
           target: "./itests/verifreg_test.go"
       
+      - test:
+          name: test-itest-wdpost_config
+          suite: itest-wdpost_config
+          target: "./itests/wdpost_config_test.go"
+      
       - test:
           name: test-itest-wdpost_dispute
           suite: itest-wdpost_dispute
           target: "./itests/wdpost_dispute_test.go"
       
+      - test:
+          name: test-itest-wdpost_no_miner_storage
+          suite: itest-wdpost_no_miner_storage
+          target: "./itests/wdpost_no_miner_storage_test.go"
+      
       - test:
           name: test-itest-wdpost
           suite: itest-wdpost
           target: "./itests/wdpost_test.go"
       
+      - test:
+          name: test-itest-wdpost_worker_config
+          suite: itest-wdpost_worker_config
+          target: "./itests/wdpost_worker_config_test.go"
+      
       - test:
           name: test-itest-worker
           suite: itest-worker
@@ -992,15 +1094,13 @@ workflows:
       - test:
           go-test-flags: "-run=TestMulticoreSDR"
           suite: multicore-sdr-check
-          target: "./extern/sector-storage/ffiwrapper"
+          target: "./storage/sealer/ffiwrapper"
           proofs-log-test: "1"
       - test-conformance:
           suite: conformance
-          codecov-upload: false
           target: "./conformance"
       - test-conformance:
           name: test-conformance-bleeding-edge
-          codecov-upload: false
           suite: conformance-bleeding-edge
           target: "./conformance"
           vectors-branch: specs-actors-v7
@@ -1010,7 +1110,7 @@ workflows:
               only:
                 - master
       - build-debug
-      - build-all:
+      - build-linux:
           filters:
             tags:
               only:
@@ -1027,11 +1127,36 @@ workflows:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-lotus-soup
       - build-macos:
+          name: publish-macos
+          publish: true
           filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+$/
+      - build-macos:
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+-rc\d+$/
+      - build-appimage:
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-appimage:
+      - publish:
+          name: publish-linux
+          linux: true
+          requires:
+            - build-linux
           filters:
             branches:
               ignore:
@@ -1040,9 +1165,9 @@ workflows:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - publish:
+          name: publish-appimage
+          appimage: true
           requires:
-            - build-all
-            - build-macos
             - build-appimage
           filters:
             branches:
@@ -1052,27 +1177,63 @@ workflows:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-and-push-image:
+          name: build-and-push/lotus-all-in-one
           dockerfile: Dockerfile.lotus
           path: .
           repo: lotus-dev
           tag: '${CIRCLE_SHA1:0:8}'
           target: lotus-all-in-one
       - build-and-push-image:
+          name: build-and-push/lotus-test
           dockerfile: Dockerfile.lotus
           path: .
           repo: lotus-test
           tag: '${CIRCLE_SHA1:0:8}'
           target: lotus-test
       - publish-snapcraft:
-          name: publish-snapcraft-stable
+          name: "Publish Snapcraft (lotus-filecoin / candidate)"
           channel: stable
+          snap-name: lotus-filecoin
           filters:
             branches:
               ignore:
                 - /.*/
             tags:
               only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+                - /^v\d+\.\d+\.\d+$/
+      - publish-snapcraft:
+          name: "Publish Snapcraft (lotus-filecoin / candidate)"
+          channel: candidate
+          snap-name: lotus-filecoin
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+-rc\d+$/
+      - publish-snapcraft:
+          name: "Publish Snapcraft (lotus / stable)"
+          channel: stable
+          snap-name: lotus
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+$/
+      - publish-snapcraft:
+          name: "Publish Snapcraft (lotus / candidate)"
+          channel: candidate
+          snap-name: lotus
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+-rc\d+$/
       - publish-dockerhub:
           name: publish-dockerhub
           tag: stable
@@ -1094,8 +1255,13 @@ workflows:
                 - master
     jobs:
       - publish-snapcraft:
-          name: publish-snapcraft-nightly
+          name: "Publish Snapcraft Nightly (lotus-filecoin / edge)"
           channel: edge
+          snap-name: lotus-filecoin
+      - publish-snapcraft:
+          name: "Publish Snapcraft Nightly (lotus / edge)"
+          channel: edge
+          snap-name: lotus
       - publish-dockerhub:
           name: publish-dockerhub-nightly
           tag: nightly

.circleci/template.yml

@@ -5,7 +5,7 @@ orbs:
 executors:
   golang:
     docker:
-      - image: cimg/go:1.17.9
+      - image: cimg/go:1.18.1
     resource_class: 2xlarge
   ubuntu:
     docker:
@@ -46,6 +46,35 @@ commands:
           steps:
             - run: sudo apt-get update
             - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
+      - when:
+          condition: <<parameters.darwin>>
+          steps:
+            - run:
+                name: Install Go
+                command: |
+                  curl https://dl.google.com/go/go1.18.1.darwin-amd64.pkg -o /tmp/go.pkg && \
+                  sudo installer -pkg /tmp/go.pkg -target /
+            - run:
+                name: Export Go
+                command: |
+                  echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
+            - run: go version
+            - run:
+                name: Install pkg-config, goreleaser, and sha512sum
+                command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils
+            - run:
+                name: Install Rust
+                command: |
+                  curl https://sh.rustup.rs -sSf | sh -s -- -y
+            - run:
+                name: Install hwloc
+                command: |
+                  mkdir ~/hwloc
+                  curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
+                  cd ~/hwloc
+                  tar -xvzpf hwloc-2.4.1.tar.gz
+                  cd hwloc-2.4.1
+                  ./configure && make && sudo make install
       - run: git submodule sync
       - run: git submodule update --init
   download-params:
@@ -77,6 +106,16 @@ commands:
           tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz
           mv go-ipfs/ipfs /usr/local/bin/ipfs
           chmod +x /usr/local/bin/ipfs
+  install_ipfs_macos:
+    steps:
+      - run: |
+          curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz
+          tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz
+          pushd kubo
+          sudo bash install.sh
+          popd
+          rm -rf kubo/
+          rm kubo_v0.14.0_darwin-amd64.tar.gz
   git_fetch_all_tags:
     steps:
       - run:
@@ -115,7 +154,7 @@ jobs:
           command: |
             git --no-pager diff go.mod go.sum
             git --no-pager diff --quiet go.mod go.sum
-  build-all:
+  build-linux:
     executor: golang
     steps:
       - install-deps
@@ -173,16 +212,6 @@ jobs:
         type: string
         default: standard-verbose
         description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
-      coverage:
-        type: string
-        default: -coverprofile=coverage.txt -coverpkg=github.com/filecoin-project/lotus/...
-        description: Coverage flag. Set to the empty string to disable.
-      codecov-upload:
-        type: boolean
-        default: true
-        description: |
-          Upload coverage report to https://codecov.io/. Requires the codecov API token to be
-          set as an environment variable for private projects.
     executor: << parameters.executor >>
     steps:
       - install-deps
@@ -205,7 +234,6 @@ jobs:
               --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
               --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \
               -- \
-              << parameters.coverage >> \
               << parameters.go-test-flags >> \
               << parameters.target >>
           no_output_timeout: 30m
@@ -213,13 +241,6 @@ jobs:
           path: /tmp/test-reports
       - store_artifacts:
           path: /tmp/test-artifacts/<< parameters.suite >>.json
-      - when:
-          condition: << parameters.codecov-upload >>
-          steps:
-            - run:
-                shell: /bin/bash -eo pipefail
-                command: |
-                  bash <(curl -s https://codecov.io/bash)
 
   test-conformance:
     description: |
@@ -353,58 +374,42 @@ jobs:
       - run:
           name: "trigger payment channel stress testplan on taas"
           command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-
   build-macos:
     description: build darwin lotus binary
+    parameters:
+      publish:
+        default: false
+        description: publish github release and homebrew?
+        type: boolean
     macos:
-      xcode: "12.5.0"
+      xcode: "13.4.1"
     working_directory: ~/go/src/github.com/filecoin-project/lotus
     steps:
       - prepare:
           linux: false
           darwin: true
-      - run:
-          name: Install go
-          command: |
-            curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \
-            sudo installer -pkg go1.17.9.darwin-amd64.pkg -target /
-      - run:
-          name: Install pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
-      - run: go version
-      - run:
-          name: Install Rust
-          command: |
-            curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - run:
-          name: Install hwloc
-          command: |
-            mkdir ~/hwloc
-            curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-            cd ~/hwloc
-            tar -xvzpf hwloc-2.4.1.tar.gz
-            cd hwloc-2.4.1
-            ./configure && make && sudo make install
+      - install_ipfs_macos
       - restore_cache:
           name: restore cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-      - run:
-          command: make build
-          no_output_timeout: 30m
-      - run:
-          name: check tag and version output match
-          command: ./scripts/version-check.sh ./lotus
+      - when:
+          condition: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist
+            - run: ./scripts/generate-checksums.sh
+            - run: ./scripts/publish-checksums.sh
+      - when:
+          condition:
+            not: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist --snapshot
+            - run: ./scripts/generate-checksums.sh
       - store_artifacts:
-          path: lotus
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
+          path: dist
       - persist_to_workspace:
           root: "."
           paths:
-            - darwin
+            - dist
       - save_cache:
           name: save cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
@@ -419,6 +424,12 @@ jobs:
       - checkout
       - attach_workspace:
           at: "."
+      - run:
+          name: Update Go
+          command: |
+            curl -L https://golang.org/dl/go1.18.1.linux-amd64.tar.gz -o /tmp/go.tar.gz && \
+            sudo tar -C /usr/local -xvf /tmp/go.tar.gz
+      - run: go version
       - run:
           name: install appimage-builder
           command: |
@@ -536,6 +547,15 @@ jobs:
   publish:
     description: publish binary artifacts
     executor: ubuntu
+    parameters:
+      linux:
+        default: false
+        description: publish linux binaries?
+        type: boolean
+      appimage:
+        default: false
+        description: publish appimage binaries?
+        type: boolean
     steps:
       - run:
           name: Install git jq curl
@@ -546,12 +566,16 @@ jobs:
       - install_ipfs
       - attach_workspace:
           at: "."
-      - run:
-          name: Create bundles
-          command: ./scripts/build-bundle.sh
-      - run:
-          name: Publish release
-          command: ./scripts/publish-release.sh
+      - when:
+          condition: << parameters.linux >>
+          steps:
+            - run: ./scripts/build-arch-bundle.sh linux
+            - run: ./scripts/publish-arch-release.sh linux
+      - when:
+          condition: << parameters.appimage >>
+          steps:
+            - run: ./scripts/build-appimage-bundle.sh
+            - run: ./scripts/publish-arch-release.sh appimage
 
   publish-snapcraft:
     description: build and push snapcraft
@@ -563,22 +587,29 @@ jobs:
         type: string
         default: "edge"
         description: snapcraft channel
+      snap-name:
+        type: string
+        default: 'lotus-filecoin'
+        description: name of snap in snap store
     steps:
       - checkout
       - run:
-          name: install snapcraft
+          name: Install snapcraft
           command: sudo snap install snapcraft --classic
       - run:
-          name: create snapcraft config file
+          name: Build << parameters.snap-name >> snap
           command: |
-            mkdir -p ~/.config/snapcraft
-            echo "$SNAPCRAFT_LOGIN_FILE" | base64 -d > ~/.config/snapcraft/snapcraft.cfg
+            if [ "<< parameters.snap-name >>" != 'lotus-filecoin' ]; then
+              cat snap/snapcraft.yaml | sed 's/lotus-filecoin/lotus/' > edited-snapcraft.yaml
+              mv edited-snapcraft.yaml snap/snapcraft.yaml
+            fi
+
+            snapcraft --use-lxd --debug
       - run:
-          name: build snap
-          command: snapcraft --use-lxd
-      - run:
-          name: publish snap
-          command: snapcraft push *.snap --release << parameters.channel >>
+          name: Publish snap to << parameters.channel >> channel
+          shell: /bin/bash -o pipefail
+          command: |
+            snapcraft upload *.snap --release << parameters.channel >>
 
   build-and-push-image:
     description: build and push docker images to public AWS ECR registry
@@ -730,26 +761,32 @@ jobs:
           name: docker build
           command: |
             docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus .
+            docker build --target lotus-gateway -t filecoin/lotus-gateway:<< parameters.tag >> -f Dockerfile.lotus .
             docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus .
             if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then
               docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus .
+              docker build --target lotus-gateway -t filecoin/lotus-gateway:$CIRCLE_SHA1 -f Dockerfile.lotus .
               docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus .
             fi
             if [["[[ ! -z $CIRCLE_TAG ]]"]]; then
               docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus .
+              docker build --target lotus-gateway -t filecoin/lotus-gateway:$CIRCLE_TAG -f Dockerfile.lotus .
               docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus .
             fi
       - run:
           name: docker push
           command: |
             docker push filecoin/lotus:<< parameters.tag >>
+            docker push filecoin/lotus-gateway:<< parameters.tag >>
             docker push filecoin/lotus-all-in-one:<< parameters.tag >>
             if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then
               docker push filecoin/lotus:$CIRCLE_SHA1
+              docker push filecoin/lotus-gateway:$CIRCLE_SHA1
               docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1
             fi
             if [["[[ ! -z $CIRCLE_TAG ]]"]]; then
               docker push filecoin/lotus:$CIRCLE_TAG
+              docker push filecoin/lotus-gateway:$CIRCLE_TAG
               docker push filecoin/lotus-all-in-one:$CIRCLE_TAG
             fi
 
@@ -782,15 +819,13 @@ workflows:
       - test:
           go-test-flags: "-run=TestMulticoreSDR"
           suite: multicore-sdr-check
-          target: "./extern/sector-storage/ffiwrapper"
+          target: "./storage/sealer/ffiwrapper"
           proofs-log-test: "1"
       - test-conformance:
           suite: conformance
-          codecov-upload: false
           target: "./conformance"
       - test-conformance:
           name: test-conformance-bleeding-edge
-          codecov-upload: false
           suite: conformance-bleeding-edge
           target: "./conformance"
           vectors-branch: specs-actors-v7
@@ -800,7 +835,7 @@ workflows:
               only:
                 - master
       - build-debug
-      - build-all:
+      - build-linux:
           filters:
             tags:
               only:
@@ -817,11 +852,36 @@ workflows:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-lotus-soup
       - build-macos:
+          name: publish-macos
+          publish: true
           filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+$/
+      - build-macos:
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+-rc\d+$/
+      - build-appimage:
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-appimage:
+      - publish:
+          name: publish-linux
+          linux: true
+          requires:
+            - build-linux
           filters:
             branches:
               ignore:
@@ -830,9 +890,9 @@ workflows:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - publish:
+          name: publish-appimage
+          appimage: true
           requires:
-            - build-all
-            - build-macos
             - build-appimage
           filters:
             branches:
@@ -842,27 +902,63 @@ workflows:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-and-push-image:
+          name: build-and-push/lotus-all-in-one
           dockerfile: Dockerfile.lotus
           path: .
           repo: lotus-dev
           tag: '${CIRCLE_SHA1:0:8}'
           target: lotus-all-in-one
       - build-and-push-image:
+          name: build-and-push/lotus-test
           dockerfile: Dockerfile.lotus
           path: .
           repo: lotus-test
           tag: '${CIRCLE_SHA1:0:8}'
           target: lotus-test
       - publish-snapcraft:
-          name: publish-snapcraft-stable
+          name: "Publish Snapcraft (lotus-filecoin / candidate)"
           channel: stable
+          snap-name: lotus-filecoin
           filters:
             branches:
               ignore:
                 - /.*/
             tags:
               only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+                - /^v\d+\.\d+\.\d+$/
+      - publish-snapcraft:
+          name: "Publish Snapcraft (lotus-filecoin / candidate)"
+          channel: candidate
+          snap-name: lotus-filecoin
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+-rc\d+$/
+      - publish-snapcraft:
+          name: "Publish Snapcraft (lotus / stable)"
+          channel: stable
+          snap-name: lotus
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+$/
+      - publish-snapcraft:
+          name: "Publish Snapcraft (lotus / candidate)"
+          channel: candidate
+          snap-name: lotus
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+-rc\d+$/
       - publish-dockerhub:
           name: publish-dockerhub
           tag: stable
@@ -884,8 +980,13 @@ workflows:
                 - master
     jobs:
       - publish-snapcraft:
-          name: publish-snapcraft-nightly
+          name: "Publish Snapcraft Nightly (lotus-filecoin / edge)"
           channel: edge
+          snap-name: lotus-filecoin
+      - publish-snapcraft:
+          name: "Publish Snapcraft Nightly (lotus / edge)"
+          channel: edge
+          snap-name: lotus
       - publish-dockerhub:
           name: publish-dockerhub-nightly
           tag: nightly

.codecov.yml

@@ -1,74 +0,0 @@
-ignore:
-  # Auto generated
-  - "^.*_gen.go$"
-  - "^.*/mock_full.go$"
-  # Old actors.
-  - "^chain/actors/builtin/[^/]*/(message|state|v)[0-4]\\.go$" # We test the latest version only.
-  # Tests
-  - "api/test/**"
-  - "conformance/**"
-  # Generators
-  - "gen/**"
-  - "chain/actors/agen/**"
-  # Non-critical utilities
-  - "api/docgen/**"
-  - "api/docgen-openrpc/**"
-coverage:
-  status:
-    patch: off
-    project:
-      threshold: 1%
-      tools-and-tests:
-        target: auto
-        threshold: 1%
-        informational: true
-        paths:
-          - "testplans"
-          - "tools"
-          - "system"
-          - "snap"
-          - "lotuspond"
-          - "conformance"
-          - "scripts"
-          - "gen"
-          - "build"
-      markets: 
-        target: auto
-        threshold: 1%
-        informational: true        
-        paths: 
-          - "markets"
-          - "paychmgr"
-      miner: 
-        target: auto
-        threshold: 1.5%
-        informational: true
-        paths: 
-          - "miner"
-          - "storage"
-      chain:
-        target: auto
-        threshold: 1%
-        informational: true
-        paths:
-          - "chain"
-      node:
-        target: auto
-        threshold: 1%
-        informational: true
-        paths: 
-          - "node"
-          - "blockstore"
-          - "metrics"
-          - "lib"
-          - "genesis"
-          - "gateway"
-          - "api"
-          - "journal"
-      cli: 
-        target: auto
-        threshold: 1%
-        informational: true
-        paths: 
-          - "cli"
-          - "cmd"

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -81,7 +81,7 @@ body:
     render: text
     description: |
       Please provide debug logs of the problem, remember you can get set log level control for:
-      * lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/#log-level-control).
+      * lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://lotus.filecoin.io/lotus/configure/defaults/#log-level-control).
       * lotus-miner:`lotus-miner log list` to get all log systems available and set level by `lotus-miner log set-level
       If you don't provide detailed logs when you raise the issue it will almost certainly be the first request I make before furthur diagnosing the problem.
   validations:

.github/workflows/codeql-analysis.yml

@@ -37,7 +37,7 @@ jobs:
 
     - uses: actions/setup-go@v1
       with:
-        go-version: '1.17.9'
+        go-version: '1.18.1'
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/testground-on-push.yml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: testground run
-        uses: coryschwartz/testground-github-action@v1.1
+        uses: testground/testground-github-action@v1
         with:
           backend_addr: ${{ matrix.backend_addr }}
           backend_proto: ${{ matrix.backend_proto }}

.gitignore

@@ -51,3 +51,5 @@ scratchpad
 
 build/builtin-actors/v*
 build/builtin-actors/*.car
+
+dist/

.goreleaser.yaml

@@ -0,0 +1,165 @@
+project_name: lotus
+before:
+  hooks:
+    - go mod tidy
+    - make deps
+
+universal_binaries:
+  - id: lotus
+    replace: true
+    name_template: lotus
+    ids:
+      - lotus_darwin_amd64
+      - lotus_darwin_arm64
+  - id: lotus-miner
+    replace: true
+    name_template: lotus-miner
+    ids:
+      - lotus-miner_darwin_amd64
+      - lotus-miner_darwin_arm64
+  - id: lotus-worker
+    replace: true
+    name_template: lotus-worker
+    ids:
+      - lotus-worker_darwin_amd64
+      - lotus-worker_darwin_arm64
+
+builds:
+  - id: lotus_darwin_amd64
+    main: ./cmd/lotus
+    binary: lotus
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-miner_darwin_amd64
+    main: ./cmd/lotus-miner
+    binary: lotus-miner
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-worker_darwin_amd64
+    main: ./cmd/lotus-worker
+    binary: lotus-worker
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus_darwin_arm64
+    main: ./cmd/lotus
+    binary: lotus
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-miner_darwin_arm64
+    main: ./cmd/lotus-miner
+    binary: lotus-miner
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-worker_darwin_arm64
+    main: ./cmd/lotus-worker
+    binary: lotus-worker
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+#  - id: linux
+#    main: ./cmd/lotus
+#    binary: lotus
+#    goos:
+#      - linux
+#    goarch:
+#      - amd64
+#    env:
+#      - CGO_ENABLED=1
+#    ldflags:
+#      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+
+archives:
+  - id: primary
+    format: tar.gz
+    wrap_in_directory: true
+    files:
+      # this is a dumb but required hack so it doesn't include the default files
+      # https://github.com/goreleaser/goreleaser/issues/602
+      - _n_o_n_e_*
+
+release:
+  github:
+    owner: filecoin-project
+    name: lotus
+  prerelease: auto
+  name_template: "Release v{{.Version}}"
+
+
+brews:
+  - tap:
+      owner: filecoin-project
+      name: homebrew-lotus
+      branch: master
+    ids:
+      - primary
+    install: |
+      bin.install "lotus"
+      bin.install "lotus-miner"
+      bin.install "lotus-worker"
+    test: |
+      system "#{bin}/lotus --version"
+      system "#{bin}/lotus-miner --version"
+      system "#{bin}/lotus-worker --version"
+    folder: Formula
+    homepage: "https://filecoin.io"
+    description: "A homebrew cask for installing filecoin-project/lotus on MacOS"
+    license: MIT
+    dependencies:
+      - name: pkg-config
+      - name: jq
+      - name: bzr
+      - name: hwloc
+
+# produced manually so we can include cid checksums
+checksum:
+  disable: true
+
+snapshot:
+  name_template: "{{ .Tag }}"

AppImageBuilder.yml

@@ -68,6 +68,4 @@ AppDir:
       use_host_x: false
 AppImage:
   arch: x86_64
-  update-information: guess
-  sign-key: None
 

CHANGELOG.md

@@ -1,7 +1,433 @@
 # Lotus changelog
 
-# 1.16.0 / 2022-06-24
+# v1.17.2 / 2022-10-05
 
+This is an OPTIONAL release of Lotus. This feature release introduces new sector number management APIs in Lotus that enables all the Sealing-as-a-Service and Lotus interactions needed to function. The default propagation delay setting for storage providers has also been changed, as well as numerous other features and enhancements. Check out the sub-bullet points in the feature and enhancement section to get a short description about each feature and enhancements.
+
+### Highlights
+
+🦭 **SaaS** 🦭
+New sector management APIs makes it possible to import partially sealed sectors into Lotus. This release implements all SaaS<->Lotus interactions needed for such services to work. Deep dive into the new APIs here: https://github.com/filecoin-project/lotus/discussions/9079#discussioncomment-3652044
+
+⏳ **Propagation delay** ⌛️
+In v1.17.2 the default PropagationDelay has been raised from 6 seconds -> 10 seconds, and you can tune this yourself with the `PROPAGATION_DELAY_SECS` environment variable. This means you will now wait for 10 seconds for other blocks to arrive from the network before computing a winningPoSt (if eligible). In your `lotus-miner` logs that means you will see this "baseDeltaSeconds": 10 as default.
+
+⚠️ **Please note that Lotus v1.17.2 will require a Go-version of v1.18.1 or higher!**
+
+## New features
+- feat: sealing: Partially sealed sector import ([filecoin-project/lotus#9210](https://github.com/filecoin-project/lotus/pull/9210))
+  - Implements support for importing (partially) sealed sectors which is needed for enabling external sealing services.
+- feat: sealing: Use bitfields to manage sector numbers ([filecoin-project/lotus#9183](https://github.com/filecoin-project/lotus/pull/9183))
+  - Needed for Sealing-as-a-Service. Move sector number assigning logic to use stored bitfields instead of stored counters. Makes it possible to reserve ranges of sector numbers, see the sector assigner state and view sector number reservations.
+- feat: env: propagation delay ([filecoin-project/lotus#9290](https://github.com/filecoin-project/lotus/pull/9290))
+  - The default propagation delay is raised to 10 seconds from 6 seconds. Ability to set it yourself with the `PROPAGATION_DELAY_SECS` environment variable.
+- feat: cli: lotus info cmd ([filecoin-project/lotus#9233](https://github.com/filecoin-project/lotus/pull/9233))
+  - A new `lotus info` command that prints useful node information in one place.
+- feat: proving: Introduce manual sector fault recovery (#9144) ([filecoin-project/lotus#9144](https://github.com/filecoin-project/lotus/pull/9144))
+  - Allow users to declare fault recovery messages manually with the `lotus-miner proving recover-faults` command, rather than waiting for it to happen automatically before windowPost.
+- feat: api: Reintroduce StateActorManifestCID ([filecoin-project/lotus#9201](https://github.com/filecoin-project/lotus/pull/9201))
+  - Adds ability to retrieve the Actor Manifest CID through the api.
+- feat: message: Add uuid to mpool message sent to chain node from miner ([filecoin-project/lotus#9174](https://github.com/filecoin-project/lotus/pull/9174))
+  - Adds a UUID to each message sent by the `lotus-miner` to the daemon. A requirement needed for https://github.com/filecoin-project/lotus/issues/9130
+- feat: message: Add retries to mpool push message from lotus-miner ([filecoin-project/lotus#9177](https://github.com/filecoin-project/lotus/pull/9177))
+  - Retries to mpool push message API in case of unavailability of the lotus chain node.
+
+**Network 17 related features :**
+- feat: network: add nv17 and integrate the corresponding go state type ([filecoin-project/lotus#9267](https://github.com/filecoin-project/lotus/pull/9267))
+- feat: cli: print beneficiary info in state miner-info ([filecoin-project/lotus#9308](https://github.com/filecoin-project/lotus/pull/9308))
+- feat: api/cli: change beneficiary propose and confirm for actors and multisigs. ([filecoin-project/lotus#9307](https://github.com/filecoin-project/lotus/pull/9307))
+- feat: api/cli: beneficiary withdraw api and cli ([filecoin-project/lotus#9296](https://github.com/filecoin-project/lotus/pull/9296))
+
+## Enhancements
+- feat: sectors renew --only-cc ([filecoin-project/lotus#9184](https://github.com/filecoin-project/lotus/pull/9184))
+  - Exlude extending deal-related sectors with the `--only-cc` option when using the `lotus-miner sectors renew`
+- feat: miner: display updated & update-cache for storage list ([filecoin-project/lotus#9323](https://github.com/filecoin-project/lotus/pull/9323))
+  - Show amount of `updated` & `update-cache` sectors in each storage path in the `lotus-miner storage list` output
+- feat: add descriptive errors to markets event handler ([filecoin-project/lotus#9326](https://github.com/filecoin-project/lotus/pull/9326))
+  - More descriptive market error logs
+- feat: cli: Add option to terminate sectors from worker address ([filecoin-project/lotus#9291](https://github.com/filecoin-project/lotus/pull/9291))
+  - Adds a flag to allow either owner address or worker address to send terminate sectors message.
+- fix: cli: actor-cids cli command now defaults to current network ([filecoin-project/lotus#9321](https://github.com/filecoin-project/lotus/pull/9321))
+  - Makes the command defaults to the current network.
+- fix: ux: Output bytes in `lotus client commP` cmd ([filecoin-project/lotus#9189](https://github.com/filecoin-project/lotus/pull/9189))
+  - Adds an additional line that outputs bytes in the `lotus client commP` command.
+- fix: sealing: Add information on what worker a job was assigned to in logs ([filecoin-project/lotus#9151](https://github.com/filecoin-project/lotus/pull/9151))
+  - Adds the worker hostname into the assignment logs.
+- refactor: sealing pipeline: Remove useless storage adapter code ([filecoin-project/lotus#9142](https://github.com/filecoin-project/lotus/pull/9142)
+  - Remove proxy code in `storage/miner.go` / `storage/miner_sealing.go`, call the pipeline directly instead.
+
+## Bug fixes
+- fix: ffiwrapper: Close readers in AddPiece ([filecoin-project/lotus#9328](https://github.com/filecoin-project/lotus/pull/9328))
+- fix: sealing: Drop unused PreCommitInfo from pipeline.SectorInfo ([filecoin-project/lotus#9325](https://github.com/filecoin-project/lotus/pull/9325))
+- fix: cli: fix panic in `lotus-miner actor control list` ([filecoin-project/lotus#9241](https://github.com/filecoin-project/lotus/pull/9241))
+- fix: sealing: Abort upgrades in sectors with no deals ([filecoin-project/lotus#9310](https://github.com/filecoin-project/lotus/pull/9310))
+- fix: sealing: Make DataCid resource env vars make more sense ([filecoin-project/lotus#9231](https://github.com/filecoin-project/lotus/pull/9231))
+- fix: cli: Option to specify --from msg sender ([filecoin-project/lotus#9237](https://github.com/filecoin-project/lotus/pull/9237))
+- fix: ux: better ledger rejection error ([filecoin-project/lotus#9242](https://github.com/filecoin-project/lotus/pull/9242))
+- fix: ux: msg receipt for actor withdrawal ([filecoin-project/lotus#9155](https://github.com/filecoin-project/lotus/pull/9155))
+- fix: ux: exclude negative available balance from spendable amount ([filecoin-project/lotus#9182](https://github.com/filecoin-project/lotus/pull/9182))
+- fix: sealing: Avoid panicking in handleUpdateActivating on startup ([filecoin-project/lotus#9331](https://github.com/filecoin-project/lotus/pull/9331))
+- fix: api: DataCid - ensure reader is closed ([filecoin-project/lotus#9230](https://github.com/filecoin-project/lotus/pull/9230))
+- fix: verifreg: serialize RmDcProposalID as int, not tuple ([filecoin-project/lotus#9206](https://github.com/filecoin-project/lotus/pull/9206))
+- fix: api: Ignore uuid check for messages with uuid not set ([filecoin-project/lotus#9303](https://github.com/filecoin-project/lotus/pull/9303))
+- fix: cgroupV1: memory.memsw.usage_in_bytes: no such file or directory ([filecoin-project/lotus#9202](https://github.com/filecoin-project/lotus/pull/9202))
+- fix: miner: init miner's with 32GiB sectors by default ([filecoin-project/lotus#9364](https://github.com/filecoin-project/lotus/pull/9364))
+- fix: worker: Close all storage paths on worker shutdown ([filecoin-project/lotus#9153](https://github.com/filecoin-project/lotus/pull/9153))
+- fix: build: set PropagationDelaySecs correctly ([filecoin-project/lotus#9358](https://github.com/filecoin-project/lotus/pull/9358))
+- fix: renew --only-cc with sectorfile ([filecoin-project/lotus#9428](https://github.com/filecoin-project/lotus/pull/9428))
+
+## Dependency updates
+- github.com/filecoin-project/go-fil-markets (v1.23.1 -> v1.24.0)
+- github.com/filecoin-project/go-jsonrpc (v0.1.5 -> v0.1.7)
+- github.com/filecoin-project/go-state-types (v0.1.10 -> v0.1.12-beta)
+- github.com/filecoin-project/go-commp-utils/nonffi (null -> v0.0.0-20220905160352-62059082a837)
+- deps: go-libp2p-pubsub v0.8.0 ([filecoin-project/lotus#9229](https://github.com/filecoin-project/lotus/pull/9229))
+- deps: libp2p v0.22 ([filecoin-project/lotus#9216](https://github.com/filecoin-project/lotus/pull/9216))
+- deps: Use latest cbor-gen ([filecoin-project/lotus#9335](https://github.com/filecoin-project/lotus/pull/9335))
+- chore: update bitswap and some libp2p packages ([filecoin-project/lotus#9279](https://github.com/filecoin-project/lotus/pull/9279))
+
+## Others
+- chore: merge releases into master after v1.17.1 release ([filecoin-project/lotus#9283](https://github.com/filecoin-project/lotus/pull/9283))
+- chore: docs: Fix dead links to docs.filecoin.io ([filecoin-project/lotus#9304](https://github.com/filecoin-project/lotus/pull/9304))
+- chore: deps: update FFI ([filecoin-project/lotus#9330](https://github.com/filecoin-project/lotus/pull/9330))
+- chore: seed: add cmd for adding signers to rkh to genesis ([filecoin-project/lotus#9198](https://github.com/filecoin-project/lotus/pull/9198))
+- chore: fix typo in comment ([filecoin-project/lotus#9161](https://github.com/filecoin-project/lotus/pull/9161))
+- chore: cli: cleanup and standardize cli ([filecoin-project/lotus#9317](https://github.com/filecoin-project/lotus/pull/9317))
+- chore: versioning: Bump version to v1.17.2-dev ([filecoin-project/lotus#9147](https://github.com/filecoin-project/lotus/pull/9147))
+- chore: release: v1.17.2-rc1 ([filecoin-project/lotus#9339](https://github.com/filecoin-project/lotus/pull/9339))
+- feat: shed: add a --max-size flag to vlog2car ([filecoin-project/lotus#9212](https://github.com/filecoin-project/lotus/pull/9212))
+- fix: docsgen: revert rename of API Name to Num ([filecoin-project/lotus#9315](https://github.com/filecoin-project/lotus/pull/9315))
+- fix: ffi: Revert accidental filecoin-ffi downgrade from #9144 ([filecoin-project/lotus#9277](https://github.com/filecoin-project/lotus/pull/9277))
+- fix: miner: Call SyncBasefeeCheck from `lotus info` ([filecoin-project/lotus#9281](https://github.com/filecoin-project/lotus/pull/9281))
+- fix: mock sealer: grab lock in ReadPiece ([filecoin-project/lotus#9207](https://github.com/filecoin-project/lotus/pull/9207))
+- refactor: use `os.ReadDir` for lightweight directory reading ([filecoin-project/lotus#9282](https://github.com/filecoin-project/lotus/pull/9282))
+- tests: cli: Don't panic with no providers in client retrieve ([filecoin-project/lotus#9232](https://github.com/filecoin-project/lotus/pull/9232))
+- build: artifacts: butterfly ([filecoin-project/lotus#9027](https://github.com/filecoin-project/lotus/pull/9027))
+- build: Use lotus snap (and fix typo) for packer builds ([filecoin-project/lotus#9152](https://github.com/filecoin-project/lotus/pull/9152))
+- build: Update xcode version for macos builds ([filecoin-project/lotus#9170](https://github.com/filecoin-project/lotus/pull/9170))
+- ci: build: Snap daemon autorun disable ([filecoin-project/lotus#9167](https://github.com/filecoin-project/lotus/pull/9167))
+- ci: Use golang 1.18.1 to build appimage ([filecoin-project/lotus#9389](https://github.com/filecoin-project/lotus/pull/9389))
+- ci: Don't publish new homebrew releases for RC builds ([filecoin-project/lotus#9350](https://github.com/filecoin-project/lotus/pull/9350))
+- Merge branch 'deps/go-libp2p-v0.21'
+
+Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| Aayush Rajasekaran | 8 | +23010/-2122 | 109 |
+| Aayush | 15 | +6168/-2679 | 360 |
+| Łukasz Magiera | 69 | +6462/-2137 | 606 |
+| Geoff Stuart | 19 | +3080/-1177 | 342 |
+| Marco Munizaga | 16 | +543/-424 | 41 |
+| Shrenuj Bansal | 30 | +485/-419 | 88 |
+| LexLuthr | 3 | +498/-12 | 19 |
+| Phi | 15 | +330/-70 | 17 |
+| Jennifer Wang | 7 | +132/-12 | 11 |
+| TippyFlitsUK | 1 | +43/-45 | 12 |
+| Steven Allen | 1 | +18/-28 | 2 |
+| Frrist | 1 | +19/-11 | 2 |
+| Eng Zer Jun | 1 | +14/-11 | 6 |
+| Dirk McCormick | 2 | +23/-1 | 3 |
+| Ian Davis | 3 | +7/-9 | 3 |
+| Masih H. Derkani | 1 | +11/-0 | 1 |
+| Anton Evangelatov | 1 | +11/-0 | 1 |
+| Yu | 2 | +4/-4 | 5 |
+| Hannah Howard | 1 | +4/-4 | 1 |
+| Phi-rjan | 1 | +1/-2 | 1 |
+| Jiaying Wang | 1 | +3/-0 | 1 |
+| nujz | 1 | +1/-1 | 1 |
+| Rob Quist | 1 | +1/-1 | 1 |
+
+# v1.17.1 / 2022-09-06
+
+This is an optional release of Lotus. This release introduces the [Splitstore v2 - beta](https://github.com/filecoin-project/lotus/blob/master/blockstore/splitstore/README.md)(beta). Splitstore aims to reduce the node performance impact that's caused by the Filecoin's very large, and continuously growing datastore. Splitstore v2 introduces  the coldstore auto prune/GC feature & some improvements for the hotstore.  We welcome all lotus users to join the early testers and try the new Splitstore out, you can leave any feedback or report issues in [this discussion](https://github.com/filecoin-project/lotus/discussions/9179) or create an issue. As always,  multiple small bug fixes, new features & improvements are also included in this release.
+
+
+## New features
+
+- feat:chain:splitstore auto prune ([filecoin-project/lotus#9123](https://github.com/filecoin-project/lotus/pull/9123))
+  - Trigger SplitStore chain prune on head events. [Link to the documentation](https://lotus.filecoin.io/lotus/manage/chain-management/#cold-store-garbage-collection)
+- feat:chain:splitstore chain prune ([filecoin-project/lotus#9056](https://github.com/filecoin-project/lotus/pull/9056))
+  - Adds `chain prune` command to trigger manual garbage collection. [Link to the documentation](https://lotus.filecoin.io/lotus/manage/chain-management/#cold-store-garbage-collection)
+- feat: storage: Path type filters ([filecoin-project/lotus#9013](https://github.com/filecoin-project/lotus/pull/9013))
+  - Adds new fields to `sectorstore.json` to allow file type filtering. [Link to the documentation](https://lotus.filecoin.io/storage-providers/operate/custom-storage-layout/#filter-sector-types-1)
+- feat: sealing: storage redeclare/detach ([filecoin-project/lotus#9032](https://github.com/filecoin-project/lotus/pull/9032))
+  - Adds new Lotus commands to detach and redeclare storage paths. [Link to the documentation](https://lotus.filecoin.io/storage-providers/operate/custom-storage-layout/#detach-storage-paths)
+- feat: worker: Add stop cmd for lotus worker ([filecoin-project/lotus#9101](https://github.com/filecoin-project/lotus/pull/9101))
+  - Adds new `lotus-worker stop` command. [Link to the documentation](https://lotus.filecoin.io/storage-providers/seal-workers/seal-workers/#stop-the-worker)
+- feat: market: Add lotus-shed cmd to get total active deal storage ([filecoin-project/lotus#9113](https://github.com/filecoin-project/lotus/pull/9113))
+  - `get-deals-total-storage` - View the total storage available in all active market deals
+- feat: wdpost: Envvar for limiting recovering sectors ([filecoin-project/lotus#9106](https://github.com/filecoin-project/lotus/pull/9106))
+  - Adds new envvar to limit the number of sectors declared in the recover message
+
+
+## Improvements
+
+- feat: sealing: Allow overriding worker hostname ([filecoin-project/lotus#9116](https://github.com/filecoin-project/lotus/pull/9116))
+- feat: build: run fiximports on make actors-gen ([filecoin-project/lotus#9114](https://github.com/filecoin-project/lotus/pull/9114))
+- feat: FVM: always enable tracing for user-triggered executions ([filecoin-project/lotus#9036](https://github.com/filecoin-project/lotus/pull/9036))
+- feat: miner cli: proving deadline command enchantments ([filecoin-project/lotus#9109](https://github.com/filecoin-project/lotus/pull/9109))
+- FVM: Use MaxInt64 for Implicit Message gas limits ([filecoin-project/lotus#9037](https://github.com/filecoin-project/lotus/pull/9037))
+- lotus shed addr decode
+- push lotus-gateway to docker hub ([filecoin-project/lotus#8969](https://github.com/filecoin-project/lotus/pull/8969))
+- Review Response
+- test: net: net and conngater tests ([filecoin-project/lotus#8084](https://github.com/filecoin-project/lotus/pull/8084))
+- Update FFI ([filecoin-project/lotus#9139](https://github.com/filecoin-project/lotus/pull/9139))
+
+## Bug Fixes
+
+- backport: 9153: detach storage on worker shutdown ([filecoin-project/lotus#9127](https://github.com/filecoin-project/lotus/pull/9165))
+- fix makegen
+- fix: build: use GOCC when building lotus-fountain ([filecoin-project/lotus#9127](https://github.com/filecoin-project/lotus/pull/9127))
+- fix: ci: Forgot a .sh on the end of a the new publish script ([filecoin-project/lotus#9088](https://github.com/filecoin-project/lotus/pull/9088))
+- fix: cli: ./lotus-miner actor control list, if the owner is not account ([filecoin-project/lotus#9072](https://github.com/filecoin-project/lotus/pull/9072))
+- fix: deps: update FFI to fix a slow memory leak ([filecoin-project/lotus#9042](https://github.com/filecoin-project/lotus/pull/9042))
+- fix: FVM: record message applied metrics ([filecoin-project/lotus#9052](https://github.com/filecoin-project/lotus/pull/9052))
+- fix: gas: estimate gas with a zero base-fee ([filecoin-project/lotus#8991](https://github.com/filecoin-project/lotus/pull/8991))
+- fix: post: restrict recoveries per deadline ([filecoin-project/lotus#9111](https://github.com/filecoin-project/lotus/pull/9111))
+- fix: sealing: Workaround for sealing bug ([filecoin-project/lotus#9043](https://github.com/filecoin-project/lotus/pull/9043))
+- fix: storage: don't panic in getCommitCutoff when precommit is not found ([filecoin-project/lotus#9141](https://github.com/filecoin-project/lotus/pull/9141))
+- fix: test: deflake TestQuotePriceForUnsealedRetrieval ([filecoin-project/lotus#9084](https://github.com/filecoin-project/lotus/pull/9084))
+
+## Dependency Updates
+
+- github.com/multiformats/go-multibase (v0.0.3 -> v0.1.1)
+
+## Others
+
+- chore: ci: Update xcode version for macos builds ([filecoin-project/lotus#9164)](https://github.com/filecoin-project/lotus/pull/9164))
+- Merge branch 'docs/ysrotciv-desc'
+- Merge branch 'feat/f8-worker-env'
+- Merge branch 'LexLuthr-feat/minerWithdrawBalanceAPI'
+- Merge branch 'LexLuthr-feat/SchedRemoveRequest'
+- base256emoji ([filecoin-project/lotus#9038)](https://github.com/filecoin-project/lotus/pull/9038))
+- chore: interop: update interop assets ([filecoin-project/lotus#9093)](https://github.com/filecoin-project/lotus/pull/9093))
+- chore: merge: releases (v1.17.0) to master ([filecoin-project/lotus#9146)](https://github.com/filecoin-project/lotus/pull/9146))
+- chore: sealer: Fixup typos ([filecoin-project/lotus#9040)](https://github.com/filecoin-project/lotus/pull/9040))
+- chore:docs:remove readme reference to deprecated specs-actors ([filecoin-project/lotus#8984)](https://github.com/filecoin-project/lotus/pull/8984))
+- ci : Change default shell options for snapcraft publish ([filecoin-project/lotus#9122)](https://github.com/filecoin-project/lotus/pull/9122))
+- ci: More tweaks to snapcraft release process ([filecoin-project/lotus#9090)](https://github.com/filecoin-project/lotus/pull/9090))
+- ci: Publish to both lotus and lotus-filecoin for snap ([filecoin-project/lotus#9119)](https://github.com/filecoin-project/lotus/pull/9119))
+- ci: Run snap builds for lotus and lotus-filecoin in parallel ([filecoin-project/lotus#9133)](https://github.com/filecoin-project/lotus/pull/9133))
+- ci: Switches goreleaser notes back to default (keep-existing) ([filecoin-project/lotus#9120)](https://github.com/filecoin-project/lotus/pull/9120))
+- ci: update snapcraft and release flow logic ([filecoin-project/lotus#8994)](https://github.com/filecoin-project/lotus/pull/8994))
+- ci: Use goreleaser to build macos universal binaries (including M1 macs) ([filecoin-project/lotus#9096)](https://github.com/filecoin-project/lotus/pull/9096))
+- ci:testing:remove codecov ([filecoin-project/lotus#9062)](https://github.com/filecoin-project/lotus/pull/9062))
+
+
+Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| Łukasz Magiera | 34 | +2329/-317 | 163 |
+| ZenGround0 | 2 | +1527/-89 | 38 |
+| Ian Davis | 14 | +751/-232 | 30 |
+| LexLuthr | 17 | +480/-225 | 63 |
+| TheMenko | 4 | +323/-61 | 5 |
+| Aayush | 10 | +285/-92 | 30 |
+| beck | 3 | +143/-93 | 3 |
+| Steven Allen | 4 | +95/-75 | 9 |
+| zenground0 | 5 | +44/-116 | 9 |
+| Shrenuj Bansal | 7 | +136/-7 | 16 |
+| Patrick Deuse | 3 | +76/-57 | 3 |
+| Jennifer Wang | 3 | +6/-52 | 11 |
+| zl | 2 | +20/-16 | 2 |
+| Aayush Rajasekaran | 2 | +6/-6 | 2 |
+| Clint Armstrong | 1 | +7/-3 | 1 |
+| Cory Schwartz | 2 | +9/-0 | 2 |
+| Jorropo | 1 | +3/-2 | 2 |
+| Geoff Stuart | 1 | +5/-0 | 1 |
+| Frank Y | 1 | +2/-2 | 2 |
+| Aloxaf | 1 | +2/-2 | 1 |
+
+
+# Lotus changelog
+
+# v1.17.0 / 2022-08-02
+
+This is an optional release of Lotus. This feature release introduces a lot of new sealing and scheduler improvements, and many other functionalities and bug fixes.
+
+PSA: Markets related features, enhancements and fixes is now lower priority for Lotus, and is going to be in the hands of [Boost](https://boost.filecoin.io), built by the amazing [Bedrock team](https://www.notion.so/pl-strflt/Bedrock-2e956d5d8143432080a1d84435ccf0ff). You can find Lotus mission scope [here](https://www.notion.so/Lotus-8352bbb6c321431abd8790a7a3401ed3#805c645f592840ad893c272723362d3d)
+
+## New features
+
+- feat: worker: lotus-worker run --no-default ([filecoin-project/lotus#8672](https://github.com/filecoin-project/lotus/pull/8672))
+  - Makes it very easy to spin up workers that make the compute tasks opt-in, instead of the default opt-out. Also makes it very easy to create storage-only workers. [Link to the documentation](https://lotus.filecoin.io/storage-providers/seal-workers/seal-workers/#run-the-worker)
+- feat: sched: Per worker concurrent task count limits ([filecoin-project/lotus#8725](https://github.com/filecoin-project/lotus/pull/8725))
+  - Set the maximum number of tasks running it parallel on workers by exporting env-variables: `[short task type]_[sector size]_MAX_CONCURRENT=[limit]`. [Link to documentation](https://lotus.filecoin.io/storage-providers/seal-workers/seal-workers/#limit-tasks-run-in-parallel)
+- feat: sched: Finalize* move selectors ([filecoin-project/lotus#8710](https://github.com/filecoin-project/lotus/pull/8710))
+  - Allows you to force all Finalize tasks to run on workers with local access to both long-term storage and the sealing path containing the sector.
+- feat: sched: Add scheduler interfaces, configurable assigner ([filecoin-project/lotus#8700](https://github.com/filecoin-project/lotus/pull/8700)) 
+  - Introduce a new simpler worker assigning logic which will attempt to assign tasks to as many workers as possible and ignore worker utilization. [Link to documentation](https://lotus.filecoin.io/storage-providers/advanced-configurations/sealing/#worker-assigning-logic)
+- feat: bench: simple sealing operations commands ([filecoin-project/lotus#8373](https://github.com/filecoin-project/lotus/pull/8373))
+  - Allows you to only test the performance of a single task. [Read the documentation](https://lotus.filecoin.io/storage-providers/operate/benchmarks/#single-task-benchmark).
+- feat: miner cli: sealing data-cid command ([filecoin-project/lotus#8715](https://github.com/filecoin-project/lotus/pull/8715))
+  - Makes it possible to compute data CIDs on lotus workers.
+- feat: precommits info ([filecoin-project/lotus#8696](https://github.com/filecoin-project/lotus/pull/8696))
+  - Check the on-chain precommit info with the `lotus-miner sectors precommits` command.
+- feat: dagstore: add dagstore register-shard command ([filecoin-project/lotus#8645](https://github.com/filecoin-project/lotus/pull/8645))
+  - If you have broken indexes in your market’s dagstore, you can try to recover it by using `lotus-miner dagstore register-shard`. [Link to documentation](https://lotus.filecoin.io/kb/dagstore-register/)
+- feat: Implement cli command for compactPartitions ([filecoin-project/lotus#8637](https://github.com/filecoin-project/lotus/pull/8637))
+  - Remove dead sectors from partitions and reduce the number of partitions used if possible with the `lotus-miner sectors compact-partitions`command. [Link to documentation](https://lotus.filecoin.io/storage-providers/operate/daily-chores/#compacting-partitions)
+- feat: recovery: Config for maximum partition count per message ([filecoin-project/lotus#8986](https://github.com/filecoin-project/lotus/pull/8986)
+  - Adds config for setting the maximum amount of partitions to declare in a DeclareFaultsRecovered message [Link to documentation](https://lotus.filecoin.io/storage-providers/seal-workers/post-workers/#multiple-partitions)
+- feat: wdpost: Config for maximum partition count per message ([filecoin-project/lotus#8982](https://github.com/filecoin-project/lotus/pull/8982))
+  - Adds config for setting the maximum amount of partitions to prove in a SubmitWindowPoSt message [Link to documentation](https://lotus.filecoin.io/storage-providers/seal-workers/post-workers/#multiple-partitions)
+- feat: sealer: Config for disabling builtin PoSt / PoSt pre-checks ([filecoin-project/lotus#8959](https://github.com/filecoin-project/lotus/pull/8959))
+  - Adds the ability to fully disable PoSt tasks on the `lotus-miner` and disabling windowPoSt pre-checks. [Link to documentation](https://lotus.filecoin.io/storage-providers/advanced-configurations/workers/#post-computations)
+- feat: add create ledger wallet address by account index command ([filecoin-project/lotus#8657](https://github.com/filecoin-project/lotus/pull/8657))
+ 
+## Improvements
+
+- feat: wdpost: Ignore faults in lotus-miner proving compute window-post ([filecoin-project/lotus#8737](https://github.com/filecoin-project/lotus/pull/8737))
+- feat: cli: Nicer net stat ([filecoin-project/lotus#8797](https://github.com/filecoin-project/lotus/pull/8797))
+- feat: networking: add healthz and livez endpoints ([filecoin-project/lotus#8692](https://github.com/filecoin-project/lotus/pull/8692))
+- feat: Snap Deals full unseal ([filecoin-project/lotus#8478](https://github.com/filecoin-project/lotus/pull/8478))
+- feat: cli: Hide sector nums in 'proving deadline' by default ([filecoin-project/lotus#8952](https://github.com/filecoin-project/lotus/pull/8952))
+- feat: Add rate limiting to the lotus gateway ([filecoin-project/lotus#8517](https://github.com/filecoin-project/lotus/pull/8517))
+- feat: networking: disconnect cmd ([filecoin-project/lotus#8955](https://github.com/filecoin-project/lotus/pull/8955))
+- feat: dagstore: Add DestroyShard() in dagstore wrapper ([filecoin-project/lotus#9010](https://github.com/filecoin-project/lotus/pull/9010))
+- feat: shed: report the "user version" ([filecoin-project/lotus#8864](https://github.com/filecoin-project/lotus/pull/8864))
+- feat: lotus-shed get remote peer hello message ([filecoin-project/lotus#8787](https://github.com/filecoin-project/lotus/pull/8787))
+- feat: only enable rcmgr by default in full nodes ([filecoin-project/lotus#8769](https://github.com/filecoin-project/lotus/pull/8769))
+- feat: refactor: actor bundling system (#8838) ([filecoin-project/lotus#8838](https://github.com/filecoin-project/lotus/pull/8838))
+- feat: migration: Implement function to migrate actors with only code changes ([filecoin-project/lotus#8843](https://github.com/filecoin-project/lotus/pull/8843))
+- feat: conformance & tvx: support ReportConsensusFault messages ([filecoin-project/lotus#8302](https://github.com/filecoin-project/lotus/pull/8302))
+- feat: wdpost: Envvar for limiting recovering sectors ([filecoin-project/lotus#9106](https://github.com/filecoin-project/lotus/pull/9106))
+- fix: post: restrict recoveries per deadline ([filecoin-project/lotus#9111](https://github.com/filecoin-project/lotus/pull/9111))
+- ux: print absolute time for proving period start in proving cli ([filecoin-project/lotus#8954](https://github.com/filecoin-project/lotus/pull/8954))
+- chore: storage refactors part 1 ([filecoin-project/lotus#8858](https://github.com/filecoin-project/lotus/pull/8858))
+- chore: sealing pipeline: Remove adapter code (storage refactors part 2) ([filecoin-project/lotus#8871](https://github.com/filecoin-project/lotus/pull/8871))
+- refactor: remove old BlockSyncProtocolID ([filecoin-project/lotus#8820](https://github.com/filecoin-project/lotus/pull/8820))
+
+## Bug Fixes
+
+- fix: format error log ([filecoin-project/lotus#8854](https://github.com/filecoin-project/lotus/pull/8854))
+- fix: build: really make macos compatible (#8853) ([filecoin-project/lotus#8853](https://github.com/filecoin-project/lotus/pull/8853))
+- fix: build: fix pack script and add calibrationnet to bundle ([filecoin-project/lotus#8852](https://github.com/filecoin-project/lotus/pull/8852))
+- fix: build: fix 2k build params ([filecoin-project/lotus#8835](https://github.com/filecoin-project/lotus/pull/8835))
+- Fix: PaychGetRestartAfterAddFundsMsg may stuck in forever waiting ([filecoin-project/lotus#8829](https://github.com/filecoin-project/lotus/pull/8829))
+- fix: paych: Print "waiting for confirmation.." ([filecoin-project/lotus#8823](https://github.com/filecoin-project/lotus/pull/8823))
+- fix: build: genesis miner network version ([filecoin-project/lotus#8756](https://github.com/filecoin-project/lotus/pull/8756))
+- fix: bench: consistency in description ([filecoin-project/lotus#8777](https://github.com/filecoin-project/lotus/pull/8777))
+- fix: worker: don't log normal storage stat calls ([filecoin-project/lotus#8744](https://github.com/filecoin-project/lotus/pull/8744))
+- fix: worker: don't check params with --no-default when not needed ([filecoin-project/lotus#8741](https://github.com/filecoin-project/lotus/pull/8741))
+- fix: Deduplicate parallel stat requests ([filecoin-project/lotus#8589](https://github.com/filecoin-project/lotus/pull/8589))
+- fix: Delegate storage auth on market nodes ([filecoin-project/lotus#8978](https://github.com/filecoin-project/lotus/pull/8978))
+- fix: post workers: check proving params on startup ([filecoin-project/lotus#8736](https://github.com/filecoin-project/lotus/pull/8736))
+- fix: rpc: readd rpc.discover aliases; lotus-gateway openrpc ([filecoin-project/lotus#8738](https://github.com/filecoin-project/lotus/pull/8738))
+- fix: change 1475 bootstrap peer ([filecoin-project/lotus#9008](https://github.com/filecoin-project/lotus/pull/9008))
+- fix: verifreg: update deprecation log ([filecoin-project/lotus#8690](https://github.com/filecoin-project/lotus/pull/8690))
+- fix: vm: support raw blocks in chain export ([filecoin-project/lotus#8691](https://github.com/filecoin-project/lotus/pull/8691))
+- fix: deps: restore butterfly network genesis from v1.14.4 ([filecoin-project/lotus#8708](https://github.com/filecoin-project/lotus/pull/8708))
+- fix: improve error message when maxPrice is too low ([filecoin-project/lotus#8818](https://github.com/filecoin-project/lotus/pull/8818))
+- fix: msig cli: Check for existing signers in add-propose ([filecoin-project/lotus#8833](https://github.com/filecoin-project/lotus/pull/8833))
+- Add new proofs version type ([filecoin-project/lotus#8848](https://github.com/filecoin-project/lotus/pull/8848))
+- fix: Make cli deal command get Block Delay specific to build ([filecoin-project/lotus#8896](https://github.com/filecoin-project/lotus/pull/8896))
+- fix: cli: Break out of retrieval if provider cancels ([filecoin-project/lotus#8912](https://github.com/filecoin-project/lotus/pull/8912))
+- fix: appimage build ([filecoin-project/lotus#8931](https://github.com/filecoin-project/lotus/pull/8931))
+- fix: incorrect usage of peer.IDFromString (should be peer.Decode) ([filecoin-project/lotus#8993](https://github.com/filecoin-project/lotus/pull/8993))
+- fix: deps: update FFI to fix a slow memory leak ([filecoin-project/lotus#9044](https://github.com/filecoin-project/lotus/pull/9044))
+- fix: sealing: hacky sealing fix backport ([filecoin-project/lotus#9055](https://github.com/filecoin-project/lotus/pull/9055))
+
+## Dependency Updates
+
+- github.com/filecoin-project/go-address (v0.0.6 -> v1.0.0)
+- github.com/filecoin-project/go-fil-markets (v1.20.1 -> v1.23.1)
+- github.com/filecoin-project/go-indexer-core (v0.2.8 -> v0.2.9)
+- github.com/filecoin-project/go-data-transfer (v1.15.1 -> v1.15.2)
+- github.com/filecoin-project/go-legs (v0.3.7 -> v0.4.4)
+- github.com/filecoin-project/go-state-types (v0.1.8 -> v0.1.10)
+- github.com/filecoin-project/index-provider (v0.5.0 -> v0.8.1)
+- github.com/filecoin-project/specs-actors (v0.9.14 -> v0.9.15)
+- github.com/filecoin-project/specs-actors/v3 (v3.1.1 -> v3.1.2)
+- github.com/filecoin-project/specs-actors/v4 (v4.0.1 -> v4.0.2)
+- github.com/filecoin-project/specs-actors/v5 (v5.0.4 -> v5.0.6)
+- github.com/filecoin-project/specs-actors/v6 (v6.0.1 -> v6.0.2)
+- github.com/filecoin-project/specs-actors/v7 (v7.0.0 -> v7.0.1)
+- github.com/filecoin-project/specs-actors/v8 (null -> v8.0.1)
+- github.com/filecoin-project/specs-storage (v0.2.4 -> v0.4.1)
+- github.com/filecoin-project/storetheindex (v0.3.5 -> v0.4.17)
+- deps: libp2p: update to the latest golibp2p tag ([filecoin-project/lotus#8704](https://github.com/filecoin-project/lotus/pull/8704))
+- chore: update and fix libp2p ([filecoin-project/lotus#8996](https://github.com/filecoin-project/lotus/pull/8996))
+
+## Others
+
+- chore: fix imports conflict ([filecoin-project/lotus#8863](https://github.com/filecoin-project/lotus/pull/8863))
+- chore: Fix imports ([filecoin-project/lotus#8859](https://github.com/filecoin-project/lotus/pull/8859))
+- chore: backport: release v1.16.0 back to master ([filecoin-project/lotus#8855](https://github.com/filecoin-project/lotus/pull/8855))
+- chore: bundle: remove wrongly committed bundle cars ([filecoin-project/lotus#8762](https://github.com/filecoin-project/lotus/pull/8762))
+- docs:sealing:fix default miner config comments ([filecoin-project/lotus#8689](https://github.com/filecoin-project/lotus/pull/8689))
+- ci: deps: Use testground-github-action from testground org ([filecoin-project/lotus#8490](https://github.com/filecoin-project/lotus/pull/8490))
+ 
+Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| Masih H. Derkani | 153 | +15515/-16832 | 660 |
+| Łukasz Magiera | 92 | +10429/-8024 | 1580 |
+| Andrew Gillis | 43 | +4149/-1765 | 208 |
+| Jennifer Wang | 10 | +1441/-1138 | 34 |
+| Geoff Stuart | 18 | +1348/-859 | 113 |
+| dirkmc | 11 | +1827/-210 | 70 |
+| Aayush | 21 | +1134/-894 | 69 |
+| Steven Allen | 9 | +743/-889 | 66 |
+| Marco Munizaga | 15 | +990/-252 | 36 |
+| gammazero | 47 | +681/-411 | 104 |
+| Will | 4 | +514/-246 | 29 |
+| web3-bot | 15 | +409/-348 | 20 |
+| Steven Fraser | 1 | +671/-0 | 36 |
+| Cory Schwartz | 27 | +520/-89 | 36 |
+| Hannah Howard | 3 | +318/-105 | 8 |
+| Piotr Galar | 2 | +337/-59 | 7 |
+| swift-mx | 14 | +264/-131 | 17 |
+| vyzo | 7 | +357/-15 | 16 |
+| Petar Maymounkov | 6 | +221/-23 | 14 |
+| LexLuthr | 7 | +182/-21 | 14 |
+| Aayush Rajasekaran | 5 | +97/-70 | 33 |
+| Raúl Kripalani | 5 | +87/-45 | 7 |
+| unknown | 1 | +114/-0 | 8 |
+| sti-bot | 44 | +54/-60 | 44 |
+| Aarsh Shah | 2 | +61/-50 | 8 |
+| Lucas Molas | 1 | +74/-27 | 3 |
+| zenground0 | 8 | +80/-18 | 14 |
+| Dirk McCormick | 3 | +52/-33 | 8 |
+| frank | 3 | +73/-7 | 3 |
+| Will Scott | 4 | +45/-11 | 5 |
+| kaola526 | 5 | +44/-11 | 5 |
+| dependabot[bot] | 3 | +16/-10 | 8 |
+| zl | 1 | +15/-4 | 4 |
+| Phi | 5 | +12/-6 | 6 |
+| Marcin Rataj | 1 | +11/-7 | 1 |
+| github-actions[bot] | 7 | +8/-8 | 7 |
+| Anton Evangelatov | 2 | +13/-0 | 4 |
+| Nicolas Gimenez | 1 | +12/-0 | 1 |
+| Marten Seemann | 2 | +5/-7 | 5 |
+| Chris Harden | 1 | +10/-0 | 2 |
+| jennijuju | 1 | +4/-4 | 7 |
+| Travis Person | 2 | +2/-6 | 2 |
+| Rod Vagg | 1 | +3/-3 | 2 |
+| Rob Quist | 1 | +3/-3 | 1 |
+| Jiaying Wang | 1 | +2/-3 | 2 |
+| zengroundumbass | 1 | +3/-1 | 1 |
+| lifei | 1 | +1/-1 | 1 |
+| Mike | 1 | +2/-0 | 1 |
+| Hubert | 1 | +1/-1 | 1 |
+| Daniel N | 1 | +1/-1 | 1 |
+| BMZ | 1 | +1/-1 | 1 |
+
+# 1.16.1 / 2022-07-07
+
+This is an OPTIONAL PATCH releases for storage providers who have failed to publish `SubmitWindowedPoSt` due to out of gas error. The error log looks like `/wdpost_run.go:xxx	estimating gas	{"error": "estimating gas used: message execution failed: exit SysErrOutOfGas(7)...`.
+
+## New Features
+
+- feat: declare fault recovery: Config for maximum partition count per message (#8988 / #8986)
+  - configure `MaxPartitionsPerRecoveryMessage` in miner configuration setting.
+- feat: wdpost: Config for maximum partition count per message (#8982 / #8986)
+  - configure `MaxPartitionsPerPoStMessage` in miner configuration setting.
+  
+# 1.16.0 / 2022-06-24
 
 This is a MANDATORY release of Lotus that introduces [Filecoin network v16,
 codenamed the Skyr upgrade](https://github.com/filecoin-project/community/discussions/74?sort=new#discussioncomment-2392151).
@@ -185,7 +611,7 @@ This is an optional release of lotus that include new APIs, some improvements an
 
 This is a highly recommended feature lotus release v1.15.2. This feature release introduces many new features and  for SPs, including PoSt workers, sealing scheduler, snap deal queue and so on.
 
-Note: You need to be using go v1.17.9&up from this release onwards.
+Note: You need to be using go v1.18.1&up from this release onwards.
 
 ## Highlights
 ### ❣️❣️❣️ PoSt Workers ❣️❣️❣️
@@ -1153,7 +1579,7 @@ storage providers and clients.
 
 ## Highlights
 - 🌟🌟🌟 Introduce Dagstore and CARv2 for deal-making (#6671) ([filecoin-project/lotus#6671](https://github.com/filecoin-project/lotus/pull/6671))
-  - **[lotus miner markets' Dagstore](https://docs.filecoin.io/mine/lotus/dagstore/#conceptual-overview)** is a
+  - **[lotus miner markets' Dagstore](https://lotus.filecoin.io/storage-providers/operate/dagstore/)** is a
     component of the `markets` subsystem in lotus-miner. It is a sharded store to hold large IPLD graphs efficiently,
     packaged as  location-transparent attachable CAR files and it replaces the former Badger staging blockstore. It
     is designed to provide high efficiency and throughput, and minimize resource utilization during deal-making operations.  
@@ -1161,18 +1587,18 @@ storage providers and clients.
     blockstores, which are served as the direct medium for data exchanges in markets for both storage and retrieval
     deal making without requiring intermediate buffers.
   - In the future, lotus will leverage and interact with Dagstore a lot for new features and improvements for deal
-    making, therefore, it's highly recommended to lotus users to go through [Lotus Miner: About the markets dagstore](https://docs.filecoin.io/mine/lotus/dagstore/#conceptual-overview) thoroughly to learn more about Dagstore's
+    making, therefore, it's highly recommended to lotus users to go through [Lotus Miner: About the markets dagstore](https://lotus.filecoin.io/storage-providers/operate/dagstore/) thoroughly to learn more about Dagstore's
     conceptual overview, terminology, directory structure, configuration and so on.
   - **Note**:
     - When you first start your lotus-miner or market subsystem with this release, a one-time/first-time **dagstore migration** will be triggered which replaces the former Badger staging blockstore with dagstore. We highly
-      recommend storage providers to read this [section](https://docs.filecoin.io/mine/lotus/dagstore/#first-time-migration) to learn more about
+      recommend storage providers to read this [section](https://lotus.filecoin.io/storage-providers/operate/dagstore/#first-time-migration) to learn more about
       what the process does, what to expect and how monitor it.
     - It is highly recommended to **wait all ongoing data transfer to finish or cancel inbound storage deals that
       are still transferring**, using the `lotus-miner data-transfers cancel` command before upgrade your market nodes. Reason being that the new dagstore changes attributes in the internal deal state objects, and the paths to the staging CARs where the deal data was being placed will be lost.
     - ‼️Having your dags initialized will become important in the near feature for you to provide a better storage
       and retrieval service. We'd suggest you to start [forced bulk initialization] soon if possible as this process
       places relatively high IP workload on your storage system and is better to be carried out gradually and over a
-      longer timeframe. Read how to do properly perform a force bulk initialization [here](https://docs.filecoin.io/mine/lotus/dagstore/#forcing-bulk-initialization).
+      longer timeframe. Read how to do properly perform a force bulk initialization [here](https://lotus.filecoin.io/storage-providers/operate/dagstore/#forcing-bulk-initialization).
     - ⏮ Rollback Alert(from v1.11.2-rcX to any version lower): If a storages deal is initiated with M1/v1.11.2(-rcX)
       release, it needs to get to the `StorageDealAwaitingPrecommit` state before you can do a version rollback or the markets process may panic.
   - 💙 **Special thanks to [MinerX fellows for testing and providing valuable feedbacks](https://github.com/filecoin-project/lotus/discussions/6852) for Dagstore in the past month!**
@@ -1302,8 +1728,8 @@ Contributors
 This is a  **highly recommended** but optional Lotus v1.11.1 release that introduces many deal making and datastore improvements and new features along with other bug fixes.
 
 ## Highlights
-- ⭐️⭐️⭐️[**lotus-miner market subsystem**](https://docs.filecoin.io/mine/lotus/split-markets-miners/#frontmatter-title) is introduced in this release! It is **highly recommended** for storage providers to run markets processes on a separate machine! Doing so, only this machine needs to exposes public ports for deal making. This also means that the other miner operations can now be completely isolated by from the deal making processes and storage providers can stop and restarts the markets process without affecting an ongoing Winning/Window PoSt!
-  - More details on the concepts, architecture and how to split the market process can be found [here](https://docs.filecoin.io/mine/lotus/split-markets-miners/#concepts).
+- ⭐️⭐️⭐️[**lotus-miner market subsystem**](https://lotus.filecoin.io/storage-providers/advanced-configurations/split-markets-miners/) is introduced in this release! It is **highly recommended** for storage providers to run markets processes on a separate machine! Doing so, only this machine needs to exposes public ports for deal making. This also means that the other miner operations can now be completely isolated by from the deal making processes and storage providers can stop and restarts the markets process without affecting an ongoing Winning/Window PoSt!
+  - More details on the concepts, architecture and how to split the market process can be found [here](https://lotus.filecoin.io/storage-providers/advanced-configurations/split-markets-miners/#concepts).
   - Base on your system setup(running on separate machines, same machine and so on), please see the suggested practice by community members [here](https://github.com/filecoin-project/lotus/discussions/7047#discussion-3515335).
     - Note: if you are running lotus-worker on a different machine, you will need to set `MARKETS_API_INFO` for certain CLI to work properly. This will be improved by #7072.
   - Huge thanks to MinerX fellows for [helping testing the implementation, reporting the issues so they were fixed by now and providing feedbacks](https://github.com/filecoin-project/lotus/discussions/6861) to user docs in the past three weeks!
@@ -1313,7 +1739,7 @@ This is a  **highly recommended** but optional Lotus v1.11.1 release that introd
     - `AvailableBalanceBuffer`: minimum available balance to keep in the miner actor before sending it with messages, default is 0FIL.
     - `DisableCollateralFallback`: whether to send collateral with messages even if there is no available balance in the miner actor, default is `false`.
 - Config for deal publishing control addresses ([filecoin-project/lotus#6697](https://github.com/filecoin-project/lotus/pull/6697))
-  - Set `DealPublishControl` to set the wallet used for sending `PublishStorageDeals` messages, instructions [here](https://docs.filecoin.io/mine/lotus/miner-addresses/#control-addresses).
+  - Set `DealPublishControl` to set the wallet used for sending `PublishStorageDeals` messages, instructions [here](https://lotus.filecoin.io/storage-providers/operate/addresses/#control-addresses).
 - Config UX improvements ([filecoin-project/lotus#6848](https://github.com/filecoin-project/lotus/pull/6848))
   - You can now preview the the default and updated node config by running `lotus/lotus-miner config default/updated`
 
@@ -1742,7 +2168,7 @@ Note that this release is built on top of Lotus v1.9.0. Enterprising users can u
 
 FIPs [0008](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0008.md) and [0013](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0013.md) combine to allow for a significant increase in the rate of onboarding storage on the Filecoin network. This aims to lead to more useful data being stored on the network, reduced network congestion, and lower network base fee.
 
-**Check out the documentation [here](https://docs.filecoin.io/mine/lotus/miner-configuration/#precommitsectorsbatch) for details on the new Lotus miner sealing config options, [here](https://docs.filecoin.io/mine/lotus/miner-configuration/#fees-section) for fee config options, and explanations of the new features.**
+**Check out the documentation [here]((https://lotus.filecoin.io/storage-providers/advanced-configurations/sealing/#precommitsectorsbatch) for details on the new Lotus miner sealing config options, [here](https://lotus.filecoin.io/storage-providers/setup/configuration/#fees-section) for fee config options, and explanations of the new features.**
 
 Note:
 - We recommend to keep `PreCommitSectorsBatch` as 1.
@@ -1758,7 +2184,7 @@ Given these assumptions:
 
 - We'd expect a network storage growth rate of around 530PiB per day. 😳 🎉 🥳 😅
 - We'd expect network bandwidth dedicated to `SubmitWindowedPoSt` to grow by about 0.02% per day.
-- We'd expect the [state-tree](https://spec.filecoin.io/#section-systems.filecoin_vm.state_tree) (and therefore [snapshot](https://docs.filecoin.io/get-started/lotus/chain/#lightweight-snapshot)) size to grow by 1.16GiB per day.
+- We'd expect the [state-tree](https://spec.filecoin.io/#section-systems.filecoin_vm.state_tree) (and therefore [snapshot](https://lotus.filecoin.io/lotus/manage/chain-management/#lightweight-snapshot)) size to grow by 1.16GiB per day.
   - Nearly all of the state-tree growth is expected to come from new sector metadata.
 - We'd expect the daily lotus datastore growth rate to increase by about 10-15% (from current ~21GiB/day).
   - Most "growth" of the lotus datastore is due to "churn", historical data that's no longer referenced by the latest state-tree.
@@ -1779,7 +2205,7 @@ Included in the HyperDrive upgrade is [FIP-0015](https://github.com/filecoin-pro
 
 - Implement FIP-0015 ([filecoin-project/lotus#6361](https://github.com/filecoin-project/lotus/pull/6361))
 - Integrate FIP0013 and FIP0008 ([filecoin-project/lotus#6235](https://github.com/filecoin-project/lotus/pull/6235))
-  - [Configuration docs and cli examples](https://docs.filecoin.io/mine/lotus/miner-configuration/#precommitsectorsbatch)
+  - [Configuration docs and cli examples](https://lotus.filecoin.io/storage-providers/advanced-configurations/sealing/#precommitsectorsbatch)
   - [cli docs](https://github.com/filecoin-project/lotus/blob/master/documentation/en/cli-lotus-miner.md#lotus-miner-sectors-batching)
   - Introduce gas prices for aggregate verifications ([filecoin-project/lotus#6347](https://github.com/filecoin-project/lotus/pull/6347))
 - Introduce v5 actors ([filecoin-project/lotus#6195](https://github.com/filecoin-project/lotus/pull/6195))
@@ -2097,7 +2523,7 @@ Note that this release does NOT set an upgrade epoch for v3 actors to take effec
 - [#5309](https://github.com/filecoin-project/lotus/pull/5309) Batch multiple deals in one `PublishStorageMessages`
   - [#5411](https://github.com/filecoin-project/lotus/pull/5411) Handle batch `PublishStorageDeals` message in sealing recovery
   - [#5505](https://github.com/filecoin-project/lotus/pull/5505) Exclude expired deals from batching in `PublishStorageDeals` messages
-  - Added `PublishMsgPeriod` and `MaxDealsPerPublishMsg` to miner `Dealmaking` [configuration](https://docs.filecoin.io/mine/lotus/miner-configuration/#dealmaking-section). See how they work [here](https://docs.filecoin.io/mine/lotus/miner-configuration/#publishing-several-deals-in-one-message).
+  - Added `PublishMsgPeriod` and `MaxDealsPerPublishMsg` to miner `Dealmaking` [configuration](https://lotus.filecoin.io/storage-providers/advanced-configurations/market/#dealmaking-section). See how they work [here](https://lotus.filecoin.io/storage-providers/advanced-configurations/market/#publishing-several-deals-in-one-message).
   - [#5538](https://github.com/filecoin-project/lotus/pull/5538), [#5549](https://github.com/filecoin-project/lotus/pull/5549) Added a command to list pending deals and force publish messages.
     - Run `lotus-miner market pending-publish`
   - [#5428](https://github.com/filecoin-project/lotus/pull/5428) Moved waiting for `PublishStorageDeals` messages' receipt from markets to lotus

Dockerfile.lotus

@@ -1,4 +1,4 @@
-FROM golang:1.17.9-buster AS builder-deps
+FROM golang:1.18.1-buster AS builder-deps
 MAINTAINER Lotus Development Team
 
 RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev

Makefile

@@ -8,9 +8,9 @@ unexport GOFLAGS
 GOCC?=go
 
 GOVERSION:=$(shell $(GOCC) version | tr ' ' '\n' | grep go1 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
-ifeq ($(shell expr $(GOVERSION) \< 1016000), 1)
+ifeq ($(shell expr $(GOVERSION) \< 1018001), 1)
 $(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell expr $(GOVERSION) % 1000000 / 1000).$(shell expr $(GOVERSION) % 1000))
-$(error Update Golang to version to at least 1.17.9)
+$(error Update Golang to version to at least 1.18.1)
 endif
 
 # git modules that need to be loaded
@@ -133,6 +133,18 @@ install-worker:
 install-app:
 	install -C ./$(APP) /usr/local/bin/$(APP)
 
+uninstall: uninstall-daemon uninstall-miner uninstall-worker
+.PHONY: uninstall
+
+uninstall-daemon:
+	rm -f /usr/local/bin/lotus
+
+uninstall-miner:
+	rm -f /usr/local/bin/lotus-miner
+
+uninstall-worker:
+	rm -f /usr/local/bin/lotus-worker
+
 # TOOLS
 
 lotus-seed: $(BUILD_DEPS)
@@ -162,8 +174,8 @@ lotus-pond-app: lotus-pond-front lotus-pond
 
 lotus-fountain:
 	rm -f lotus-fountain
-	go build $(GOFLAGS) -o lotus-fountain ./cmd/lotus-fountain
-	go run github.com/GeertJohan/go.rice/rice append --exec lotus-fountain -i ./cmd/lotus-fountain -i ./build
+	$(GOCC) build $(GOFLAGS) -o lotus-fountain ./cmd/lotus-fountain
+	$(GOCC) run github.com/GeertJohan/go.rice/rice append --exec lotus-fountain -i ./cmd/lotus-fountain -i ./build
 .PHONY: lotus-fountain
 BINS+=lotus-fountain
 
@@ -290,11 +302,14 @@ type-gen: api-gen
 method-gen: api-gen
 	(cd ./lotuspond/front/src/chain && $(GOCC) run ./methodgen.go)
 
-actors-gen:
+actors-code-gen:
 	$(GOCC) run ./gen/inline-gen . gen/inlinegen-data.json
 	$(GOCC) run ./chain/actors/agen
 	$(GOCC) fmt ./...
 
+actors-gen: actors-code-gen fiximports
+.PHONY: actors-gen
+
 bundle-gen:
 	$(GOCC) run ./gen/bundle
 	$(GOCC) fmt ./build/...
@@ -319,7 +334,7 @@ appimage: lotus
 	cp ./lotus AppDir/usr/bin/
 	appimage-builder
 
-docsgen: docsgen-md docsgen-openrpc
+docsgen: docsgen-md docsgen-openrpc fiximports
 
 docsgen-md-bin: api-gen actors-gen
 	$(GOCC) build $(GOFLAGS) -o docgen-md ./api/docgen/cmd
@@ -336,7 +351,7 @@ docsgen-md-storage: docsgen-md-bin
 docsgen-md-worker: docsgen-md-bin
 	./docgen-md "api/api_worker.go" "Worker" "api" "./api" > documentation/en/api-v0-methods-worker.md
 
-docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker
+docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker docsgen-openrpc-gateway
 
 docsgen-openrpc-full: docsgen-openrpc-bin
 	./docgen-openrpc "api/api_full.go" "FullNode" "api" "./api" -gzip > build/openrpc/full.json.gz
@@ -344,10 +359,15 @@ docsgen-openrpc-storage: docsgen-openrpc-bin
 	./docgen-openrpc "api/api_storage.go" "StorageMiner" "api" "./api" -gzip > build/openrpc/miner.json.gz
 docsgen-openrpc-worker: docsgen-openrpc-bin
 	./docgen-openrpc "api/api_worker.go" "Worker" "api" "./api" -gzip > build/openrpc/worker.json.gz
+docsgen-openrpc-gateway: docsgen-openrpc-bin
+	./docgen-openrpc "api/api_gateway.go" "Gateway" "api" "./api" -gzip > build/openrpc/gateway.json.gz
 
 .PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin
 
-gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci bundle-gen
+fiximports:
+	./scripts/fiximports
+
+gen: actors-code-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci bundle-gen fiximports
 	@echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO MAKE docsgen-cli"
 .PHONY: gen
 

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <a href="https://docs.filecoin.io/" title="Filecoin Docs">
+  <a href="https://lotus.filecoin.io/" title="Filecoin Docs">
     <img src="documentation/images/lotus_logo_h.png" alt="Project Lotus Logo" width="244" />
   </a>
 </p>
@@ -20,7 +20,7 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more
 
 > Note: The default `master` branch is the dev branch, please use with caution. For the latest stable version, checkout the most recent [`Latest release`](https://github.com/filecoin-project/lotus/releases).
  
-For complete instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/). Basic build instructions can be found further down in this readme.
+For complete instructions on how to build, install and setup lotus, please visit [https://lotus.filecoin.io](https://lotus.filecoin.io/lotus/install/prerequisites/#supported-platforms). Basic build instructions can be found further down in this readme.
 
 ## Reporting a Vulnerability
 
@@ -31,7 +31,7 @@ Please send an email to security@filecoin.org. See our [security policy](SECURIT
 These repos are independent and reusable modules, but are tightly integrated into Lotus to make up a fully featured Filecoin implementation:
 
 - [go-fil-markets](https://github.com/filecoin-project/go-fil-markets) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/markets-shared-components-5daa144a7046a60001c6e253/board)
-- [specs-actors](https://github.com/filecoin-project/specs-actors) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/actors-5ee6f3aa87591f0016c05685/board)
+- [builtin-actors](https://github.com/filecoin-project/builtin-actors)
 
 ## Contribute
 
@@ -64,17 +64,17 @@ sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config cu
 
 Fedora:
 ```
-sudo dnf -y install gcc make git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm wget hwloc libhwloc-dev
+sudo dnf -y install gcc make git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm wget hwloc hwloc-devel
 ```
 
-For other distributions you can find the required dependencies [here.](https://docs.filecoin.io/get-started/lotus/installation/#system-specific) For instructions specific to macOS, you can find them [here.](https://docs.filecoin.io/get-started/lotus/installation/#macos)
+For other distributions you can find the required dependencies [here.](https://lotus.filecoin.io/lotus/install/prerequisites/#supported-platforms) For instructions specific to macOS, you can find them [here.](https://lotus.filecoin.io/lotus/install/macos/)
 
 #### Go
 
-To build Lotus, you need a working installation of [Go 1.17.9 or higher](https://golang.org/dl/):
+To build Lotus, you need a working installation of [Go 1.18.1 or higher](https://golang.org/dl/):
 
 ```bash
-wget -c https://golang.org/dl/go1.17.9.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
+wget -c https://golang.org/dl/go1.18.1.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
 ```
 
 **TIP:**
@@ -101,7 +101,7 @@ Note: The default branch `master` is the dev branch where the latest new feature
 
 2. To join mainnet, checkout the [latest release](https://github.com/filecoin-project/lotus/releases).
 
-   If you are changing networks from a previous Lotus installation or there has been a network reset, read the [Switch networks guide](https://docs.filecoin.io/get-started/lotus/switch-networks/) before proceeding.
+   If you are changing networks from a previous Lotus installation or there has been a network reset, read the [Switch networks guide](https://lotus.filecoin.io/lotus/manage/switch-networks/) before proceeding.
 
    For networks other than mainnet, look up the current branch or tag/commit for the network you want to join in the [Filecoin networks dashboard](https://network.filecoin.io), then build Lotus for your specific network below.
 
@@ -113,8 +113,8 @@ Note: The default branch `master` is the dev branch where the latest new feature
 
    Currently, the latest code on the _master_ branch corresponds to mainnet.
 
-3. If you are in China, see "[Lotus: tips when running in China](https://docs.filecoin.io/get-started/lotus/tips-running-in-china/)".
-4. This build instruction uses the prebuilt proofs binaries. If you want to build the proof binaries from source check the [complete instructions](https://docs.filecoin.io/get-started/lotus/installation/#build-and-install-lotus). Note, if you are building the proof binaries from source, [installing rustup](https://docs.filecoin.io/get-started/lotus/installation/#rustup) is also needed.
+3. If you are in China, see "[Lotus: tips when running in China](https://lotus.filecoin.io/lotus/configure/nodes-in-china/)".
+4. This build instruction uses the prebuilt proofs binaries. If you want to build the proof binaries from source check the [complete instructions](https://lotus.filecoin.io/lotus/install/prerequisites/). Note, if you are building the proof binaries from source, [installing rustup](https://lotus.filecoin.io/lotus/install/linux/#rustup) is also needed.
 
 5. Build and install Lotus:
 
@@ -129,9 +129,9 @@ Note: The default branch `master` is the dev branch where the latest new feature
 
    This will put `lotus`, `lotus-miner` and `lotus-worker` in `/usr/local/bin`.
 
-   `lotus` will use the `$HOME/.lotus` folder by default for storage (configuration, chain data, wallets, etc). See [advanced options](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/) for information on how to customize the Lotus folder.
+   `lotus` will use the `$HOME/.lotus` folder by default for storage (configuration, chain data, wallets, etc). See [advanced options](https://lotus.filecoin.io/lotus/configure/defaults/#environment-variables) for information on how to customize the Lotus folder.
 
-6. You should now have Lotus installed. You can now [start the Lotus daemon and sync the chain](https://docs.filecoin.io/get-started/lotus/installation/#start-the-lotus-daemon-and-sync-the-chain).
+6. You should now have Lotus installed. You can now [start the Lotus daemon and sync the chain](https://lotus.filecoin.io/lotus/install/linux/#start-the-lotus-daemon-and-sync-the-chain).
 
 ## License
 

api/api_common.go

@@ -4,12 +4,12 @@ import (
 	"context"
 	"fmt"
 
-	apitypes "github.com/filecoin-project/lotus/api/types"
-	"github.com/filecoin-project/lotus/journal/alerting"
-
 	"github.com/google/uuid"
 
 	"github.com/filecoin-project/go-jsonrpc/auth"
+
+	apitypes "github.com/filecoin-project/lotus/api/types"
+	"github.com/filecoin-project/lotus/journal/alerting"
 )
 
 //                       MODIFYING THE API INTERFACE

api/api_full.go

@@ -6,26 +6,24 @@ import (
 	"fmt"
 	"time"
 
-	abinetwork "github.com/filecoin-project/go-state-types/network"
-
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
-	"github.com/libp2p/go-libp2p-core/peer"
+	"github.com/libp2p/go-libp2p/core/peer"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/go-state-types/crypto"
-	"github.com/filecoin-project/go-state-types/dline"
-
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
-
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/builtin/v8/market"
-	"github.com/filecoin-project/go-state-types/builtin/v8/miner"
 	"github.com/filecoin-project/go-state-types/builtin/v8/paych"
+	"github.com/filecoin-project/go-state-types/builtin/v9/miner"
+	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/go-state-types/dline"
+	abinetwork "github.com/filecoin-project/go-state-types/network"
+
 	apitypes "github.com/filecoin-project/lotus/api/types"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
@@ -171,6 +169,10 @@ type FullNode interface {
 	// If oldmsgskip is set, messages from before the requested roots are also not included.
 	ChainExport(ctx context.Context, nroots abi.ChainEpoch, oldmsgskip bool, tsk types.TipSetKey) (<-chan []byte, error) //perm:read
 
+	// ChainPrune prunes the stored chain state and garbage collects; only supported if you
+	// are using the splitstore
+	ChainPrune(ctx context.Context, opts PruneOpts) error //perm:admin
+
 	// ChainCheckBlockstore performs an (asynchronous) health check on the chain/state blockstore
 	// if supported by the underlying implementation.
 	ChainCheckBlockstore(context.Context) error //perm:admin
@@ -404,7 +406,7 @@ type FullNode interface {
 	StateCall(context.Context, *types.Message, types.TipSetKey) (*InvocResult, error) //perm:read
 	// StateReplay replays a given message, assuming it was included in a block in the specified tipset.
 	//
-	// If a tipset key is provided, and a replacing message is found on chain,
+	// If a tipset key is provided, and a replacing message is not found on chain,
 	// the method will return an error saying that the message wasn't found
 	//
 	// If no tipset key is provided, the appropriate tipset is looked up, and if
@@ -460,10 +462,15 @@ type FullNode interface {
 	StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) //perm:read
 	// StateMinerAvailableBalance returns the portion of a miner's balance that can be withdrawn or spent
 	StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) //perm:read
-	// StateMinerSectorAllocated checks if a sector is allocated
+	// StateMinerSectorAllocated checks if a sector number is marked as allocated.
 	StateMinerSectorAllocated(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (bool, error) //perm:read
-	// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector
-	StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) //perm:read
+	// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector.
+	// Returns nil and no error if the sector isn't precommitted.
+	//
+	// Note that the sector number may be allocated while PreCommitInfo is nil. This means that either allocated sector
+	// numbers were compacted, and the sector number was marked as allocated in order to reduce size of the allocated
+	// sectors bitfield, or that the sector was precommitted, but the precommit has expired.
+	StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorPreCommitOnChainInfo, error) //perm:read
 	// StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found
 	// NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate
 	// expiration epoch
@@ -520,6 +527,8 @@ type FullNode interface {
 	StateMarketDeals(context.Context, types.TipSetKey) (map[string]*MarketDeal, error) //perm:read
 	// StateMarketStorageDeal returns information about the indicated deal
 	StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*MarketDeal, error) //perm:read
+	// StateComputeDataCID computes DataCID from a set of on-chain deals
+	StateComputeDataCID(ctx context.Context, maddr address.Address, sectorType abi.RegisteredSealProof, deals []abi.DealID, tsk types.TipSetKey) (cid.Cid, error) //perm:read
 	// StateLookupID retrieves the ID address of the given address
 	StateLookupID(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
 	// StateAccountKey returns the public key address of the given ID address for secp and bls accounts
@@ -531,6 +540,8 @@ type FullNode interface {
 	StateChangedActors(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) //perm:read
 	// StateMinerSectorCount returns the number of sectors in a miner's sector set and proving set
 	StateMinerSectorCount(context.Context, address.Address, types.TipSetKey) (MinerSectors, error) //perm:read
+	// StateMinerAllocated returns a bitfield containing all sector numbers marked as allocated in miner state
+	StateMinerAllocated(context.Context, address.Address, types.TipSetKey) (*bitfield.BitField, error) //perm:read
 	// StateCompute is a flexible command that applies the given messages on the given tipset.
 	// The messages are run as though the VM were at the provided height.
 	//
@@ -572,7 +583,7 @@ type FullNode interface {
 	// Returns nil if there is no entry in the data cap table for the
 	// address.
 	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) //perm:read
-	// StateVerifiedClientStatus returns the address of the Verified Registry's root key
+	// StateVerifiedRegistryRootKey returns the address of the Verified Registry's root key
 	StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) //perm:read
 	// StateDealProviderCollateralBounds returns the min and max collateral a storage provider
 	// can issue. It takes the deal size and verified status as parameters.
@@ -588,6 +599,8 @@ type FullNode interface {
 	StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) //perm:read
 	// StateActorCodeCIDs returns the CIDs of all the builtin actors for the given network version
 	StateActorCodeCIDs(context.Context, abinetwork.Version) (map[string]cid.Cid, error) //perm:read
+	// StateActorManifestCID returns the CID of the builtin actors manifest for the given network version
+	StateActorManifestCID(context.Context, abinetwork.Version) (cid.Cid, error) //perm:read
 
 	// StateGetRandomnessFromTickets is used to sample the chain for randomness.
 	StateGetRandomnessFromTickets(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
@@ -1214,3 +1227,8 @@ type MsigTransaction struct {
 
 	Approved []address.Address
 }
+
+type PruneOpts struct {
+	MovingGC    bool
+	RetainState int64
+}

api/api_gateway.go

@@ -8,9 +8,9 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/builtin/v9/miner"
 	"github.com/filecoin-project/go-state-types/dline"
 
-	"github.com/filecoin-project/go-state-types/builtin/v8/miner"
 	apitypes "github.com/filecoin-project/lotus/api/types"
 	"github.com/filecoin-project/lotus/chain/types"
 )
@@ -68,4 +68,5 @@ type Gateway interface {
 	StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
 	WalletBalance(context.Context, address.Address) (types.BigInt, error)
 	Version(context.Context) (APIVersion, error)
+	Discover(context.Context) (apitypes.OpenRPCDocument, error)
 }

api/api_net.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"time"
 
-	metrics "github.com/libp2p/go-libp2p-core/metrics"
-	"github.com/libp2p/go-libp2p-core/network"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"github.com/libp2p/go-libp2p-core/protocol"
+	"github.com/libp2p/go-libp2p/core/metrics"
+	"github.com/libp2p/go-libp2p/core/network"
+	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/libp2p/go-libp2p/core/protocol"
 )
 
 //                       MODIFYING THE API INTERFACE

api/api_storage.go

@@ -5,28 +5,28 @@ import (
 	"context"
 	"time"
 
-	"github.com/filecoin-project/go-state-types/builtin/v8/miner"
-	"github.com/filecoin-project/lotus/chain/actors/builtin"
-
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
-	"github.com/libp2p/go-libp2p-core/peer"
+	"github.com/libp2p/go-libp2p/core/peer"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-bitfield"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/builtin/v8/market"
-	abinetwork "github.com/filecoin-project/go-state-types/network"
-	"github.com/filecoin-project/specs-storage/storage"
-
 	"github.com/filecoin-project/go-fil-markets/piecestore"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/filecoin-project/go-jsonrpc/auth"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/builtin/v8/market"
+	"github.com/filecoin-project/go-state-types/builtin/v9/miner"
+	abinetwork "github.com/filecoin-project/go-state-types/network"
 
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-	"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
+	"github.com/filecoin-project/lotus/storage/pipeline/sealiface"
+	"github.com/filecoin-project/lotus/storage/sealer/fsutil"
+	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
 //                       MODIFYING THE API INTERFACE
@@ -50,11 +50,21 @@ type StorageMiner interface {
 	ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error) //perm:read
 	ActorAddressConfig(ctx context.Context) (AddressConfig, error)            //perm:read
 
+	// WithdrawBalance allows to withdraw balance from miner actor to owner address
+	// Specify amount as "0" to withdraw full balance. This method returns a message CID
+	// and does not wait for message execution
+	ActorWithdrawBalance(ctx context.Context, amount abi.TokenAmount) (cid.Cid, error) //perm:admin
+
+	// BeneficiaryWithdrawBalance allows the beneficiary of a miner to withdraw balance from miner actor
+	// Specify amount as "0" to withdraw full balance. This method returns a message CID
+	// and does not wait for message execution
+	BeneficiaryWithdrawBalance(context.Context, abi.TokenAmount) (cid.Cid, error) //perm:admin
+
 	MiningBase(context.Context) (*types.TipSet, error) //perm:read
 
 	ComputeWindowPoSt(ctx context.Context, dlIdx uint64, tsk types.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) //perm:admin
 
-	ComputeDataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) //perm:admin
+	ComputeDataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (abi.PieceInfo, error) //perm:admin
 
 	// Temp api for testing
 	PledgeSector(context.Context) (abi.SectorID, error) //perm:write
@@ -65,9 +75,9 @@ type StorageMiner interface {
 	// Add piece to an open sector. If no sectors with enough space are open,
 	// either a new sector will be created, or this call will block until more
 	// sectors can be created.
-	SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storage.Data, d PieceDealInfo) (SectorOffset, error) //perm:admin
+	SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storiface.Data, d PieceDealInfo) (SectorOffset, error) //perm:admin
 
-	SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error //perm:admin
+	SectorsUnsealPiece(ctx context.Context, sector storiface.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error //perm:admin
 
 	// List all staged sectors
 	SectorsList(context.Context) ([]abi.SectorNumber, error) //perm:read
@@ -120,50 +130,88 @@ type StorageMiner interface {
 	// SectorAbortUpgrade can be called on sectors that are in the process of being upgraded to abort it
 	SectorAbortUpgrade(context.Context, abi.SectorNumber) error //perm:admin
 
+	// SectorNumAssignerMeta returns sector number assigner metadata - reserved/allocated
+	SectorNumAssignerMeta(ctx context.Context) (NumAssignerMeta, error) //perm:read
+	// SectorNumReservations returns a list of sector number reservations
+	SectorNumReservations(ctx context.Context) (map[string]bitfield.BitField, error) //perm:read
+	// SectorNumReserve creates a new sector number reservation. Will fail if any other reservation has colliding
+	// numbers or name. Set force to true to override safety checks.
+	// Valid characters for name: a-z, A-Z, 0-9, _, -
+	SectorNumReserve(ctx context.Context, name string, sectors bitfield.BitField, force bool) error //perm:admin
+	// SectorNumReserveCount creates a new sector number reservation for `count` sector numbers.
+	// by default lotus will allocate lowest-available sector numbers to the reservation.
+	// For restrictions on `name` see SectorNumReserve
+	SectorNumReserveCount(ctx context.Context, name string, count uint64) (bitfield.BitField, error) //perm:admin
+	// SectorNumFree drops a sector reservation
+	SectorNumFree(ctx context.Context, name string) error //perm:admin
+
+	SectorReceive(ctx context.Context, meta RemoteSectorMeta) error //perm:admin
+
 	// WorkerConnect tells the node to connect to workers RPC
 	WorkerConnect(context.Context, string) error                              //perm:admin retry:true
 	WorkerStats(context.Context) (map[uuid.UUID]storiface.WorkerStats, error) //perm:admin
 	WorkerJobs(context.Context) (map[uuid.UUID][]storiface.WorkerJob, error)  //perm:admin
 
 	//storiface.WorkerReturn
-	ReturnDataCid(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error                                       //perm:admin retry:true
-	ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error                                      //perm:admin retry:true
-	ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storage.PreCommit1Out, err *storiface.CallError) error                       //perm:admin retry:true
-	ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storage.SectorCids, err *storiface.CallError) error                       //perm:admin retry:true
-	ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storage.Commit1Out, err *storiface.CallError) error                             //perm:admin retry:true
-	ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storage.Proof, err *storiface.CallError) error                                //perm:admin retry:true
-	ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                  //perm:admin retry:true
-	ReturnReplicaUpdate(ctx context.Context, callID storiface.CallID, out storage.ReplicaUpdateOut, err *storiface.CallError) error                     //perm:admin retry:true
-	ReturnProveReplicaUpdate1(ctx context.Context, callID storiface.CallID, vanillaProofs storage.ReplicaVanillaProofs, err *storiface.CallError) error //perm:admin retry:true
-	ReturnProveReplicaUpdate2(ctx context.Context, callID storiface.CallID, proof storage.ReplicaUpdateProof, err *storiface.CallError) error           //perm:admin retry:true
-	ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                       //perm:admin retry:true
-	ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                           //perm:admin retry:true
-	ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                 //perm:admin retry:true
-	ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                     //perm:admin retry:true
-	ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                     //perm:admin retry:true
-	ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err *storiface.CallError) error                                              //perm:admin retry:true
-	ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                           //perm:admin retry:true
+	ReturnDataCid(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error                                         //perm:admin retry:true
+	ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error                                        //perm:admin retry:true
+	ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storiface.PreCommit1Out, err *storiface.CallError) error                       //perm:admin retry:true
+	ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storiface.SectorCids, err *storiface.CallError) error                       //perm:admin retry:true
+	ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storiface.Commit1Out, err *storiface.CallError) error                             //perm:admin retry:true
+	ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storiface.Proof, err *storiface.CallError) error                                //perm:admin retry:true
+	ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                    //perm:admin retry:true
+	ReturnReplicaUpdate(ctx context.Context, callID storiface.CallID, out storiface.ReplicaUpdateOut, err *storiface.CallError) error                     //perm:admin retry:true
+	ReturnProveReplicaUpdate1(ctx context.Context, callID storiface.CallID, vanillaProofs storiface.ReplicaVanillaProofs, err *storiface.CallError) error //perm:admin retry:true
+	ReturnProveReplicaUpdate2(ctx context.Context, callID storiface.CallID, proof storiface.ReplicaUpdateProof, err *storiface.CallError) error           //perm:admin retry:true
+	ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                         //perm:admin retry:true
+	ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                             //perm:admin retry:true
+	ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                   //perm:admin retry:true
+	ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                       //perm:admin retry:true
+	ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                       //perm:admin retry:true
+	ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err *storiface.CallError) error                                                //perm:admin retry:true
+	ReturnDownloadSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                    //perm:admin retry:true
+	ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                             //perm:admin retry:true
 
 	// SealingSchedDiag dumps internal sealing scheduler state
 	SealingSchedDiag(ctx context.Context, doSched bool) (interface{}, error) //perm:admin
 	SealingAbort(ctx context.Context, call storiface.CallID) error           //perm:admin
+	//SealingSchedRemove removes a request from sealing pipeline
+	SealingRemoveRequest(ctx context.Context, schedId uuid.UUID) error //perm:admin
 
-	// SectorIndex
-	StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error                                                                                             //perm:admin
-	StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error)                                                                                              //perm:admin
-	StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error                                                                                       //perm:admin
-	StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error                                     //perm:admin
-	StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error                                                      //perm:admin
+	// paths.SectorIndex
+	StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error                                                         //perm:admin
+	StorageDetach(ctx context.Context, id storiface.ID, url string) error                                                              //perm:admin
+	StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error)                                                          //perm:admin
+	StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error                                                   //perm:admin
+	StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error //perm:admin
+	StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error                  //perm:admin
+	// StorageFindSector returns list of paths where the specified sector files exist.
+	//
+	// If allowFetch is set, list of paths to which the sector can be fetched will also be returned.
+	// - Paths which have sector files locally (don't require fetching) will be listed first.
+	// - Paths which have sector files locally will not be filtered based on based on AllowTypes/DenyTypes.
+	// - Paths which require fetching will be filtered based on AllowTypes/DenyTypes. If multiple
+	//   file types are specified, each type will be considered individually, and a union of all paths
+	//   which can accommodate each file type will be returned.
 	StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error) //perm:admin
-	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error)           //perm:admin
-	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                                             //perm:admin
-	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                                  //perm:admin
-	StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error)                                                                                            //perm:admin
-	StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error)                                                                                                    //perm:admin
+	// StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.
+	// Paths with more weight and more % of free space are preferred.
+	// Note: This method doesn't filter paths based on AllowTypes/DenyTypes.
+	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) //perm:admin
+	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                                   //perm:admin
+	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                        //perm:admin
+	StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error)                                                                                  //perm:admin
+	StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error)                                                                                          //perm:admin
 
 	StorageLocal(ctx context.Context) (map[storiface.ID]string, error)       //perm:admin
 	StorageStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error) //perm:admin
 
+	StorageAuthVerify(ctx context.Context, token string) ([]auth.Permission, error) //perm:read
+
+	StorageAddLocal(ctx context.Context, path string) error                              //perm:admin
+	StorageDetachLocal(ctx context.Context, path string) error                           //perm:admin
+	StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error //perm:admin
+
 	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error                                                                                                        //perm:write
 	MarketListDeals(ctx context.Context) ([]*MarketDeal, error)                                                                                                                          //perm:read
 	MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error)                                                                                           //perm:read
@@ -227,6 +275,9 @@ type StorageMiner interface {
 	// DagstoreGC runs garbage collection on the DAG store.
 	DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin
 
+	// DagstoreRegisterShard registers a shard manually with dagstore with given pieceCID
+	DagstoreRegisterShard(ctx context.Context, key string) error //perm:admin
+
 	// IndexerAnnounceDeal informs indexer nodes that a new deal was received,
 	// so they can download its index
 	IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin
@@ -258,8 +309,6 @@ type StorageMiner interface {
 	DealsConsiderUnverifiedStorageDeals(context.Context) (bool, error)           //perm:admin
 	DealsSetConsiderUnverifiedStorageDeals(context.Context, bool) error          //perm:admin
 
-	StorageAddLocal(ctx context.Context, path string) error //perm:admin
-
 	PiecesListPieces(ctx context.Context) ([]cid.Cid, error)                                 //perm:read
 	PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error)                               //perm:read
 	PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) //perm:read
@@ -271,9 +320,14 @@ type StorageMiner interface {
 	// the path specified when calling CreateBackup is within the base path
 	CreateBackup(ctx context.Context, fpath string) error //perm:admin
 
-	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
+	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storiface.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
 
 	ComputeProof(ctx context.Context, ssi []builtin.ExtendedSectorInfo, rand abi.PoStRandomness, poStEpoch abi.ChainEpoch, nv abinetwork.Version) ([]builtin.PoStProof, error) //perm:read
+
+	// RecoverFault can be used to declare recoveries manually. It sends messages
+	// to the miner actor with details of recovered sectors and returns the CID of messages. It honors the
+	// maxPartitionsPerRecoveryMessage from the config
+	RecoverFault(ctx context.Context, sectors []abi.SectorNumber) ([]cid.Cid, error) //perm:admin
 }
 
 var _ storiface.WorkerReturn = *new(StorageMiner)
@@ -444,3 +498,119 @@ type DagstoreInitializeAllEvent struct {
 	Total   int
 	Current int
 }
+
+type NumAssignerMeta struct {
+	Reserved  bitfield.BitField
+	Allocated bitfield.BitField
+
+	// ChainAllocated+Reserved+Allocated
+	InUse bitfield.BitField
+
+	Next abi.SectorNumber
+}
+
+type RemoteSectorMeta struct {
+	////////
+	// BASIC SECTOR INFORMATION
+
+	// State specifies the first state the sector will enter after being imported
+	// Must be one of the following states:
+	// * Packing
+	// * GetTicket
+	// * PreCommitting
+	// * SubmitCommit
+	// * Proving/Available
+	State SectorState
+
+	Sector abi.SectorID
+	Type   abi.RegisteredSealProof
+
+	////////
+	// SEALING METADATA
+	// (allows lotus to continue the sealing process)
+
+	// Required in Packing and later
+	Pieces []SectorPiece // todo better type?
+
+	// Required in PreCommitting and later
+	TicketValue   abi.SealRandomness
+	TicketEpoch   abi.ChainEpoch
+	PreCommit1Out storiface.PreCommit1Out // todo specify better
+
+	CommD *cid.Cid
+	CommR *cid.Cid // SectorKey
+
+	// Required in SubmitCommit and later
+	PreCommitInfo    *miner.SectorPreCommitInfo
+	PreCommitDeposit *big.Int
+	PreCommitMessage *cid.Cid
+	PreCommitTipSet  types.TipSetKey
+
+	SeedValue abi.InteractiveSealRandomness
+	SeedEpoch abi.ChainEpoch
+
+	CommitProof []byte
+
+	// Required in Proving/Available
+	CommitMessage *cid.Cid
+
+	// Optional sector metadata to import
+	Log []SectorLog
+
+	////////
+	// SECTOR DATA SOURCE
+
+	// Sector urls - lotus will use those for fetching files into local storage
+
+	// Required in all states
+	DataUnsealed *storiface.SectorLocation
+
+	// Required in PreCommitting and later
+	DataSealed *storiface.SectorLocation
+	DataCache  *storiface.SectorLocation
+
+	////////
+	// SEALING SERVICE HOOKS
+
+	// URL
+	// RemoteCommit1Endpoint is an URL of POST endpoint which lotus will call requesting Commit1 (seal_commit_phase1)
+	// request body will be json-serialized RemoteCommit1Params struct
+	RemoteCommit1Endpoint string
+
+	// RemoteCommit2Endpoint is an URL of POST endpoint which lotus will call requesting Commit2 (seal_commit_phase2)
+	// request body will be json-serialized RemoteCommit2Params struct
+	RemoteCommit2Endpoint string
+
+	// RemoteSealingDoneEndpoint is called after the sector exists the sealing pipeline
+	// request body will be json-serialized RemoteSealingDoneParams struct
+	RemoteSealingDoneEndpoint string
+}
+
+type RemoteCommit1Params struct {
+	Ticket, Seed []byte
+
+	Unsealed cid.Cid
+	Sealed   cid.Cid
+
+	ProofType abi.RegisteredSealProof
+}
+
+type RemoteCommit2Params struct {
+	Sector    abi.SectorID
+	ProofType abi.RegisteredSealProof
+
+	// todo spec better
+	Commit1Out storiface.Commit1Out
+}
+
+type RemoteSealingDoneParams struct {
+	// Successful is true if the sector has entered state considered as "successfully sealed"
+	Successful bool
+
+	// State is the state the sector has entered
+	// For example "Proving" / "Removing"
+	State string
+
+	// Optional commit message CID
+	CommitMessage *cid.Cid
+}

api/api_test.go

@@ -1,4 +1,4 @@
-//stm: #unit
+// stm: #unit
 package api
 
 import (

api/api_worker.go

@@ -3,15 +3,14 @@ package api
 import (
 	"context"
 
-	"github.com/filecoin-project/go-state-types/proof"
-
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-	"github.com/filecoin-project/specs-storage/storage"
+	"github.com/filecoin-project/go-state-types/proof"
+
+	"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
+	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
 //                       MODIFYING THE API INTERFACE
@@ -34,22 +33,23 @@ type Worker interface {
 	Info(context.Context) (storiface.WorkerInfo, error)                 //perm:admin
 
 	// storiface.WorkerCalls
-	DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (storiface.CallID, error)                                                                                      //perm:admin
-	AddPiece(ctx context.Context, sector storage.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (storiface.CallID, error)                    //perm:admin
-	SealPreCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error)                                                           //perm:admin
-	SealPreCommit2(ctx context.Context, sector storage.SectorRef, pc1o storage.PreCommit1Out) (storiface.CallID, error)                                                                                  //perm:admin
-	SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error) //perm:admin
-	SealCommit2(ctx context.Context, sector storage.SectorRef, c1o storage.Commit1Out) (storiface.CallID, error)                                                                                         //perm:admin
-	FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                                //perm:admin
-	FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                         //perm:admin
-	ReplicaUpdate(ctx context.Context, sector storage.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error)                                                                                       //perm:admin
-	ProveReplicaUpdate1(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error)                                                              //perm:admin
-	ProveReplicaUpdate2(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storage.ReplicaVanillaProofs) (storiface.CallID, error)                  //perm:admin
-	GenerateSectorKeyFromData(ctx context.Context, sector storage.SectorRef, commD cid.Cid) (storiface.CallID, error)                                                                                    //perm:admin
-	ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) (storiface.CallID, error)                                                                                 //perm:admin
-	MoveStorage(ctx context.Context, sector storage.SectorRef, types storiface.SectorFileType) (storiface.CallID, error)                                                                                 //perm:admin
-	UnsealPiece(context.Context, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error)                                           //perm:admin
-	Fetch(context.Context, storage.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error)                                                             //perm:admin
+	DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (storiface.CallID, error)                                                                                        //perm:admin
+	AddPiece(ctx context.Context, sector storiface.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (storiface.CallID, error)                    //perm:admin
+	SealPreCommit1(ctx context.Context, sector storiface.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error)                                                             //perm:admin
+	SealPreCommit2(ctx context.Context, sector storiface.SectorRef, pc1o storiface.PreCommit1Out) (storiface.CallID, error)                                                                                  //perm:admin
+	SealCommit1(ctx context.Context, sector storiface.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storiface.SectorCids) (storiface.CallID, error) //perm:admin
+	SealCommit2(ctx context.Context, sector storiface.SectorRef, c1o storiface.Commit1Out) (storiface.CallID, error)                                                                                         //perm:admin
+	FinalizeSector(ctx context.Context, sector storiface.SectorRef, keepUnsealed []storiface.Range) (storiface.CallID, error)                                                                                //perm:admin
+	FinalizeReplicaUpdate(ctx context.Context, sector storiface.SectorRef, keepUnsealed []storiface.Range) (storiface.CallID, error)                                                                         //perm:admin
+	ReplicaUpdate(ctx context.Context, sector storiface.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error)                                                                                         //perm:admin
+	ProveReplicaUpdate1(ctx context.Context, sector storiface.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error)                                                                //perm:admin
+	ProveReplicaUpdate2(ctx context.Context, sector storiface.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storiface.ReplicaVanillaProofs) (storiface.CallID, error)                  //perm:admin
+	GenerateSectorKeyFromData(ctx context.Context, sector storiface.SectorRef, commD cid.Cid) (storiface.CallID, error)                                                                                      //perm:admin
+	ReleaseUnsealed(ctx context.Context, sector storiface.SectorRef, safeToFree []storiface.Range) (storiface.CallID, error)                                                                                 //perm:admin
+	MoveStorage(ctx context.Context, sector storiface.SectorRef, types storiface.SectorFileType) (storiface.CallID, error)                                                                                   //perm:admin
+	UnsealPiece(context.Context, storiface.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error)                                             //perm:admin
+	Fetch(context.Context, storiface.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error)                                                               //perm:admin
+	DownloadSectorData(ctx context.Context, sector storiface.SectorRef, finalized bool, src map[storiface.SectorFileType]storiface.SectorLocation) (storiface.CallID, error)                                 //perm:admin
 
 	GenerateWinningPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, randomness abi.PoStRandomness) ([]proof.PoStProof, error)                           //perm:admin
 	GenerateWindowPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, partitionIdx int, randomness abi.PoStRandomness) (storiface.WindowPoStResult, error) //perm:admin
@@ -60,7 +60,11 @@ type Worker interface {
 	// Storage / Other
 	Remove(ctx context.Context, sector abi.SectorID) error //perm:admin
 
-	StorageAddLocal(ctx context.Context, path string) error //perm:admin
+	StorageLocal(ctx context.Context) (map[storiface.ID]string, error)                   //perm:admin
+	StorageAddLocal(ctx context.Context, path string) error                              //perm:admin
+	StorageDetachLocal(ctx context.Context, path string) error                           //perm:admin
+	StorageDetachAll(ctx context.Context) error                                          //perm:admin
+	StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error //perm:admin
 
 	// SetEnabled marks the worker as enabled/disabled. Not that this setting
 	// may take a few seconds to propagate to task scheduler
@@ -77,6 +81,10 @@ type Worker interface {
 
 	// Like ProcessSession, but returns an error when worker is disabled
 	Session(context.Context) (uuid.UUID, error) //perm:admin
+
+	// Trigger shutdown
+	Shutdown(context.Context) error //perm:admin
+
 }
 
 var _ storiface.WorkerCalls = *new(Worker)

api/cbor_gen.go

@@ -8,12 +8,13 @@ import (
 	"math"
 	"sort"
 
-	abi "github.com/filecoin-project/go-state-types/abi"
-	market "github.com/filecoin-project/go-state-types/builtin/v8/market"
-	paych "github.com/filecoin-project/go-state-types/builtin/v8/paych"
 	cid "github.com/ipfs/go-cid"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	xerrors "golang.org/x/xerrors"
+
+	abi "github.com/filecoin-project/go-state-types/abi"
+	market "github.com/filecoin-project/go-state-types/builtin/v8/market"
+	paych "github.com/filecoin-project/go-state-types/builtin/v8/paych"
 )
 
 var _ = xerrors.Errorf
@@ -1004,6 +1005,129 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) (err error) {
 
 	return nil
 }
+func (t *SectorPiece) MarshalCBOR(w io.Writer) error {
+	if t == nil {
+		_, err := w.Write(cbg.CborNull)
+		return err
+	}
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{162}); err != nil {
+		return err
+	}
+
+	// t.Piece (abi.PieceInfo) (struct)
+	if len("Piece") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"Piece\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Piece"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("Piece")); err != nil {
+		return err
+	}
+
+	if err := t.Piece.MarshalCBOR(cw); err != nil {
+		return err
+	}
+
+	// t.DealInfo (api.PieceDealInfo) (struct)
+	if len("DealInfo") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"DealInfo\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("DealInfo"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("DealInfo")); err != nil {
+		return err
+	}
+
+	if err := t.DealInfo.MarshalCBOR(cw); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (t *SectorPiece) UnmarshalCBOR(r io.Reader) (err error) {
+	*t = SectorPiece{}
+
+	cr := cbg.NewCborReader(r)
+
+	maj, extra, err := cr.ReadHeader()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
+	if maj != cbg.MajMap {
+		return fmt.Errorf("cbor input should be of type map")
+	}
+
+	if extra > cbg.MaxLength {
+		return fmt.Errorf("SectorPiece: map struct too large (%d)", extra)
+	}
+
+	var name string
+	n := extra
+
+	for i := uint64(0); i < n; i++ {
+
+		{
+			sval, err := cbg.ReadString(cr)
+			if err != nil {
+				return err
+			}
+
+			name = string(sval)
+		}
+
+		switch name {
+		// t.Piece (abi.PieceInfo) (struct)
+		case "Piece":
+
+			{
+
+				if err := t.Piece.UnmarshalCBOR(cr); err != nil {
+					return xerrors.Errorf("unmarshaling t.Piece: %w", err)
+				}
+
+			}
+			// t.DealInfo (api.PieceDealInfo) (struct)
+		case "DealInfo":
+
+			{
+
+				b, err := cr.ReadByte()
+				if err != nil {
+					return err
+				}
+				if b != cbg.CborNull[0] {
+					if err := cr.UnreadByte(); err != nil {
+						return err
+					}
+					t.DealInfo = new(PieceDealInfo)
+					if err := t.DealInfo.UnmarshalCBOR(cr); err != nil {
+						return xerrors.Errorf("unmarshaling t.DealInfo pointer: %w", err)
+					}
+				}
+
+			}
+
+		default:
+			// Field doesn't exist on this type, so ignore it
+			cbg.ScanForLinks(r, func(cid.Cid) {})
+		}
+	}
+
+	return nil
+}
 func (t *DealSchedule) MarshalCBOR(w io.Writer) error {
 	if t == nil {
 		_, err := w.Write(cbg.CborNull)

api/docgen-openrpc/cmd/docgen_openrpc.go

@@ -8,7 +8,6 @@ import (
 	"os"
 
 	"github.com/filecoin-project/lotus/api/docgen"
-
 	docgen_openrpc "github.com/filecoin-project/lotus/api/docgen-openrpc"
 )
 

api/docgen-openrpc/openrpc.go

@@ -8,10 +8,11 @@ import (
 
 	"github.com/alecthomas/jsonschema"
 	go_openrpc_reflect "github.com/etclabscore/go-openrpc-reflect"
-	"github.com/filecoin-project/lotus/api/docgen"
-	"github.com/filecoin-project/lotus/build"
 	"github.com/ipfs/go-cid"
 	meta_schema "github.com/open-rpc/meta-schema"
+
+	"github.com/filecoin-project/lotus/api/docgen"
+	"github.com/filecoin-project/lotus/build"
 )
 
 // schemaDictEntry represents a type association passed to the jsonschema reflector.

api/docgen/docgen.go

@@ -6,31 +6,31 @@ import (
 	"go/ast"
 	"go/parser"
 	"go/token"
+	"net/http"
 	"path/filepath"
 	"reflect"
 	"strings"
 	"time"
 	"unicode"
 
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-bitfield"
 	"github.com/google/uuid"
+	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-graphsync"
-	"github.com/libp2p/go-libp2p-core/metrics"
-	"github.com/libp2p/go-libp2p-core/network"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"github.com/libp2p/go-libp2p-core/protocol"
+	textselector "github.com/ipld/go-ipld-selector-text-lite"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	"github.com/libp2p/go-libp2p/core/metrics"
+	"github.com/libp2p/go-libp2p/core/network"
+	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/libp2p/go-libp2p/core/protocol"
 	"github.com/multiformats/go-multiaddr"
 
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-bitfield"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
-	filestore "github.com/filecoin-project/go-fil-markets/filestore"
+	"github.com/filecoin-project/go-fil-markets/filestore"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-jsonrpc/auth"
-	blocks "github.com/ipfs/go-block-format"
-	textselector "github.com/ipld/go-ipld-selector-text-lite"
-
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/exitcode"
@@ -40,11 +40,11 @@ import (
 	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/repo/imports"
+	sealing "github.com/filecoin-project/lotus/storage/pipeline"
+	"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
+	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
 var ExampleValues = map[reflect.Type]interface{}{
@@ -273,6 +273,8 @@ func init() {
 			Read:   [storiface.FileTypes]uint{2, 3, 0},
 		},
 	})
+	storifaceid := storiface.ID("1399aa04-2625-44b1-bad4-bd07b59b22c4")
+	addExample(&storifaceid)
 
 	// worker specific
 	addExample(storiface.AcquireMove)
@@ -335,7 +337,21 @@ func init() {
 		Conns:           4,
 		FD:              5,
 	})
+	addExample(map[string]bitfield.BitField{
+		"": bitfield.NewFromSet([]uint64{5, 6, 7, 10}),
+	})
 
+	addExample(http.Header{
+		"Authorization": []string{"Bearer ey.."},
+	})
+
+	addExample(map[storiface.SectorFileType]storiface.SectorLocation{
+		storiface.FTSealed: {
+			Local:   false,
+			URL:     "https://example.com/sealingservice/sectors/s-f0123-12345",
+			Headers: nil,
+		},
+	})
 }
 
 func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) {
@@ -359,6 +375,10 @@ func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []r
 			i = &api.WorkerStruct{}
 			t = reflect.TypeOf(new(struct{ api.Worker })).Elem()
 			permStruct = append(permStruct, reflect.TypeOf(api.WorkerStruct{}.Internal))
+		case "Gateway":
+			i = &api.GatewayStruct{}
+			t = reflect.TypeOf(new(struct{ api.Gateway })).Elem()
+			permStruct = append(permStruct, reflect.TypeOf(api.GatewayStruct{}.Internal))
 		default:
 			panic("unknown type")
 		}

api/mocks/mock_full.go

@@ -10,6 +10,15 @@ import (
 	reflect "reflect"
 	time "time"
 
+	gomock "github.com/golang/mock/gomock"
+	uuid "github.com/google/uuid"
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
+	metrics "github.com/libp2p/go-libp2p/core/metrics"
+	network0 "github.com/libp2p/go-libp2p/core/network"
+	peer "github.com/libp2p/go-libp2p/core/peer"
+	protocol "github.com/libp2p/go-libp2p/core/protocol"
+
 	address "github.com/filecoin-project/go-address"
 	bitfield "github.com/filecoin-project/go-bitfield"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
@@ -17,11 +26,12 @@ import (
 	auth "github.com/filecoin-project/go-jsonrpc/auth"
 	abi "github.com/filecoin-project/go-state-types/abi"
 	big "github.com/filecoin-project/go-state-types/big"
-	miner "github.com/filecoin-project/go-state-types/builtin/v8/miner"
 	paych "github.com/filecoin-project/go-state-types/builtin/v8/paych"
+	miner "github.com/filecoin-project/go-state-types/builtin/v9/miner"
 	crypto "github.com/filecoin-project/go-state-types/crypto"
 	dline "github.com/filecoin-project/go-state-types/dline"
 	network "github.com/filecoin-project/go-state-types/network"
+
 	api "github.com/filecoin-project/lotus/api"
 	apitypes "github.com/filecoin-project/lotus/api/types"
 	miner0 "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
@@ -29,14 +39,6 @@ import (
 	alerting "github.com/filecoin-project/lotus/journal/alerting"
 	dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
 	imports "github.com/filecoin-project/lotus/node/repo/imports"
-	gomock "github.com/golang/mock/gomock"
-	uuid "github.com/google/uuid"
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-	metrics "github.com/libp2p/go-libp2p-core/metrics"
-	network0 "github.com/libp2p/go-libp2p-core/network"
-	peer "github.com/libp2p/go-libp2p-core/peer"
-	protocol "github.com/libp2p/go-libp2p-core/protocol"
 )
 
 // MockFullNode is a mock of FullNode interface.
@@ -375,6 +377,20 @@ func (mr *MockFullNodeMockRecorder) ChainNotify(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainNotify", reflect.TypeOf((*MockFullNode)(nil).ChainNotify), arg0)
 }
 
+// ChainPrune mocks base method.
+func (m *MockFullNode) ChainPrune(arg0 context.Context, arg1 api.PruneOpts) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ChainPrune", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ChainPrune indicates an expected call of ChainPrune.
+func (mr *MockFullNodeMockRecorder) ChainPrune(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainPrune", reflect.TypeOf((*MockFullNode)(nil).ChainPrune), arg0, arg1)
+}
+
 // ChainPutObj mocks base method.
 func (m *MockFullNode) ChainPutObj(arg0 context.Context, arg1 blocks.Block) error {
 	m.ctrl.T.Helper()
@@ -2286,6 +2302,21 @@ func (mr *MockFullNodeMockRecorder) StateActorCodeCIDs(arg0, arg1 interface{}) *
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateActorCodeCIDs", reflect.TypeOf((*MockFullNode)(nil).StateActorCodeCIDs), arg0, arg1)
 }
 
+// StateActorManifestCID mocks base method.
+func (m *MockFullNode) StateActorManifestCID(arg0 context.Context, arg1 network.Version) (cid.Cid, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "StateActorManifestCID", arg0, arg1)
+	ret0, _ := ret[0].(cid.Cid)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// StateActorManifestCID indicates an expected call of StateActorManifestCID.
+func (mr *MockFullNodeMockRecorder) StateActorManifestCID(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateActorManifestCID", reflect.TypeOf((*MockFullNode)(nil).StateActorManifestCID), arg0, arg1)
+}
+
 // StateAllMinerFaults mocks base method.
 func (m *MockFullNode) StateAllMinerFaults(arg0 context.Context, arg1 abi.ChainEpoch, arg2 types.TipSetKey) ([]*api.Fault, error) {
 	m.ctrl.T.Helper()
@@ -2361,6 +2392,21 @@ func (mr *MockFullNodeMockRecorder) StateCompute(arg0, arg1, arg2, arg3 interfac
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateCompute", reflect.TypeOf((*MockFullNode)(nil).StateCompute), arg0, arg1, arg2, arg3)
 }
 
+// StateComputeDataCID mocks base method.
+func (m *MockFullNode) StateComputeDataCID(arg0 context.Context, arg1 address.Address, arg2 abi.RegisteredSealProof, arg3 []abi.DealID, arg4 types.TipSetKey) (cid.Cid, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "StateComputeDataCID", arg0, arg1, arg2, arg3, arg4)
+	ret0, _ := ret[0].(cid.Cid)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// StateComputeDataCID indicates an expected call of StateComputeDataCID.
+func (mr *MockFullNodeMockRecorder) StateComputeDataCID(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateComputeDataCID", reflect.TypeOf((*MockFullNode)(nil).StateComputeDataCID), arg0, arg1, arg2, arg3, arg4)
+}
+
 // StateDealProviderCollateralBounds mocks base method.
 func (m *MockFullNode) StateDealProviderCollateralBounds(arg0 context.Context, arg1 abi.PaddedPieceSize, arg2 bool, arg3 types.TipSetKey) (api.DealCollateralBounds, error) {
 	m.ctrl.T.Helper()
@@ -2631,6 +2677,21 @@ func (mr *MockFullNodeMockRecorder) StateMinerActiveSectors(arg0, arg1, arg2 int
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateMinerActiveSectors", reflect.TypeOf((*MockFullNode)(nil).StateMinerActiveSectors), arg0, arg1, arg2)
 }
 
+// StateMinerAllocated mocks base method.
+func (m *MockFullNode) StateMinerAllocated(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (*bitfield.BitField, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "StateMinerAllocated", arg0, arg1, arg2)
+	ret0, _ := ret[0].(*bitfield.BitField)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// StateMinerAllocated indicates an expected call of StateMinerAllocated.
+func (mr *MockFullNodeMockRecorder) StateMinerAllocated(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateMinerAllocated", reflect.TypeOf((*MockFullNode)(nil).StateMinerAllocated), arg0, arg1, arg2)
+}
+
 // StateMinerAvailableBalance mocks base method.
 func (m *MockFullNode) StateMinerAvailableBalance(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (big.Int, error) {
 	m.ctrl.T.Helper()
@@ -2947,10 +3008,10 @@ func (mr *MockFullNodeMockRecorder) StateSectorPartition(arg0, arg1, arg2, arg3
 }
 
 // StateSectorPreCommitInfo mocks base method.
-func (m *MockFullNode) StateSectorPreCommitInfo(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) {
+func (m *MockFullNode) StateSectorPreCommitInfo(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner.SectorPreCommitOnChainInfo, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateSectorPreCommitInfo", arg0, arg1, arg2, arg3)
-	ret0, _ := ret[0].(miner.SectorPreCommitOnChainInfo)
+	ret0, _ := ret[0].(*miner.SectorPreCommitOnChainInfo)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }

api/proxy_util_test.go

@@ -1,4 +1,4 @@
-//stm: #unit
+// stm: #unit
 package api
 
 import (

api/types.go

@@ -5,21 +5,22 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/google/uuid"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-graphsync"
+	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	"github.com/libp2p/go-libp2p/core/network"
+	"github.com/libp2p/go-libp2p/core/peer"
+	ma "github.com/multiformats/go-multiaddr"
+
 	"github.com/filecoin-project/go-address"
-
-	"github.com/libp2p/go-libp2p-core/network"
-
 	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/builtin/v9/miner"
+
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
-	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-graphsync"
-
-	"github.com/libp2p/go-libp2p-core/peer"
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
-	ma "github.com/multiformats/go-multiaddr"
 )
 
 type MultiaddrSlice []ma.Multiaddr
@@ -54,7 +55,8 @@ type PubsubScore struct {
 }
 
 type MessageSendSpec struct {
-	MaxFee abi.TokenAmount
+	MaxFee  abi.TokenAmount
+	MsgUuid uuid.UUID
 }
 
 // GraphSyncDataTransfer provides diagnostics on a data transfer happening over graphsync
@@ -135,13 +137,7 @@ type NetStat struct {
 }
 
 type NetLimit struct {
-	Dynamic bool `json:",omitempty"`
-	// set if Dynamic is false
 	Memory int64 `json:",omitempty"`
-	// set if Dynamic is true
-	MemoryFraction float64 `json:",omitempty"`
-	MinMemory      int64   `json:",omitempty"`
-	MaxMemory      int64   `json:",omitempty"`
 
 	Streams, StreamsInbound, StreamsOutbound int
 	Conns, ConnsInbound, ConnsOutbound       int
@@ -253,10 +249,10 @@ type RestrievalRes struct {
 }
 
 // Selector specifies ipld selector string
-// - if the string starts with '{', it's interpreted as json selector string
-//   see https://ipld.io/specs/selectors/ and https://ipld.io/specs/selectors/fixtures/selector-fixtures-1/
-// - otherwise the string is interpreted as ipld-selector-text-lite (simple ipld path)
-//   see https://github.com/ipld/go-ipld-selector-text-lite
+//   - if the string starts with '{', it's interpreted as json selector string
+//     see https://ipld.io/specs/selectors/ and https://ipld.io/specs/selectors/fixtures/selector-fixtures-1/
+//   - otherwise the string is interpreted as ipld-selector-text-lite (simple ipld path)
+//     see https://github.com/ipld/go-ipld-selector-text-lite
 type Selector string
 
 type DagSpec struct {
@@ -300,6 +296,9 @@ type MinerInfo struct {
 	SectorSize                 abi.SectorSize
 	WindowPoStPartitionSectors uint64
 	ConsensusFaultElapsed      abi.ChainEpoch
+	Beneficiary                address.Address
+	BeneficiaryTerm            *miner.BeneficiaryTerm
+	PendingBeneficiaryTerm     *miner.PendingBeneficiaryChange
 }
 
 type NetworkParams struct {

api/v0api/full.go

@@ -3,7 +3,10 @@ package v0api
 import (
 	"context"
 
-	abinetwork "github.com/filecoin-project/go-state-types/network"
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	textselector "github.com/ipld/go-ipld-selector-text-lite"
+	"github.com/libp2p/go-libp2p/core/peer"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
@@ -11,15 +14,12 @@ import (
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/builtin/v8/paych"
+	"github.com/filecoin-project/go-state-types/builtin/v9/miner"
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/dline"
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	textselector "github.com/ipld/go-ipld-selector-text-lite"
-	"github.com/libp2p/go-libp2p-core/peer"
+	abinetwork "github.com/filecoin-project/go-state-types/network"
 
-	"github.com/filecoin-project/go-state-types/builtin/v8/miner"
-	"github.com/filecoin-project/go-state-types/builtin/v8/paych"
 	"github.com/filecoin-project/lotus/api"
 	apitypes "github.com/filecoin-project/lotus/api/types"
 	lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
@@ -386,7 +386,7 @@ type FullNode interface {
 	StateCall(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) //perm:read
 	// StateReplay replays a given message, assuming it was included in a block in the specified tipset.
 	//
-	// If a tipset key is provided, and a replacing message is found on chain,
+	// If a tipset key is provided, and a replacing message is not found on chain,
 	// the method will return an error saying that the message wasn't found
 	//
 	// If no tipset key is provided, the appropriate tipset is looked up, and if
@@ -591,7 +591,7 @@ type FullNode interface {
 	// Returns nil if there is no entry in the data cap table for the
 	// address.
 	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) //perm:read
-	// StateVerifiedClientStatus returns the address of the Verified Registry's root key
+	// StateVerifiedRegistryRootKey returns the address of the Verified Registry's root key
 	StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) //perm:read
 	// StateDealProviderCollateralBounds returns the min and max collateral a storage provider
 	// can issue. It takes the deal size and verified status as parameters.
@@ -607,6 +607,8 @@ type FullNode interface {
 	StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) //perm:read
 	// StateActorCodeCIDs returns the CIDs of all the builtin actors for the given network version
 	StateActorCodeCIDs(context.Context, abinetwork.Version) (map[string]cid.Cid, error) //perm:read
+	// StateActorManifestCID returns the CID of the builtin actors manifest for the given network version
+	StateActorManifestCID(context.Context, abinetwork.Version) (cid.Cid, error) //perm:read
 
 	// StateGetRandomnessFromTickets is used to sample the chain for randomness.
 	StateGetRandomnessFromTickets(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read

api/v0api/gateway.go

@@ -8,10 +8,10 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/builtin/v9/miner"
 	"github.com/filecoin-project/go-state-types/dline"
 	abinetwork "github.com/filecoin-project/go-state-types/network"
 
-	"github.com/filecoin-project/go-state-types/builtin/v8/miner"
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/types"
 )

api/v0api/permissioned.go

@@ -2,6 +2,7 @@ package v0api
 
 import (
 	"github.com/filecoin-project/go-jsonrpc/auth"
+
 	"github.com/filecoin-project/lotus/api"
 )
 

api/v0api/proxy_gen.go

@@ -5,17 +5,23 @@ package v0api
 import (
 	"context"
 
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	"github.com/libp2p/go-libp2p/core/peer"
+	"golang.org/x/xerrors"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/builtin/v8/miner"
 	"github.com/filecoin-project/go-state-types/builtin/v8/paych"
+	"github.com/filecoin-project/go-state-types/builtin/v9/miner"
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/dline"
 	abinetwork "github.com/filecoin-project/go-state-types/network"
+
 	"github.com/filecoin-project/lotus/api"
 	apitypes "github.com/filecoin-project/lotus/api/types"
 	lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
@@ -23,10 +29,6 @@ import (
 	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/repo/imports"
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"golang.org/x/xerrors"
 )
 
 var ErrNotSupported = xerrors.New("method not supported")
@@ -257,6 +259,8 @@ type FullNodeStruct struct {
 
 		StateActorCodeCIDs func(p0 context.Context, p1 abinetwork.Version) (map[string]cid.Cid, error) `perm:"read"`
 
+		StateActorManifestCID func(p0 context.Context, p1 abinetwork.Version) (cid.Cid, error) `perm:"read"`
+
 		StateAllMinerFaults func(p0 context.Context, p1 abi.ChainEpoch, p2 types.TipSetKey) ([]*api.Fault, error) `perm:"read"`
 
 		StateCall func(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*api.InvocResult, error) `perm:"read"`
@@ -1690,6 +1694,17 @@ func (s *FullNodeStub) StateActorCodeCIDs(p0 context.Context, p1 abinetwork.Vers
 	return *new(map[string]cid.Cid), ErrNotSupported
 }
 
+func (s *FullNodeStruct) StateActorManifestCID(p0 context.Context, p1 abinetwork.Version) (cid.Cid, error) {
+	if s.Internal.StateActorManifestCID == nil {
+		return *new(cid.Cid), ErrNotSupported
+	}
+	return s.Internal.StateActorManifestCID(p0, p1)
+}
+
+func (s *FullNodeStub) StateActorManifestCID(p0 context.Context, p1 abinetwork.Version) (cid.Cid, error) {
+	return *new(cid.Cid), ErrNotSupported
+}
+
 func (s *FullNodeStruct) StateAllMinerFaults(p0 context.Context, p1 abi.ChainEpoch, p2 types.TipSetKey) ([]*api.Fault, error) {
 	if s.Internal.StateAllMinerFaults == nil {
 		return *new([]*api.Fault), ErrNotSupported

api/v0api/v0mocks/mock_full.go

@@ -9,6 +9,15 @@ import (
 	reflect "reflect"
 	time "time"
 
+	gomock "github.com/golang/mock/gomock"
+	uuid "github.com/google/uuid"
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
+	metrics "github.com/libp2p/go-libp2p/core/metrics"
+	network0 "github.com/libp2p/go-libp2p/core/network"
+	peer "github.com/libp2p/go-libp2p/core/peer"
+	protocol "github.com/libp2p/go-libp2p/core/protocol"
+
 	address "github.com/filecoin-project/go-address"
 	bitfield "github.com/filecoin-project/go-bitfield"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
@@ -17,11 +26,12 @@ import (
 	auth "github.com/filecoin-project/go-jsonrpc/auth"
 	abi "github.com/filecoin-project/go-state-types/abi"
 	big "github.com/filecoin-project/go-state-types/big"
-	miner "github.com/filecoin-project/go-state-types/builtin/v8/miner"
 	paych "github.com/filecoin-project/go-state-types/builtin/v8/paych"
+	miner "github.com/filecoin-project/go-state-types/builtin/v9/miner"
 	crypto "github.com/filecoin-project/go-state-types/crypto"
 	dline "github.com/filecoin-project/go-state-types/dline"
 	network "github.com/filecoin-project/go-state-types/network"
+
 	api "github.com/filecoin-project/lotus/api"
 	apitypes "github.com/filecoin-project/lotus/api/types"
 	v0api "github.com/filecoin-project/lotus/api/v0api"
@@ -31,14 +41,6 @@ import (
 	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 	dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
 	imports "github.com/filecoin-project/lotus/node/repo/imports"
-	gomock "github.com/golang/mock/gomock"
-	uuid "github.com/google/uuid"
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-	metrics "github.com/libp2p/go-libp2p-core/metrics"
-	network0 "github.com/libp2p/go-libp2p-core/network"
-	peer "github.com/libp2p/go-libp2p-core/peer"
-	protocol "github.com/libp2p/go-libp2p-core/protocol"
 )
 
 // MockFullNode is a mock of FullNode interface.
@@ -2185,6 +2187,21 @@ func (mr *MockFullNodeMockRecorder) StateActorCodeCIDs(arg0, arg1 interface{}) *
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateActorCodeCIDs", reflect.TypeOf((*MockFullNode)(nil).StateActorCodeCIDs), arg0, arg1)
 }
 
+// StateActorManifestCID mocks base method.
+func (m *MockFullNode) StateActorManifestCID(arg0 context.Context, arg1 network.Version) (cid.Cid, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "StateActorManifestCID", arg0, arg1)
+	ret0, _ := ret[0].(cid.Cid)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// StateActorManifestCID indicates an expected call of StateActorManifestCID.
+func (mr *MockFullNodeMockRecorder) StateActorManifestCID(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateActorManifestCID", reflect.TypeOf((*MockFullNode)(nil).StateActorManifestCID), arg0, arg1)
+}
+
 // StateAllMinerFaults mocks base method.
 func (m *MockFullNode) StateAllMinerFaults(arg0 context.Context, arg1 abi.ChainEpoch, arg2 types.TipSetKey) ([]*api.Fault, error) {
 	m.ctrl.T.Helper()

api/v0api/v1_wrapper.go

@@ -3,30 +3,40 @@ package v0api
 import (
 	"context"
 
-	"github.com/filecoin-project/go-fil-markets/storagemarket"
-	"github.com/libp2p/go-libp2p-core/peer"
-
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/go-state-types/crypto"
-	marketevents "github.com/filecoin-project/lotus/markets/loggers"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/ipfs/go-cid"
+	"github.com/libp2p/go-libp2p/core/peer"
 	"golang.org/x/xerrors"
 
-	"github.com/ipfs/go-cid"
-
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/builtin/v9/miner"
+	"github.com/filecoin-project/go-state-types/crypto"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/v1api"
+	"github.com/filecoin-project/lotus/chain/types"
+	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 )
 
 type WrapperV1Full struct {
 	v1api.FullNode
 }
 
+func (w *WrapperV1Full) StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, s abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) {
+	pi, err := w.FullNode.StateSectorPreCommitInfo(ctx, maddr, s, tsk)
+	if err != nil {
+		return miner.SectorPreCommitOnChainInfo{}, err
+	}
+	if pi == nil {
+		return miner.SectorPreCommitOnChainInfo{}, xerrors.Errorf("precommit info does not exist")
+	}
+
+	return *pi, nil
+}
+
 func (w *WrapperV1Full) StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error) {
 	return w.FullNode.StateSearchMsg(ctx, types.EmptyTSK, msg, api.LookbackNoLimit, true)
 }

api/v1api/latest.go

@@ -7,6 +7,8 @@ import (
 type FullNode = api.FullNode
 type FullNodeStruct = api.FullNodeStruct
 
+type RawFullNodeAPI FullNode
+
 func PermissionedFullAPI(a FullNode) FullNode {
 	return api.PermissionedFullAPI(a)
 }

api/version.go

@@ -3,7 +3,7 @@ package api
 import (
 	"fmt"
 
-	xerrors "golang.org/x/xerrors"
+	"golang.org/x/xerrors"
 )
 
 type Version uint32
@@ -55,7 +55,7 @@ func VersionForType(nodeType NodeType) (Version, error) {
 // semver versions of the rpc api exposed
 var (
 	FullAPIVersion0 = newVer(1, 5, 0)
-	FullAPIVersion1 = newVer(2, 2, 0)
+	FullAPIVersion1 = newVer(2, 3, 0)
 
 	MinerAPIVersion0  = newVer(1, 5, 0)
 	WorkerAPIVersion0 = newVer(1, 6, 0)

blockstore/autobatch.go

@@ -5,10 +5,10 @@ import (
 	"sync"
 	"time"
 
-	"golang.org/x/xerrors"
-
 	block "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
+	"golang.org/x/xerrors"
 )
 
 // autolog is a logger for the autobatching blockstore. It is subscoped from the
@@ -176,7 +176,7 @@ func (bs *AutobatchBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block,
 		return blk, nil
 	}
 
-	if err != ErrNotFound {
+	if !ipld.IsNotFound(err) {
 		return blk, err
 	}
 
@@ -214,7 +214,7 @@ func (bs *AutobatchBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error)
 	if err == nil {
 		return true, nil
 	}
-	if err == ErrNotFound {
+	if ipld.IsNotFound(err) {
 		return false, nil
 	}
 

blockstore/badger/blockstore.go

@@ -13,13 +13,13 @@ import (
 	"github.com/dgraph-io/badger/v2"
 	"github.com/dgraph-io/badger/v2/options"
 	"github.com/dgraph-io/badger/v2/pb"
-	"github.com/multiformats/go-base32"
-	"go.uber.org/zap"
-
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
 	logger "github.com/ipfs/go-log/v2"
 	pool "github.com/libp2p/go-buffer-pool"
+	"github.com/multiformats/go-base32"
+	"go.uber.org/zap"
 
 	"github.com/filecoin-project/lotus/blockstore"
 )
@@ -544,7 +544,7 @@ func (b *Blockstore) View(ctx context.Context, cid cid.Cid, fn func([]byte) erro
 		case nil:
 			return item.Value(fn)
 		case badger.ErrKeyNotFound:
-			return blockstore.ErrNotFound
+			return ipld.ErrNotFound{Cid: cid}
 		default:
 			return fmt.Errorf("failed to view block from badger blockstore: %w", err)
 		}
@@ -584,7 +584,7 @@ func (b *Blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 // Get implements Blockstore.Get.
 func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
 	if !cid.Defined() {
-		return nil, blockstore.ErrNotFound
+		return nil, ipld.ErrNotFound{Cid: cid}
 	}
 
 	if err := b.access(); err != nil {
@@ -607,7 +607,7 @@ func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
 			val, err = item.ValueCopy(nil)
 			return err
 		case badger.ErrKeyNotFound:
-			return blockstore.ErrNotFound
+			return ipld.ErrNotFound{Cid: cid}
 		default:
 			return fmt.Errorf("failed to get block from badger blockstore: %w", err)
 		}
@@ -639,7 +639,7 @@ func (b *Blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
 		case nil:
 			size = int(item.ValueSize())
 		case badger.ErrKeyNotFound:
-			return blockstore.ErrNotFound
+			return ipld.ErrNotFound{Cid: cid}
 		default:
 			return fmt.Errorf("failed to get block size from badger blockstore: %w", err)
 		}

blockstore/badger/blockstore_test.go

@@ -1,4 +1,4 @@
-//stm: #unit
+// stm: #unit
 package badgerbs
 
 import (
@@ -10,12 +10,11 @@ import (
 	"strings"
 	"testing"
 
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
 
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-
 	"github.com/filecoin-project/lotus/blockstore"
 )
 

blockstore/badger/blockstore_test_suite.go

@@ -1,4 +1,4 @@
-//stm: #unit
+// stm: #unit
 package badgerbs
 
 import (
@@ -12,10 +12,10 @@ import (
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	u "github.com/ipfs/go-ipfs-util"
+	ipld "github.com/ipfs/go-ipld-format"
+	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/lotus/blockstore"
-
-	"github.com/stretchr/testify/require"
 )
 
 // TODO: move this to go-ipfs-blockstore.
@@ -56,7 +56,7 @@ func (s *Suite) TestGetWhenKeyNotPresent(t *testing.T) {
 	c := cid.NewCidV0(u.Hash([]byte("stuff")))
 	bl, err := bs.Get(ctx, c)
 	require.Nil(t, bl)
-	require.Equal(t, blockstore.ErrNotFound, err)
+	require.Equal(t, ipld.ErrNotFound{Cid: c}, err)
 }
 
 func (s *Suite) TestGetWhenKeyIsNil(t *testing.T) {
@@ -69,7 +69,7 @@ func (s *Suite) TestGetWhenKeyIsNil(t *testing.T) {
 	}
 
 	_, err := bs.Get(ctx, cid.Undef)
-	require.Equal(t, blockstore.ErrNotFound, err)
+	require.Equal(t, ipld.ErrNotFound{Cid: cid.Undef}, err)
 }
 
 func (s *Suite) TestPutThenGetBlock(t *testing.T) {
@@ -164,8 +164,9 @@ func (s *Suite) TestPutThenGetSizeBlock(t *testing.T) {
 	require.NoError(t, err)
 	require.Zero(t, emptySize)
 
-	missingSize, err := bs.GetSize(ctx, missingBlock.Cid())
-	require.Equal(t, blockstore.ErrNotFound, err)
+	missingCid := missingBlock.Cid()
+	missingSize, err := bs.GetSize(ctx, missingCid)
+	require.Equal(t, ipld.ErrNotFound{Cid: missingCid}, err)
 	require.Equal(t, -1, missingSize)
 }
 

blockstore/blockstore.go

@@ -3,17 +3,14 @@ package blockstore
 import (
 	"context"
 
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 	ds "github.com/ipfs/go-datastore"
-	logging "github.com/ipfs/go-log/v2"
-
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	logging "github.com/ipfs/go-log/v2"
 )
 
 var log = logging.Logger("blockstore")
 
-var ErrNotFound = blockstore.ErrNotFound
-
 // Blockstore is the blockstore interface used by Lotus. It is the union
 // of the basic go-ipfs blockstore, with other capabilities required by Lotus,
 // e.g. View or Sync.

blockstore/buffered.go

@@ -6,6 +6,7 @@ import (
 
 	block "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
 )
 
 // buflog is a logger for the buffered blockstore. It is subscoped from the
@@ -106,7 +107,7 @@ func (bs *BufferedBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) er
 
 func (bs *BufferedBlockstore) View(ctx context.Context, c cid.Cid, callback func([]byte) error) error {
 	// both stores are viewable.
-	if err := bs.write.View(ctx, c, callback); err == ErrNotFound {
+	if err := bs.write.View(ctx, c, callback); ipld.IsNotFound(err) {
 		// not found in write blockstore; fall through.
 	} else {
 		return err // propagate errors, or nil, i.e. found.
@@ -116,7 +117,7 @@ func (bs *BufferedBlockstore) View(ctx context.Context, c cid.Cid, callback func
 
 func (bs *BufferedBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block, error) {
 	if out, err := bs.write.Get(ctx, c); err != nil {
-		if err != ErrNotFound {
+		if !ipld.IsNotFound(err) {
 			return nil, err
 		}
 	} else {
@@ -128,7 +129,7 @@ func (bs *BufferedBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block,
 
 func (bs *BufferedBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
 	s, err := bs.read.GetSize(ctx, c)
-	if err == ErrNotFound || s == 0 {
+	if ipld.IsNotFound(err) || s == 0 {
 		return bs.write.GetSize(ctx, c)
 	}
 

blockstore/discard.go

@@ -5,7 +5,7 @@ import (
 	"io"
 
 	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 )
 
 var _ Blockstore = (*discardstore)(nil)

blockstore/fallback.go

@@ -5,10 +5,10 @@ import (
 	"sync"
 	"time"
 
-	"golang.org/x/xerrors"
-
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
+	"golang.org/x/xerrors"
 )
 
 // UnwrapFallbackStore takes a blockstore, and returns the underlying blockstore
@@ -56,7 +56,7 @@ func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
 
 		if fbs.missFn == nil {
 			log.Errorw("fallbackstore: missFn not configured yet")
-			return nil, ErrNotFound
+			return nil, ipld.ErrNotFound{Cid: c}
 		}
 	}
 
@@ -79,10 +79,10 @@ func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
 
 func (fbs *FallbackStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
 	b, err := fbs.Blockstore.Get(ctx, c)
-	switch err {
-	case nil:
+	switch {
+	case err == nil:
 		return b, nil
-	case ErrNotFound:
+	case ipld.IsNotFound(err):
 		return fbs.getFallback(c)
 	default:
 		return b, err
@@ -91,10 +91,10 @@ func (fbs *FallbackStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, err
 
 func (fbs *FallbackStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
 	sz, err := fbs.Blockstore.GetSize(ctx, c)
-	switch err {
-	case nil:
+	switch {
+	case err == nil:
 		return sz, nil
-	case ErrNotFound:
+	case ipld.IsNotFound(err):
 		b, err := fbs.getFallback(c)
 		if err != nil {
 			return 0, err

blockstore/idstore.go

@@ -4,11 +4,10 @@ import (
 	"context"
 	"io"
 
-	"golang.org/x/xerrors"
-
 	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 	mh "github.com/multiformats/go-multihash"
+	"golang.org/x/xerrors"
 )
 
 var _ Blockstore = (*idstore)(nil)
@@ -103,6 +102,14 @@ func (b *idstore) Put(ctx context.Context, blk blocks.Block) error {
 	return b.bs.Put(ctx, blk)
 }
 
+func (b *idstore) ForEachKey(f func(cid.Cid) error) error {
+	iterBstore, ok := b.bs.(BlockstoreIterator)
+	if !ok {
+		return xerrors.Errorf("underlying blockstore (type %T) doesn't support fast iteration", b.bs)
+	}
+	return iterBstore.ForEachKey(f)
+}
+
 func (b *idstore) PutMany(ctx context.Context, blks []blocks.Block) error {
 	toPut := make([]blocks.Block, 0, len(blks))
 	for _, blk := range blks {

blockstore/ipfs.go

@@ -5,17 +5,15 @@ import (
 	"context"
 	"io/ioutil"
 
-	"golang.org/x/xerrors"
-
-	"github.com/multiformats/go-multiaddr"
-	"github.com/multiformats/go-multihash"
-
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	httpapi "github.com/ipfs/go-ipfs-http-client"
 	iface "github.com/ipfs/interface-go-ipfs-core"
 	"github.com/ipfs/interface-go-ipfs-core/options"
 	"github.com/ipfs/interface-go-ipfs-core/path"
+	"github.com/multiformats/go-multiaddr"
+	"github.com/multiformats/go-multihash"
+	"golang.org/x/xerrors"
 )
 
 type IPFSBlockstore struct {
@@ -130,7 +128,7 @@ func (i *IPFSBlockstore) Put(ctx context.Context, block blocks.Block) error {
 
 	_, err = i.api.Block().Put(ctx, bytes.NewReader(block.RawData()),
 		options.Block.Hash(mhd.Code, mhd.Length),
-		options.Block.Format(cid.CodecToStr[block.Cid().Type()]))
+		options.Block.Format(multihash.Codes[block.Cid().Type()]))
 	return err
 }
 

blockstore/mem.go

@@ -5,6 +5,7 @@ import (
 
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
 )
 
 // NewMemory returns a temporary memory-backed blockstore.
@@ -13,46 +14,47 @@ func NewMemory() MemBlockstore {
 }
 
 // MemBlockstore is a terminal blockstore that keeps blocks in memory.
-type MemBlockstore map[cid.Cid]blocks.Block
+// To match behavior of badger blockstore we index by multihash only.
+type MemBlockstore map[string]blocks.Block
 
 func (m MemBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
-	delete(m, k)
+	delete(m, string(k.Hash()))
 	return nil
 }
 
 func (m MemBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error {
 	for _, k := range ks {
-		delete(m, k)
+		delete(m, string(k.Hash()))
 	}
 	return nil
 }
 
 func (m MemBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error) {
-	_, ok := m[k]
+	_, ok := m[string(k.Hash())]
 	return ok, nil
 }
 
 func (m MemBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error {
-	b, ok := m[k]
+	b, ok := m[string(k.Hash())]
 	if !ok {
-		return ErrNotFound
+		return ipld.ErrNotFound{Cid: k}
 	}
 	return callback(b.RawData())
 }
 
 func (m MemBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) {
-	b, ok := m[k]
+	b, ok := m[string(k.Hash())]
 	if !ok {
-		return nil, ErrNotFound
+		return nil, ipld.ErrNotFound{Cid: k}
 	}
 	return b, nil
 }
 
 // GetSize returns the CIDs mapped BlockSize
 func (m MemBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
-	b, ok := m[k]
+	b, ok := m[string(k.Hash())]
 	if !ok {
-		return 0, ErrNotFound
+		return 0, ipld.ErrNotFound{Cid: k}
 	}
 	return len(b.RawData()), nil
 }
@@ -61,7 +63,7 @@ func (m MemBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
 func (m MemBlockstore) Put(ctx context.Context, b blocks.Block) error {
 	// Convert to a basic block for safety, but try to reuse the existing
 	// block if it's already a basic block.
-	k := b.Cid()
+	k := string(b.Cid().Hash())
 	if _, ok := b.(*blocks.BasicBlock); !ok {
 		// If we already have the block, abort.
 		if _, ok := m[k]; ok {
@@ -70,7 +72,7 @@ func (m MemBlockstore) Put(ctx context.Context, b blocks.Block) error {
 		// the error is only for debugging.
 		b, _ = blocks.NewBlockWithCid(b.RawData(), b.Cid())
 	}
-	m[b.Cid()] = b
+	m[k] = b
 	return nil
 }
 
@@ -88,8 +90,8 @@ func (m MemBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
 // the given context, closing the channel if it becomes Done.
 func (m MemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
 	ch := make(chan cid.Cid, len(m))
-	for k := range m {
-		ch <- k
+	for _, b := range m {
+		ch <- b.Cid()
 	}
 	close(ch)
 	return ch, nil

blockstore/splitstore/checkpoint.go

@@ -5,10 +5,9 @@ import (
 	"io"
 	"os"
 
-	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 	mh "github.com/multiformats/go-multihash"
+	"golang.org/x/xerrors"
 )
 
 type Checkpoint struct {

blockstore/splitstore/coldset.go

@@ -5,9 +5,8 @@ import (
 	"io"
 	"os"
 
+	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
 )
 
 type ColdSetWriter struct {

blockstore/splitstore/debug.go

@@ -12,11 +12,10 @@ import (
 	"sync"
 	"time"
 
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
 	"go.uber.org/multierr"
 	"golang.org/x/xerrors"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
 )
 
 type debugLog struct {

blockstore/splitstore/markset.go

@@ -3,9 +3,8 @@ package splitstore
 import (
 	"errors"
 
+	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
 )
 
 var errMarkSetClosed = errors.New("markset closed")

blockstore/splitstore/markset_badger.go

@@ -6,13 +6,11 @@ import (
 	"runtime"
 	"sync"
 
-	"golang.org/x/xerrors"
-
 	"github.com/dgraph-io/badger/v2"
 	"github.com/dgraph-io/badger/v2/options"
+	"github.com/ipfs/go-cid"
 	"go.uber.org/zap"
-
-	cid "github.com/ipfs/go-cid"
+	"golang.org/x/xerrors"
 )
 
 type BadgerMarkSetEnv struct {

blockstore/splitstore/markset_map.go

@@ -7,9 +7,8 @@ import (
 	"path/filepath"
 	"sync"
 
+	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
 )
 
 type MapMarkSetEnv struct {

blockstore/splitstore/markset_test.go

@@ -1,10 +1,10 @@
-//stm: #unit
+// stm: #unit
 package splitstore
 
 import (
 	"testing"
 
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 	"github.com/multiformats/go-multihash"
 )
 

blockstore/splitstore/splitstore.go

@@ -8,22 +8,22 @@ import (
 	"sync/atomic"
 	"time"
 
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	dstore "github.com/ipfs/go-datastore"
+	ipld "github.com/ipfs/go-ipld-format"
+	logging "github.com/ipfs/go-log/v2"
+	"go.opencensus.io/stats"
 	"go.uber.org/multierr"
 	"golang.org/x/xerrors"
 
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-	dstore "github.com/ipfs/go-datastore"
-	logging "github.com/ipfs/go-log/v2"
-
 	"github.com/filecoin-project/go-state-types/abi"
+
 	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/metrics"
-
-	"go.opencensus.io/stats"
 )
 
 var (
@@ -43,8 +43,16 @@ var (
 	// compactionIndexKey stores the compaction index (serial number)
 	compactionIndexKey = dstore.NewKey("/splitstore/compactionIndex")
 
+	// stores the prune index (serial number)
+	pruneIndexKey = dstore.NewKey("/splitstore/pruneIndex")
+
+	// stores the base epoch of last prune in the metadata store
+	pruneEpochKey = dstore.NewKey("/splitstore/pruneEpoch")
+
 	log = logging.Logger("splitstore")
 
+	errClosing = errors.New("splitstore is closing")
+
 	// set this to true if you are debugging the splitstore to enable debug logging
 	enableDebugLog = false
 	// set this to true if you want to track origin stack traces in the write log
@@ -54,6 +62,16 @@ var (
 	upgradeBoundary = build.Finality
 )
 
+type CompactType int
+
+const (
+	none CompactType = iota
+	warmup
+	hot
+	cold
+	check
+)
+
 func init() {
 	if os.Getenv("LOTUS_SPLITSTORE_DEBUG_LOG") == "1" {
 		enableDebugLog = true
@@ -93,6 +111,21 @@ type Config struct {
 	// A positive value is the number of compactions before a full GC is performed;
 	// a value of 1 will perform full GC in every compaction.
 	HotStoreFullGCFrequency uint64
+
+	// EnableColdStoreAutoPrune turns on compaction of the cold store i.e. pruning
+	// where hotstore compaction occurs every finality epochs pruning happens every 3 finalities
+	// Default is false
+	EnableColdStoreAutoPrune bool
+
+	// ColdStoreFullGCFrequency specifies how often to performa a full (moving) GC on the coldstore.
+	// Only applies if auto prune is enabled.  A value of 0 disables while a value of 1 will do
+	// full GC in every prune.
+	// Default is 7 (about once every a week)
+	ColdStoreFullGCFrequency uint64
+
+	// ColdStoreRetention specifies the retention policy for data reachable from the chain, in
+	// finalities beyond the compaction boundary, default is 0, -1 retains everything
+	ColdStoreRetention int64
 }
 
 // ChainAccessor allows the Splitstore to access the chain. It will most likely
@@ -117,8 +150,9 @@ type hotstore interface {
 }
 
 type SplitStore struct {
-	compacting int32 // compaction/prune/warmup in progress
-	closing    int32 // the splitstore is closing
+	compacting  int32       // flag for when compaction is in progress
+	compactType CompactType // compaction type, protected by compacting atomic, only meaningful when compacting == 1
+	closing     int32       // the splitstore is closing
 
 	cfg  *Config
 	path string
@@ -126,6 +160,7 @@ type SplitStore struct {
 	mx          sync.Mutex
 	warmupEpoch abi.ChainEpoch // protected by mx
 	baseEpoch   abi.ChainEpoch // protected by compaction lock
+	pruneEpoch  abi.ChainEpoch // protected by compaction lock
 
 	headChangeMx sync.Mutex
 
@@ -140,6 +175,7 @@ type SplitStore struct {
 	markSetSize int64
 
 	compactionIndex int64
+	pruneIndex      int64
 
 	ctx    context.Context
 	cancel func()
@@ -227,6 +263,13 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 			return nil, xerrors.Errorf("error resuming compaction: %w", err)
 		}
 	}
+	if ss.pruneCheckpointExists() {
+		log.Info("found prune checkpoint; resuming prune")
+		if err := ss.completePrune(); err != nil {
+			markSetEnv.Close() //nolint:errcheck
+			return nil, xerrors.Errorf("error resuming prune: %w", err)
+		}
+	}
 
 	return ss, nil
 }
@@ -260,8 +303,14 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 		if has {
 			return s.has(cid)
 		}
-
-		return s.cold.Has(ctx, cid)
+		switch s.compactType {
+		case hot:
+			return s.cold.Has(ctx, cid)
+		case cold:
+			return s.hot.Has(ctx, cid)
+		default:
+			return false, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
+		}
 	}
 
 	has, err := s.hot.Has(ctx, cid)
@@ -276,8 +325,11 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 	}
 
 	has, err = s.cold.Has(ctx, cid)
-	if has && bstore.IsHotView(ctx) {
-		s.reifyColdObject(cid)
+	if has {
+		s.trackTxnRef(cid)
+		if bstore.IsHotView(ctx) {
+			s.reifyColdObject(cid)
+		}
 	}
 
 	return has, err
@@ -307,24 +359,31 @@ func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
 		if has {
 			return s.get(cid)
 		}
-
-		return s.cold.Get(ctx, cid)
+		switch s.compactType {
+		case hot:
+			return s.cold.Get(ctx, cid)
+		case cold:
+			return s.hot.Get(ctx, cid)
+		default:
+			return nil, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
+		}
 	}
 
 	blk, err := s.hot.Get(ctx, cid)
 
-	switch err {
-	case nil:
+	switch {
+	case err == nil:
 		s.trackTxnRef(cid)
 		return blk, nil
 
-	case bstore.ErrNotFound:
+	case ipld.IsNotFound(err):
 		if s.isWarm() {
 			s.debug.LogReadMiss(cid)
 		}
 
 		blk, err = s.cold.Get(ctx, cid)
 		if err == nil {
+			s.trackTxnRef(cid)
 			if bstore.IsHotView(ctx) {
 				s.reifyColdObject(cid)
 			}
@@ -361,24 +420,31 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
 		if has {
 			return s.getSize(cid)
 		}
-
-		return s.cold.GetSize(ctx, cid)
+		switch s.compactType {
+		case hot:
+			return s.cold.GetSize(ctx, cid)
+		case cold:
+			return s.hot.GetSize(ctx, cid)
+		default:
+			return 0, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
+		}
 	}
 
 	size, err := s.hot.GetSize(ctx, cid)
 
-	switch err {
-	case nil:
+	switch {
+	case err == nil:
 		s.trackTxnRef(cid)
 		return size, nil
 
-	case bstore.ErrNotFound:
+	case ipld.IsNotFound(err):
 		if s.isWarm() {
 			s.debug.LogReadMiss(cid)
 		}
 
 		size, err = s.cold.GetSize(ctx, cid)
 		if err == nil {
+			s.trackTxnRef(cid)
 			if bstore.IsHotView(ctx) {
 				s.reifyColdObject(cid)
 			}
@@ -408,12 +474,12 @@ func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error {
 	s.debug.LogWrite(blk)
 
 	// critical section
-	if s.txnMarkSet != nil {
+	if s.txnMarkSet != nil && s.compactType == hot { // puts only touch hot store
 		s.markLiveRefs([]cid.Cid{blk.Cid()})
 		return nil
 	}
-
 	s.trackTxnRef(blk.Cid())
+
 	return nil
 }
 
@@ -459,12 +525,12 @@ func (s *SplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {
 	s.debug.LogWriteMany(blks)
 
 	// critical section
-	if s.txnMarkSet != nil {
+	if s.txnMarkSet != nil && s.compactType == hot { // puts only touch hot store
 		s.markLiveRefs(batch)
 		return nil
 	}
-
 	s.trackTxnRefMany(batch)
+
 	return nil
 }
 
@@ -536,8 +602,14 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
 		if has {
 			return s.view(cid, cb)
 		}
-
-		return s.cold.View(ctx, cid, cb)
+		switch s.compactType {
+		case hot:
+			return s.cold.View(ctx, cid, cb)
+		case cold:
+			return s.hot.View(ctx, cid, cb)
+		default:
+			return xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
+		}
 	}
 
 	// views are (optimistically) protected two-fold:
@@ -552,8 +624,7 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
 	defer s.viewDone()
 
 	err := s.hot.View(ctx, cid, cb)
-	switch err {
-	case bstore.ErrNotFound:
+	if ipld.IsNotFound(err) {
 		if s.isWarm() {
 			s.debug.LogReadMiss(cid)
 		}
@@ -567,10 +638,8 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
 			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return err
-
-	default:
-		return err
 	}
+	return err
 }
 
 func (s *SplitStore) isWarm() bool {
@@ -626,6 +695,23 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
 		return xerrors.Errorf("error loading base epoch: %w", err)
 	}
 
+	// load prune epoch from metadata ds
+	bs, err = s.ds.Get(s.ctx, pruneEpochKey)
+	switch err {
+	case nil:
+		s.pruneEpoch = bytesToEpoch(bs)
+	case dstore.ErrNotFound:
+		if curTs == nil {
+			//this can happen in some tests
+			break
+		}
+		if err := s.setPruneEpoch(curTs.Height()); err != nil {
+			return xerrors.Errorf("error saving prune epoch: %w", err)
+		}
+	default:
+		return xerrors.Errorf("error loading prune epoch: %w", err)
+	}
+
 	// load warmup epoch from metadata ds
 	bs, err = s.ds.Get(s.ctx, warmupEpochKey)
 	switch err {
@@ -725,3 +811,8 @@ func (s *SplitStore) setBaseEpoch(epoch abi.ChainEpoch) error {
 	s.baseEpoch = epoch
 	return s.ds.Put(s.ctx, baseEpochKey, epochToBytes(epoch))
 }
+
+func (s *SplitStore) setPruneEpoch(epoch abi.ChainEpoch) error {
+	s.pruneEpoch = epoch
+	return s.ds.Put(s.ctx, pruneEpochKey, epochToBytes(epoch))
+}

blockstore/splitstore/splitstore_check.go

@@ -8,10 +8,9 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
 
-	cid "github.com/ipfs/go-cid"
-
 	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
 )
@@ -26,6 +25,7 @@ func (s *SplitStore) Check() error {
 	if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
 		return xerrors.Errorf("can't acquire compaction lock; compacting operation in progress")
 	}
+	s.compactType = check
 
 	if s.compactionIndex == 0 {
 		atomic.StoreInt32(&s.compacting, 0)
@@ -147,6 +147,8 @@ func (s *SplitStore) Info() map[string]interface{} {
 	info["base epoch"] = s.baseEpoch
 	info["warmup epoch"] = s.warmupEpoch
 	info["compactions"] = s.compactionIndex
+	info["prunes"] = s.pruneIndex
+	info["compacting"] = s.compacting == 1
 
 	sizer, ok := s.hot.(bstore.BlockstoreSize)
 	if ok {

blockstore/splitstore/splitstore_compact.go

@@ -10,20 +10,20 @@ import (
 	"sync/atomic"
 	"time"
 
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	"go.opencensus.io/stats"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/xerrors"
 
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-	cbg "github.com/whyrusleeping/cbor-gen"
-
 	"github.com/filecoin-project/go-state-types/abi"
+
 	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/metrics"
-
-	"go.opencensus.io/stats"
 )
 
 var (
@@ -53,6 +53,12 @@ var (
 	// SyncWaitTime is the time delay from a tipset's min timestamp before we decide
 	// we have synced.
 	SyncWaitTime = 30 * time.Second
+
+	// This is a testing flag that should always be true when running a node. itests rely on the rough hack
+	// of starting genesis so far in the past that they exercise catchup mining to mine
+	// blocks quickly and so disabling syncgap checking is necessary to test compaction
+	// without a deep structural improvement of itests.
+	CheckSyncGap = true
 )
 
 var (
@@ -82,7 +88,7 @@ func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
 	//       this is guaranteed by the chainstore, and it is pervasive in all lotus
 	//       -- if that ever changes then all hell will break loose in general and
 	//       we will have a rance to protectTipSets here.
-	//      Reagrdless, we put a mutex in HeadChange just to be safe
+	//      Regardless, we put a mutex in HeadChange just to be safe
 
 	if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
 		// we are currently compacting -- protect the new tipset(s)
@@ -97,7 +103,8 @@ func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
 	}
 
 	timestamp := time.Unix(int64(curTs.MinTimestamp()), 0)
-	if time.Since(timestamp) > SyncGapTime {
+
+	if CheckSyncGap && time.Since(timestamp) > SyncGapTime {
 		// don't attempt compaction before we have caught up syncing
 		atomic.StoreInt32(&s.compacting, 0)
 		return nil
@@ -109,9 +116,12 @@ func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
 		return nil
 	}
 
+	// Prioritize hot store compaction over cold store prune
+
 	if epoch-s.baseEpoch > CompactionThreshold {
 		// it's time to compact -- prepare the transaction and go!
 		s.beginTxnProtect()
+		s.compactType = hot
 		go func() {
 			defer atomic.StoreInt32(&s.compacting, 0)
 			defer s.endTxnProtect()
@@ -123,6 +133,40 @@ func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
 
 			log.Infow("compaction done", "took", time.Since(start))
 		}()
+		// only prune if auto prune is enabled and after at least one compaction
+	} else if s.cfg.EnableColdStoreAutoPrune && epoch-s.pruneEpoch > PruneThreshold && s.compactionIndex > 0 {
+		s.beginTxnProtect()
+		s.compactType = cold
+		go func() {
+			defer atomic.StoreInt32(&s.compacting, 0)
+			defer s.endTxnProtect()
+
+			log.Info("pruning splitstore")
+			start := time.Now()
+
+			var retainP func(int64) bool
+			switch {
+			case s.cfg.ColdStoreRetention > int64(0):
+				retainP = func(depth int64) bool {
+					return depth <= int64(CompactionBoundary)+s.cfg.ColdStoreRetention*int64(build.Finality)
+				}
+			case s.cfg.ColdStoreRetention < 0:
+				retainP = func(_ int64) bool { return true }
+			default:
+				retainP = func(depth int64) bool {
+					return depth <= int64(CompactionBoundary)
+				}
+			}
+			movingGC := s.cfg.ColdStoreFullGCFrequency > 0 && s.pruneIndex%int64(s.cfg.ColdStoreFullGCFrequency) == 0
+			var gcOpts []bstore.BlockstoreGCOption
+			if movingGC {
+				gcOpts = append(gcOpts, bstore.WithFullGC(true))
+			}
+			doGC := func() error { return s.gcBlockstore(s.cold, gcOpts) }
+
+			s.prune(curTs, retainP, doGC)
+			log.Infow("prune done", "took", time.Since(start))
+		}()
 	} else {
 		// no compaction necessary
 		atomic.StoreInt32(&s.compacting, 0)
@@ -475,6 +519,7 @@ func (s *SplitStore) applyProtectors() error {
 // - At this point we are ready to begin purging:
 //   - We sort cold objects heaviest first, so as to never delete the consituents of a DAG before the DAG itself (which would leave dangling references)
 //   - We delete in small batches taking a lock; each batch is checked again for marks, from the concurrent transactional mark, so as to never delete anything live
+//
 // - We then end the transaction and compact/gc the hotstore.
 func (s *SplitStore) compact(curTs *types.TipSet) {
 	log.Info("waiting for active views to complete")
@@ -588,7 +633,6 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 
 	// some stats for logging
 	var hotCnt, coldCnt int
-
 	err = s.hot.ForEachKey(func(c cid.Cid) error {
 		// was it marked?
 		mark, err := markSet.Has(c)
@@ -609,7 +653,6 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 
 		return nil
 	})
-
 	if err != nil {
 		return xerrors.Errorf("error collecting cold objects: %w", err)
 	}
@@ -778,6 +821,10 @@ func (s *SplitStore) beginCriticalSection(markSet MarkSet) error {
 
 func (s *SplitStore) waitForSync() {
 	log.Info("waiting for sync")
+	if !CheckSyncGap {
+		log.Warnf("If you see this outside of test it is a serious splitstore issue")
+		return
+	}
 	startWait := time.Now()
 	defer func() {
 		log.Infow("waiting for sync done", "took", time.Since(startWait))
@@ -1065,13 +1112,10 @@ func (s *SplitStore) view(c cid.Cid, cb func([]byte) error) error {
 	}
 
 	err := s.hot.View(s.ctx, c, cb)
-	switch err {
-	case bstore.ErrNotFound:
+	if ipld.IsNotFound(err) {
 		return s.cold.View(s.ctx, c, cb)
-
-	default:
-		return err
 	}
+	return err
 }
 
 func (s *SplitStore) has(c cid.Cid) (bool, error) {
@@ -1090,10 +1134,10 @@ func (s *SplitStore) has(c cid.Cid) (bool, error) {
 
 func (s *SplitStore) get(c cid.Cid) (blocks.Block, error) {
 	blk, err := s.hot.Get(s.ctx, c)
-	switch err {
-	case nil:
+	switch {
+	case err == nil:
 		return blk, nil
-	case bstore.ErrNotFound:
+	case ipld.IsNotFound(err):
 		return s.cold.Get(s.ctx, c)
 	default:
 		return nil, err
@@ -1102,10 +1146,10 @@ func (s *SplitStore) get(c cid.Cid) (blocks.Block, error) {
 
 func (s *SplitStore) getSize(c cid.Cid) (int, error) {
 	sz, err := s.hot.GetSize(s.ctx, c)
-	switch err {
-	case nil:
+	switch {
+	case err == nil:
 		return sz, nil
-	case bstore.ErrNotFound:
+	case ipld.IsNotFound(err):
 		return s.cold.GetSize(s.ctx, c)
 	default:
 		return 0, err
@@ -1119,10 +1163,9 @@ func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
 		if err := s.checkClosing(); err != nil {
 			return err
 		}
-
 		blk, err := s.hot.Get(s.ctx, c)
 		if err != nil {
-			if err == bstore.ErrNotFound {
+			if ipld.IsNotFound(err) {
 				log.Warnf("hotstore missing block %s", c)
 				return nil
 			}
@@ -1137,6 +1180,7 @@ func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
 				return xerrors.Errorf("error putting batch to coldstore: %w", err)
 			}
 			batch = batch[:0]
+
 		}
 
 		return nil
@@ -1225,8 +1269,17 @@ func (s *SplitStore) purgeBatch(batch, deadCids []cid.Cid, checkpoint *Checkpoin
 		return 0, liveCnt, nil
 	}
 
-	if err := s.hot.DeleteMany(s.ctx, deadCids); err != nil {
-		return 0, liveCnt, xerrors.Errorf("error purging cold objects: %w", err)
+	switch s.compactType {
+	case hot:
+		if err := s.hot.DeleteMany(s.ctx, deadCids); err != nil {
+			return 0, liveCnt, xerrors.Errorf("error purging cold objects: %w", err)
+		}
+	case cold:
+		if err := s.cold.DeleteMany(s.ctx, deadCids); err != nil {
+			return 0, liveCnt, xerrors.Errorf("error purging dead objects: %w", err)
+		}
+	default:
+		return 0, liveCnt, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
 	}
 
 	s.debug.LogDelete(deadCids)
@@ -1243,15 +1296,28 @@ func (s *SplitStore) coldSetPath() string {
 	return filepath.Join(s.path, "coldset")
 }
 
+func (s *SplitStore) deadSetPath() string {
+	return filepath.Join(s.path, "deadset")
+}
+
 func (s *SplitStore) checkpointPath() string {
 	return filepath.Join(s.path, "checkpoint")
 }
 
+func (s *SplitStore) pruneCheckpointPath() string {
+	return filepath.Join(s.path, "prune-checkpoint")
+}
+
 func (s *SplitStore) checkpointExists() bool {
 	_, err := os.Stat(s.checkpointPath())
 	return err == nil
 }
 
+func (s *SplitStore) pruneCheckpointExists() bool {
+	_, err := os.Stat(s.pruneCheckpointPath())
+	return err == nil
+}
+
 func (s *SplitStore) completeCompaction() error {
 	checkpoint, last, err := OpenCheckpoint(s.checkpointPath())
 	if err != nil {
@@ -1272,6 +1338,7 @@ func (s *SplitStore) completeCompaction() error {
 	defer markSet.Close() //nolint:errcheck
 
 	// PURGE
+	s.compactType = hot
 	log.Info("purging cold objects from the hotstore")
 	startPurge := time.Now()
 	err = s.completePurge(coldr, checkpoint, last, markSet)
@@ -1294,6 +1361,7 @@ func (s *SplitStore) completeCompaction() error {
 	if err := os.Remove(s.coldSetPath()); err != nil {
 		log.Warnf("error removing coldset: %s", err)
 	}
+	s.compactType = none
 
 	// Note: at this point we can start the splitstore; a compaction should run on
 	//       the first head change, which will trigger gc on the hotstore.

blockstore/splitstore/splitstore_expose.go

@@ -5,7 +5,8 @@ import (
 	"errors"
 
 	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
 
 	bstore "github.com/filecoin-project/lotus/blockstore"
 )
@@ -42,6 +43,7 @@ func (es *exposedSplitStore) Has(ctx context.Context, c cid.Cid) (bool, error) {
 }
 
 func (es *exposedSplitStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
+
 	if isIdentiyCid(c) {
 		data, err := decodeIdentityCid(c)
 		if err != nil {
@@ -52,12 +54,10 @@ func (es *exposedSplitStore) Get(ctx context.Context, c cid.Cid) (blocks.Block,
 	}
 
 	blk, err := es.s.hot.Get(ctx, c)
-	switch err {
-	case bstore.ErrNotFound:
+	if ipld.IsNotFound(err) {
 		return es.s.cold.Get(ctx, c)
-	default:
-		return blk, err
 	}
+	return blk, err
 }
 
 func (es *exposedSplitStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
@@ -71,12 +71,10 @@ func (es *exposedSplitStore) GetSize(ctx context.Context, c cid.Cid) (int, error
 	}
 
 	size, err := es.s.hot.GetSize(ctx, c)
-	switch err {
-	case bstore.ErrNotFound:
+	if ipld.IsNotFound(err) {
 		return es.s.cold.GetSize(ctx, c)
-	default:
-		return size, err
 	}
+	return size, err
 }
 
 func (es *exposedSplitStore) Put(ctx context.Context, blk blocks.Block) error {
@@ -104,11 +102,9 @@ func (es *exposedSplitStore) View(ctx context.Context, c cid.Cid, f func([]byte)
 	}
 
 	err := es.s.hot.View(ctx, c, f)
-	switch err {
-	case bstore.ErrNotFound:
+	if ipld.IsNotFound(err) {
 		return es.s.cold.View(ctx, c, f)
-
-	default:
-		return err
 	}
+
+	return err
 }

blockstore/splitstore/splitstore_gc.go

@@ -27,7 +27,7 @@ func (s *SplitStore) gcBlockstore(b bstore.Blockstore, opts []bstore.BlockstoreG
 			return err
 		}
 
-		log.Infow("garbage collecting hotstore done", "took", time.Since(startGC))
+		log.Infow("garbage collecting blockstore done", "took", time.Since(startGC))
 		return nil
 	}
 

blockstore/splitstore/splitstore_prune.go

@@ -0,0 +1,574 @@
+package splitstore
+
+import (
+	"bytes"
+	"os"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	cid "github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	"go.opencensus.io/stats"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/api"
+	bstore "github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/metrics"
+)
+
+var (
+	// PruneOnline is a prune option that instructs PruneChain to use online gc for reclaiming space;
+	// there is no value associated with this option.
+	PruneOnlineGC = "splitstore.PruneOnlineGC"
+
+	// PruneMoving is a prune option that instructs PruneChain to use moving gc for reclaiming space;
+	// the value associated with this option is the path of the new coldstore.
+	PruneMovingGC = "splitstore.PruneMovingGC"
+
+	// PruneRetainState is a prune option that instructs PruneChain as to how many finalities worth
+	// of state to retain in the coldstore.
+	// The value is an integer:
+	// - if it is -1 then all state objects reachable from the chain will be retained in the coldstore.
+	//   this is useful for garbage collecting side-chains and other garbage in archival nodes.
+	//   This is the (safe) default.
+	// - if it is 0 then no state objects that are unreachable within the compaction boundary will
+	//   be retained in the coldstore.
+	// - if it is a positive integer, then it's the number of finalities past the compaction boundary
+	//   for which chain-reachable state objects are retained.
+	PruneRetainState = "splitstore.PruneRetainState"
+
+	// PruneThreshold is the number of epochs that need to have elapsed
+	// from the previously pruned epoch to trigger a new prune
+	PruneThreshold = 7 * build.Finality
+)
+
+// PruneChain instructs the SplitStore to prune chain state in the coldstore, according to the
+// options specified.
+func (s *SplitStore) PruneChain(opts api.PruneOpts) error {
+	retainState := opts.RetainState
+
+	var gcOpts []bstore.BlockstoreGCOption
+	if opts.MovingGC {
+		gcOpts = append(gcOpts, bstore.WithFullGC(true))
+	}
+	doGC := func() error { return s.gcBlockstore(s.cold, gcOpts) }
+
+	var retainStateP func(int64) bool
+	switch {
+	case retainState > 0:
+		retainStateP = func(depth int64) bool {
+			return depth <= int64(CompactionBoundary)+retainState*int64(build.Finality)
+		}
+	case retainState < 0:
+		retainStateP = func(_ int64) bool { return true }
+	default:
+		retainStateP = func(depth int64) bool {
+			return depth <= int64(CompactionBoundary)
+		}
+	}
+
+	if _, ok := s.cold.(bstore.BlockstoreIterator); !ok {
+		return xerrors.Errorf("coldstore does not support efficient iteration")
+	}
+
+	return s.pruneChain(retainStateP, doGC)
+}
+
+func (s *SplitStore) pruneChain(retainStateP func(int64) bool, doGC func() error) error {
+	// inhibit compaction while we are setting up
+	s.headChangeMx.Lock()
+	defer s.headChangeMx.Unlock()
+
+	// take the compaction lock; fail if there is a compaction in progress
+	if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
+		return xerrors.Errorf("compaction, prune or warmup in progress")
+	}
+
+	// check if we are actually closing first
+	if atomic.LoadInt32(&s.closing) == 1 {
+		atomic.StoreInt32(&s.compacting, 0)
+		return errClosing
+	}
+
+	// ensure that we have compacted at least once
+	if s.compactionIndex == 0 {
+		atomic.StoreInt32(&s.compacting, 0)
+		return xerrors.Errorf("splitstore has not compacted yet")
+	}
+
+	// get the current tipset
+	curTs := s.chain.GetHeaviestTipSet()
+
+	// begin the transaction and go
+	s.beginTxnProtect()
+	s.compactType = cold
+	go func() {
+		defer atomic.StoreInt32(&s.compacting, 0)
+		defer s.endTxnProtect()
+
+		log.Info("pruning splitstore")
+		start := time.Now()
+
+		s.prune(curTs, retainStateP, doGC)
+
+		log.Infow("prune done", "took", time.Since(start))
+	}()
+
+	return nil
+}
+
+func (s *SplitStore) prune(curTs *types.TipSet, retainStateP func(int64) bool, doGC func() error) {
+	log.Debug("waiting for active views to complete")
+	start := time.Now()
+	s.viewWait()
+	log.Debugw("waiting for active views done", "took", time.Since(start))
+
+	err := s.doPrune(curTs, retainStateP, doGC)
+	if err != nil {
+		log.Errorf("PRUNE ERROR: %s", err)
+	}
+}
+
+func (s *SplitStore) doPrune(curTs *types.TipSet, retainStateP func(int64) bool, doGC func() error) error {
+	currentEpoch := curTs.Height()
+	boundaryEpoch := currentEpoch - CompactionBoundary
+
+	log.Infow("running prune", "currentEpoch", currentEpoch, "pruneEpoch", s.pruneEpoch)
+
+	markSet, err := s.markSetEnv.New("live", s.markSetSize)
+	if err != nil {
+		return xerrors.Errorf("error creating mark set: %w", err)
+	}
+	defer markSet.Close() //nolint:errcheck
+	defer s.debug.Flush()
+
+	if err := s.checkClosing(); err != nil {
+		return err
+	}
+
+	// 0. track all protected references at beginning of compaction; anything added later should
+	//    be transactionally protected by the write
+	log.Info("protecting references with registered protectors")
+	err = s.applyProtectors()
+	if err != nil {
+		return err
+	}
+
+	// 1. mark reachable objects by walking the chain from the current epoch; we keep all messages
+	//    and chain headers; state and reciepts are retained only if it is within retention policy scope
+	log.Info("marking reachable objects")
+	startMark := time.Now()
+
+	count := new(int64)
+	err = s.walkChainDeep(curTs, retainStateP,
+		func(c cid.Cid) error {
+			if isUnitaryObject(c) {
+				return errStopWalk
+			}
+
+			mark, err := markSet.Has(c)
+			if err != nil {
+				return xerrors.Errorf("error checking markset: %w", err)
+			}
+
+			if mark {
+				return errStopWalk
+			}
+
+			atomic.AddInt64(count, 1)
+			return markSet.Mark(c)
+		})
+
+	if err != nil {
+		return xerrors.Errorf("error marking: %w", err)
+	}
+
+	log.Infow("marking done", "took", time.Since(startMark), "marked", count)
+
+	if err := s.checkClosing(); err != nil {
+		return err
+	}
+
+	// 1.1 protect transactional refs
+	err = s.protectTxnRefs(markSet)
+	if err != nil {
+		return xerrors.Errorf("error protecting transactional refs: %w", err)
+	}
+
+	if err := s.checkClosing(); err != nil {
+		return err
+	}
+
+	// 2. iterate through the coldstore to collect dead objects
+	log.Info("collecting dead objects")
+	startCollect := time.Now()
+
+	deadw, err := NewColdSetWriter(s.deadSetPath())
+	if err != nil {
+		return xerrors.Errorf("error creating coldset: %w", err)
+	}
+	defer deadw.Close() //nolint:errcheck
+
+	// some stats for logging
+	var liveCnt, deadCnt int
+
+	err = s.cold.(bstore.BlockstoreIterator).ForEachKey(func(c cid.Cid) error {
+		// was it marked?
+		mark, err := markSet.Has(c)
+		if err != nil {
+			return xerrors.Errorf("error checking mark set for %s: %w", c, err)
+		}
+
+		if mark {
+			liveCnt++
+			return nil
+		}
+
+		// Note: a possible optimization here is to also purge objects that are in the hotstore
+		//       but this needs special care not to purge genesis state, so we don't bother (yet)
+
+		// it's dead in the coldstore, mark it as candidate for purge
+
+		if err := deadw.Write(c); err != nil {
+			return xerrors.Errorf("error writing cid to coldstore: %w", err)
+		}
+		deadCnt++
+
+		return nil
+	})
+
+	if err != nil {
+		return xerrors.Errorf("error dead objects: %w", err)
+	}
+
+	if err := deadw.Close(); err != nil {
+		return xerrors.Errorf("error closing deadset: %w", err)
+	}
+
+	stats.Record(s.ctx, metrics.SplitstoreCompactionDead.M(int64(deadCnt)))
+
+	log.Infow("dead collection done", "took", time.Since(startCollect))
+	log.Infow("prune stats", "live", liveCnt, "dead", deadCnt)
+
+	if err := s.checkClosing(); err != nil {
+		return err
+	}
+
+	// now that we have collected dead objects, check for missing references from transactional i/o
+	// this is carried over from hot compaction for completeness
+	s.waitForMissingRefs(markSet)
+
+	if err := s.checkClosing(); err != nil {
+		return err
+	}
+
+	deadr, err := NewColdSetReader(s.deadSetPath())
+	if err != nil {
+		return xerrors.Errorf("error opening deadset: %w", err)
+	}
+	defer deadr.Close() //nolint:errcheck
+
+	// 3. Purge dead objects with checkpointing for recovery.
+	// This is the critical section of prune, whereby any dead object not in the markSet is
+	// considered already deleted.
+	// We delete dead objects in batches, holding the transaction lock, where we check the markSet
+	// again for new references created by the caller.
+	// After each batch we write a checkpoint to disk; if the process is interrupted before completion
+	// the process will continue from the checkpoint in the next recovery.
+	if err := s.beginCriticalSection(markSet); err != nil {
+		return xerrors.Errorf("error beginning critical section: %w", err)
+	}
+
+	if err := s.checkClosing(); err != nil {
+		return err
+	}
+
+	checkpoint, err := NewCheckpoint(s.pruneCheckpointPath())
+	if err != nil {
+		return xerrors.Errorf("error creating checkpoint: %w", err)
+	}
+	defer checkpoint.Close() //nolint:errcheck
+
+	log.Info("purging dead objects from the coldstore")
+	startPurge := time.Now()
+	err = s.purge(deadr, checkpoint, markSet)
+	if err != nil {
+		return xerrors.Errorf("error purging dead objects: %w", err)
+	}
+	log.Infow("purging dead objects from coldstore done", "took", time.Since(startPurge))
+
+	s.endCriticalSection()
+
+	if err := checkpoint.Close(); err != nil {
+		log.Warnf("error closing checkpoint: %s", err)
+	}
+	if err := os.Remove(s.pruneCheckpointPath()); err != nil {
+		log.Warnf("error removing checkpoint: %s", err)
+	}
+	if err := deadr.Close(); err != nil {
+		log.Warnf("error closing deadset: %s", err)
+	}
+	if err := os.Remove(s.deadSetPath()); err != nil {
+		log.Warnf("error removing deadset: %s", err)
+	}
+
+	// we are done; do some housekeeping
+	s.endTxnProtect()
+	err = doGC()
+	if err != nil {
+		log.Warnf("error garbage collecting cold store: %s", err)
+	}
+
+	if err := s.setPruneEpoch(boundaryEpoch); err != nil {
+		return xerrors.Errorf("error saving prune base epoch: %w", err)
+	}
+
+	s.pruneIndex++
+	err = s.ds.Put(s.ctx, pruneIndexKey, int64ToBytes(s.compactionIndex))
+	if err != nil {
+		return xerrors.Errorf("error saving compaction index: %w", err)
+	}
+
+	return nil
+}
+
+func (s *SplitStore) completePrune() error {
+	checkpoint, last, err := OpenCheckpoint(s.pruneCheckpointPath())
+	if err != nil {
+		return xerrors.Errorf("error opening checkpoint: %w", err)
+	}
+	defer checkpoint.Close() //nolint:errcheck
+
+	deadr, err := NewColdSetReader(s.deadSetPath())
+	if err != nil {
+		return xerrors.Errorf("error opening deadset: %w", err)
+	}
+	defer deadr.Close() //nolint:errcheck
+
+	markSet, err := s.markSetEnv.Recover("live")
+	if err != nil {
+		return xerrors.Errorf("error recovering markset: %w", err)
+	}
+	defer markSet.Close() //nolint:errcheck
+
+	// PURGE!
+	s.compactType = cold
+	log.Info("purging dead objects from the coldstore")
+	startPurge := time.Now()
+	err = s.completePurge(deadr, checkpoint, last, markSet)
+	if err != nil {
+		return xerrors.Errorf("error purgin dead objects: %w", err)
+	}
+	log.Infow("purging dead objects from the coldstore done", "took", time.Since(startPurge))
+
+	markSet.EndCriticalSection()
+	s.compactType = none
+
+	if err := checkpoint.Close(); err != nil {
+		log.Warnf("error closing checkpoint: %s", err)
+	}
+	if err := os.Remove(s.pruneCheckpointPath()); err != nil {
+		log.Warnf("error removing checkpoint: %s", err)
+	}
+	if err := deadr.Close(); err != nil {
+		log.Warnf("error closing deadset: %s", err)
+	}
+	if err := os.Remove(s.deadSetPath()); err != nil {
+		log.Warnf("error removing deadset: %s", err)
+	}
+
+	return nil
+}
+
+// like walkChain but peforms a deep walk, using parallel walking with walkObjectLax,
+// whereby all extant messages are retained and state roots are retained if they satisfy
+// the given predicate.
+// missing references are ignored, as we expect to have plenty for snapshot syncs.
+func (s *SplitStore) walkChainDeep(ts *types.TipSet, retainStateP func(int64) bool,
+	f func(cid.Cid) error) error {
+	visited := cid.NewSet()
+	toWalk := ts.Cids()
+	walkCnt := 0
+
+	workers := runtime.NumCPU() / 2
+	if workers < 2 {
+		workers = 2
+	}
+
+	var wg sync.WaitGroup
+	workch := make(chan cid.Cid, 16*workers)
+	errch := make(chan error, workers)
+
+	var once sync.Once
+	defer once.Do(func() { close(workch) })
+
+	push := func(c cid.Cid) error {
+		if !visited.Visit(c) {
+			return nil
+		}
+
+		select {
+		case workch <- c:
+			return nil
+		case err := <-errch:
+			return err
+		}
+	}
+
+	worker := func() {
+		defer wg.Done()
+		for c := range workch {
+			err := s.walkObjectLax(c, f)
+			if err != nil {
+				errch <- xerrors.Errorf("error walking object (cid: %s): %w", c, err)
+				return
+			}
+		}
+	}
+
+	for i := 0; i < workers; i++ {
+		wg.Add(1)
+		go worker()
+	}
+
+	baseEpoch := ts.Height()
+	minEpoch := baseEpoch // for progress report
+	log.Infof("walking at epoch %d", minEpoch)
+
+	walkBlock := func(c cid.Cid) error {
+		if !visited.Visit(c) {
+			return nil
+		}
+
+		walkCnt++
+
+		if err := f(c); err != nil {
+			return err
+		}
+
+		var hdr types.BlockHeader
+		err := s.view(c, func(data []byte) error {
+			return hdr.UnmarshalCBOR(bytes.NewBuffer(data))
+		})
+
+		if err != nil {
+			return xerrors.Errorf("error unmarshaling block header (cid: %s): %w", c, err)
+		}
+
+		if hdr.Height < minEpoch {
+			minEpoch = hdr.Height
+			if minEpoch%10_000 == 0 {
+				log.Infof("walking at epoch %d (walked: %d)", minEpoch, walkCnt)
+			}
+		}
+
+		depth := int64(baseEpoch - hdr.Height)
+		retainState := retainStateP(depth)
+
+		if hdr.Height > 0 {
+			if err := push(hdr.Messages); err != nil {
+				return err
+			}
+			if retainState {
+				if err := push(hdr.ParentMessageReceipts); err != nil {
+					return err
+				}
+			}
+		}
+
+		if retainState || hdr.Height == 0 {
+			if err := push(hdr.ParentStateRoot); err != nil {
+				return err
+			}
+		}
+
+		if hdr.Height > 0 {
+			toWalk = append(toWalk, hdr.Parents...)
+		}
+
+		return nil
+	}
+
+	for len(toWalk) > 0 {
+		// walking can take a while, so check this with every opportunity
+		if err := s.checkClosing(); err != nil {
+			return err
+		}
+
+		select {
+		case err := <-errch:
+			return err
+		default:
+		}
+
+		walking := toWalk
+		toWalk = nil
+		for _, c := range walking {
+			if err := walkBlock(c); err != nil {
+				return xerrors.Errorf("error walking block (cid: %s): %w", c, err)
+			}
+		}
+	}
+
+	once.Do(func() { close(workch) })
+	wg.Wait()
+	select {
+	case err := <-errch:
+		return err
+	default:
+	}
+
+	log.Infow("chain walk done", "walked", walkCnt)
+
+	return nil
+}
+
+// like walkObject but treats missing references laxly; faster version of walkObjectIncomplete
+// without an occurs check.
+func (s *SplitStore) walkObjectLax(c cid.Cid, f func(cid.Cid) error) error {
+	if err := f(c); err != nil {
+		if err == errStopWalk {
+			return nil
+		}
+
+		return err
+	}
+
+	if c.Prefix().Codec != cid.DagCBOR {
+		return nil
+	}
+
+	// check this before recursing
+	if err := s.checkClosing(); err != nil {
+		return err
+	}
+
+	var links []cid.Cid
+	err := s.view(c, func(data []byte) error {
+		return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
+			links = append(links, c)
+		})
+	})
+
+	if err != nil {
+		if ipld.IsNotFound(err) { // not a problem for deep walks
+			return nil
+		}
+
+		return xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err)
+	}
+
+	for _, c := range links {
+		err := s.walkObjectLax(c, f)
+		if err != nil {
+			return xerrors.Errorf("error walking link (cid: %s): %w", c, err)
+		}
+	}
+
+	return nil
+}

blockstore/splitstore/splitstore_reify.go

@@ -5,10 +5,9 @@ import (
 	"runtime"
 	"sync/atomic"
 
-	"golang.org/x/xerrors"
-
 	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
+	"golang.org/x/xerrors"
 )
 
 var (
@@ -84,7 +83,7 @@ func (s *SplitStore) reifyWorker(workch chan cid.Cid) {
 }
 
 func (s *SplitStore) doReify(c cid.Cid) {
-	var toreify, totrack, toforget []cid.Cid
+	var toreify, toforget []cid.Cid
 
 	defer func() {
 		s.reifyMx.Lock()
@@ -93,9 +92,6 @@ func (s *SplitStore) doReify(c cid.Cid) {
 		for _, c := range toreify {
 			delete(s.reifyInProgress, c)
 		}
-		for _, c := range totrack {
-			delete(s.reifyInProgress, c)
-		}
 		for _, c := range toforget {
 			delete(s.reifyInProgress, c)
 		}
@@ -132,19 +128,10 @@ func (s *SplitStore) doReify(c cid.Cid) {
 				return xerrors.Errorf("error checking hotstore: %w", err)
 			}
 
+			// All reified blocks are tracked at reification start
 			if has {
-				if s.txnMarkSet != nil {
-					hasMark, err := s.txnMarkSet.Has(c)
-					if err != nil {
-						log.Warnf("error checking markset: %s", err)
-					} else if hasMark {
-						toforget = append(toforget, c)
-						return errStopWalk
-					}
-				} else {
-					totrack = append(totrack, c)
-					return errStopWalk
-				}
+				toforget = append(toforget, c)
+				return errStopWalk
 			}
 
 			toreify = append(toreify, c)
@@ -156,7 +143,7 @@ func (s *SplitStore) doReify(c cid.Cid) {
 		})
 
 	if err != nil {
-		if xerrors.Is(err, errReifyLimit) {
+		if errors.Is(err, errReifyLimit) {
 			log.Debug("reification aborted; reify limit reached")
 			return
 		}
@@ -191,24 +178,4 @@ func (s *SplitStore) doReify(c cid.Cid) {
 		}
 	}
 
-	if s.txnMarkSet != nil {
-		if len(toreify) > 0 {
-			if err := s.txnMarkSet.MarkMany(toreify); err != nil {
-				log.Warnf("error marking reified objects: %s", err)
-			}
-		}
-		if len(totrack) > 0 {
-			if err := s.txnMarkSet.MarkMany(totrack); err != nil {
-				log.Warnf("error marking tracked objects: %s", err)
-			}
-		}
-	} else {
-		// if txnActive is false these are noops
-		if len(toreify) > 0 {
-			s.trackTxnRefMany(toreify)
-		}
-		if len(totrack) > 0 {
-			s.trackTxnRefMany(totrack)
-		}
-	}
 }

blockstore/splitstore/splitstore_test.go

@@ -1,4 +1,4 @@
-//stm: #unit
+// stm: #unit
 package splitstore
 
 import (
@@ -11,18 +11,20 @@ import (
 	"testing"
 	"time"
 
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-datastore"
+	dssync "github.com/ipfs/go-datastore/sync"
+	ipld "github.com/ipfs/go-ipld-format"
+	logging "github.com/ipfs/go-log/v2"
+	mh "github.com/multiformats/go-multihash"
+
 	"github.com/filecoin-project/go-state-types/abi"
+
 	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/types/mock"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-	datastore "github.com/ipfs/go-datastore"
-	dssync "github.com/ipfs/go-datastore/sync"
-	logging "github.com/ipfs/go-log/v2"
-	mh "github.com/multiformats/go-multihash"
 )
 
 func init() {
@@ -696,7 +698,7 @@ func (b *mockStore) Get(_ context.Context, cid cid.Cid) (blocks.Block, error) {
 
 	blk, ok := b.set[b.keyOf(cid)]
 	if !ok {
-		return nil, blockstore.ErrNotFound
+		return nil, ipld.ErrNotFound{Cid: cid}
 	}
 	return blk, nil
 }

blockstore/splitstore/splitstore_util.go

@@ -3,10 +3,9 @@ package splitstore
 import (
 	"encoding/binary"
 
-	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 	mh "github.com/multiformats/go-multihash"
+	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-state-types/abi"
 )

blockstore/splitstore/splitstore_warmup.go

@@ -5,13 +5,13 @@ import (
 	"sync/atomic"
 	"time"
 
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
 	"golang.org/x/xerrors"
 
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-
 	"github.com/filecoin-project/go-state-types/abi"
-	bstore "github.com/filecoin-project/lotus/blockstore"
+
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 )
@@ -21,14 +21,14 @@ var (
 	WarmupBoundary = build.Finality
 )
 
-// warmup acuiqres the compaction lock and spawns a goroutine to warm up the hotstore;
+// warmup acquires the compaction lock and spawns a goroutine to warm up the hotstore;
 // this is necessary when we sync from a snapshot or when we enable the splitstore
 // on top of an existing blockstore (which becomes the coldstore).
 func (s *SplitStore) warmup(curTs *types.TipSet) error {
 	if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
 		return xerrors.Errorf("error locking compaction")
 	}
-
+	s.compactType = warmup
 	go func() {
 		defer atomic.StoreInt32(&s.compacting, 0)
 
@@ -88,7 +88,7 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 
 			blk, err := s.cold.Get(s.ctx, c)
 			if err != nil {
-				if err == bstore.ErrNotFound {
+				if ipld.IsNotFound(err) {
 					atomic.AddInt64(missing, 1)
 					return errStopWalk
 				}

blockstore/splitstore/visitor.go

@@ -3,7 +3,7 @@ package splitstore
 import (
 	"sync"
 
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 )
 
 // ObjectVisitor is an interface for deduplicating objects during walks

blockstore/timed.go

@@ -8,6 +8,7 @@ import (
 
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
 	"github.com/raulk/clock"
 	"go.uber.org/multierr"
 )
@@ -112,7 +113,7 @@ func (t *TimedCacheBlockstore) View(ctx context.Context, k cid.Cid, callback fun
 	// calling an arbitrary callback while holding a lock.
 	t.mu.RLock()
 	block, err := t.active.Get(ctx, k)
-	if err == ErrNotFound {
+	if ipld.IsNotFound(err) {
 		block, err = t.inactive.Get(ctx, k)
 	}
 	t.mu.RUnlock()
@@ -127,7 +128,7 @@ func (t *TimedCacheBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 	b, err := t.active.Get(ctx, k)
-	if err == ErrNotFound {
+	if ipld.IsNotFound(err) {
 		b, err = t.inactive.Get(ctx, k)
 	}
 	return b, err
@@ -137,7 +138,7 @@ func (t *TimedCacheBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, err
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 	size, err := t.active.GetSize(ctx, k)
-	if err == ErrNotFound {
+	if ipld.IsNotFound(err) {
 		size, err = t.inactive.GetSize(ctx, k)
 	}
 	return size, err
@@ -175,11 +176,12 @@ func (t *TimedCacheBlockstore) AllKeysChan(_ context.Context) (<-chan cid.Cid, e
 	defer t.mu.RUnlock()
 
 	ch := make(chan cid.Cid, len(t.active)+len(t.inactive))
-	for c := range t.active {
-		ch <- c
+	for _, b := range t.active {
+		ch <- b.Cid()
 	}
-	for c := range t.inactive {
-		if _, ok := t.active[c]; ok {
+	for _, b := range t.inactive {
+		c := b.Cid()
+		if _, ok := t.active[string(c.Hash())]; ok {
 			continue
 		}
 		ch <- c

blockstore/timed_test.go

@@ -1,4 +1,4 @@
-//stm: #unit
+// stm: #unit
 package blockstore
 
 import (
@@ -6,11 +6,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/raulk/clock"
-	"github.com/stretchr/testify/require"
-
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	"github.com/raulk/clock"
+	"github.com/stretchr/testify/require"
 )
 
 func TestTimedCacheBlockstoreSimple(t *testing.T) {

blockstore/union.go

@@ -5,16 +5,16 @@ import (
 
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
 )
 
 type unionBlockstore []Blockstore
 
 // Union returns an unioned blockstore.
 //
-// * Reads return from the first blockstore that has the value, querying in the
-//   supplied order.
-// * Writes (puts and deltes) are broadcast to all stores.
-//
+//   - Reads return from the first blockstore that has the value, querying in the
+//     supplied order.
+//   - Writes (puts and deletes) are broadcast to all stores.
 func Union(stores ...Blockstore) Blockstore {
 	return unionBlockstore(stores)
 }
@@ -30,7 +30,7 @@ func (m unionBlockstore) Has(ctx context.Context, cid cid.Cid) (has bool, err er
 
 func (m unionBlockstore) Get(ctx context.Context, cid cid.Cid) (blk blocks.Block, err error) {
 	for _, bs := range m {
-		if blk, err = bs.Get(ctx, cid); err == nil || err != ErrNotFound {
+		if blk, err = bs.Get(ctx, cid); err == nil || !ipld.IsNotFound(err) {
 			break
 		}
 	}
@@ -39,7 +39,7 @@ func (m unionBlockstore) Get(ctx context.Context, cid cid.Cid) (blk blocks.Block
 
 func (m unionBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) (err error) {
 	for _, bs := range m {
-		if err = bs.View(ctx, cid, callback); err == nil || err != ErrNotFound {
+		if err = bs.View(ctx, cid, callback); err == nil || !ipld.IsNotFound(err) {
 			break
 		}
 	}
@@ -48,7 +48,7 @@ func (m unionBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]
 
 func (m unionBlockstore) GetSize(ctx context.Context, cid cid.Cid) (size int, err error) {
 	for _, bs := range m {
-		if size, err = bs.GetSize(ctx, cid); err == nil || err != ErrNotFound {
+		if size, err = bs.GetSize(ctx, cid); err == nil || !ipld.IsNotFound(err) {
 			break
 		}
 	}

blockstore/union_test.go

@@ -1,4 +1,4 @@
-//stm: #unit
+// stm: #unit
 package blockstore
 
 import (

build/bootstrap.go

@@ -6,9 +6,9 @@ import (
 	"path"
 	"strings"
 
-	"github.com/filecoin-project/lotus/lib/addrutil"
+	"github.com/libp2p/go-libp2p/core/peer"
 
-	"github.com/libp2p/go-libp2p-core/peer"
+	"github.com/filecoin-project/lotus/lib/addrutil"
 )
 
 //go:embed bootstrap

build/bootstrap/butterflynet.pi

@@ -1,2 +1,2 @@
-/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWSUZhAY3eyoPUboJ1ZWe4dNPFWTr1EPoDjbTDSAN15uhY
-/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWDfvNrSRVGWAGbn3sm9C8z98W2x25qCZjaXGHXmGiH24e
+/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWHc8xB2S1wFeF9ar9bVdXoEEaBPGLqfKxVQH55c4nNmxs
+/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWPcNcwS3cKarWrN7MfANWNpzXmZA9Ag6eH9FHFdLQ3LFQ

build/bootstrap/interopnet.pi

@@ -1,2 +1,2 @@
-/dns4/bootstrap-0.interop.fildev.network/tcp/1347/p2p/12D3KooWDY249nj6gxSiBTocNBnkbBBvnfPgkxq5SBUVrccjwRnr
-/dns4/bootstrap-1.interop.fildev.network/tcp/1347/p2p/12D3KooWCwFsKt4NyoJQk7vjiACXL9LznBWANXKjCebKJ7MHh3h4
+/dns4/bootstrap-0.interop.fildev.network/tcp/1347/p2p/12D3KooWDpppr8csCNvEPnD2Z83KTPdBTM7iJhL66qK8LK3bB5NU
+/dns4/bootstrap-1.interop.fildev.network/tcp/1347/p2p/12D3KooWR3K1sXWoDYcXWqDF26mFEM1o1g7e7fcVR3NYE7rn24Gs

build/bootstrap/mainnet.pi

@@ -13,4 +13,4 @@
 /dns4/node.glif.io/tcp/1235/p2p/12D3KooWBF8cpp65hp2u9LK5mh19x67ftAam84z9LsfaquTDSBpt
 /dns4/bootstrap-0.ipfsmain.cn/tcp/34721/p2p/12D3KooWQnwEGNqcM2nAcPtRR9rAX8Hrg4k9kJLCHoTR5chJfz6d
 /dns4/bootstrap-1.ipfsmain.cn/tcp/34723/p2p/12D3KooWMKxMkD5DMpSWsW7dBddKxKT7L2GgbNuckz9otxvkvByP
-/dns4/bootstarp-0.1475.io/tcp/61256/p2p/12D3KooWQjaNmbz9b1XmheQB3RWsRjKSzuRLfjeiDZHyX7Y5RcBr
+/dns4/bootstarp-0.1475.io/tcp/61256/p2p/12D3KooWRzCVDwHUkgdK7eRgnoXbjDAELhxPErjHzbRLguSV1aRt

build/builtin_actors.go

@@ -12,15 +12,17 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/filecoin-project/lotus/blockstore"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/actors/adt"
-
 	"github.com/DataDog/zstd"
 	"github.com/ipfs/go-cid"
 	cbor "github.com/ipfs/go-ipld-cbor"
 	"github.com/ipld/go-car"
 	"golang.org/x/xerrors"
+
+	actorstypes "github.com/filecoin-project/go-state-types/actors"
+
+	"github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/adt"
 )
 
 //go:embed actors/*.tar.zst
@@ -28,14 +30,14 @@ var embeddedBuiltinActorReleases embed.FS
 
 func init() {
 	if BundleOverrides == nil {
-		BundleOverrides = make(map[actors.Version]string)
+		BundleOverrides = make(map[actorstypes.Version]string)
 	}
 	for _, av := range actors.Versions {
 		path := os.Getenv(fmt.Sprintf("LOTUS_BUILTIN_ACTORS_V%d_BUNDLE", av))
 		if path == "" {
 			continue
 		}
-		BundleOverrides[actors.Version(av)] = path
+		BundleOverrides[actorstypes.Version(av)] = path
 	}
 	if err := loadManifests(NetworkBundle); err != nil {
 		panic(err)
@@ -55,7 +57,7 @@ func UseNetworkBundle(netw string) error {
 }
 
 func loadManifests(netw string) error {
-	overridden := make(map[actors.Version]struct{})
+	overridden := make(map[actorstypes.Version]struct{})
 	var newMetadata []*BuiltinActorsMetadata
 	// First, prefer overrides.
 	for av, path := range BundleOverrides {
@@ -94,7 +96,7 @@ func loadManifests(netw string) error {
 
 type BuiltinActorsMetadata struct {
 	Network     string
-	Version     actors.Version
+	Version     actorstypes.Version
 	ManifestCid cid.Cid
 	Actors      map[string]cid.Cid
 }
@@ -182,7 +184,7 @@ func readEmbeddedBuiltinActorsMetadata(bundle string) ([]*BuiltinActorsMetadata,
 		}
 		bundles = append(bundles, &BuiltinActorsMetadata{
 			Network:     name,
-			Version:     actors.Version(version),
+			Version:     actorstypes.Version(version),
 			ManifestCid: root,
 			Actors:      actorCids,
 		})
@@ -229,7 +231,7 @@ func readBundleManifest(r io.Reader) (cid.Cid, map[string]cid.Cid, error) {
 }
 
 // GetEmbeddedBuiltinActorsBundle returns the builtin-actors bundle for the given actors version.
-func GetEmbeddedBuiltinActorsBundle(version actors.Version) ([]byte, bool) {
+func GetEmbeddedBuiltinActorsBundle(version actorstypes.Version) ([]byte, bool) {
 	fi, err := embeddedBuiltinActorReleases.Open(fmt.Sprintf("actors/v%d.tar.zst", version))
 	if err != nil {
 		return nil, false

build/builtin_actors_gen.go

@@ -23,6 +23,23 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 		"system":           MustParseCid("bafk2bzacec6xctjxybp7r3kkhase56o6jsaiua7ure5ttu2xfuojt4jhlsoa6"),
 		"verifiedregistry": MustParseCid("bafk2bzacec2hcqlqcfacylfcrhhliwkisvh4y3adwt47xkf2gdvodwu6ccepc"),
 	},
+}, {
+	Network:     "butterflynet",
+	Version:     9,
+	ManifestCid: MustParseCid("bafy2bzacebor5miojgkvov7gtiirg7hqxgm54x3hxytbho3qloqryg2r67qaq"),
+	Actors: map[string]cid.Cid{
+		"account":          MustParseCid("bafk2bzacecd7kgfzbyu4sylskgf7wjqabo43y3d7xqgvz7r67pzr3ovn3sj7y"),
+		"cron":             MustParseCid("bafk2bzacea6dc2a6skmp7saa43dcofp7zvtklvya75bai2k4k6qnkpcfzklug"),
+		"init":             MustParseCid("bafk2bzacean5v7lbsspxmdkcwhac44sk332uwyfteibtxyhgoeabwbzbh736c"),
+		"multisig":         MustParseCid("bafk2bzacea43lypvbof5kwnk2flzrwdrqqvzypom5xyheeuaab6dmuiafngae"),
+		"paymentchannel":   MustParseCid("bafk2bzacecsdimcgztl4tfodnqc35jojoy4mzaqqidubialqmqaa3zpv4h3rw"),
+		"reward":           MustParseCid("bafk2bzacedy3utj67mewe7rpieaex3sf7qxfkgiwtlgg4wlqyl74la65njbn2"),
+		"storagemarket":    MustParseCid("bafk2bzacec4xpazf3l2p5qkknjsku2sd3skqevk7gxryumbelwhfbs7mgjnpa"),
+		"storageminer":     MustParseCid("bafk2bzaceb5scwcdcou56vyhxq3c7vcefjau44v2h6nvsus45wqopt3hadzae"),
+		"storagepower":     MustParseCid("bafk2bzacebxl4ssugntkckifkq6sydczcas463bmonwi2m5hddurzzcukdzyg"),
+		"system":           MustParseCid("bafk2bzacedtw2ebtl3uccbqfcm6jd7teinlpktcglopuaxa6ev334jhs46hso"),
+		"verifiedregistry": MustParseCid("bafk2bzaceds62ys6rpbtjjsemohalxnkr3jtqhflifabn3xlhpi45mvi757po"),
+	},
 }, {
 	Network:     "calibrationnet",
 	Version:     8,
@@ -40,6 +57,23 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 		"system":           MustParseCid("bafk2bzaceaqrkllksxv2jsfgjvmuewx5vbzrammw5mdscod6gkdr3ijih2q64"),
 		"verifiedregistry": MustParseCid("bafk2bzaceaihibfu625lbtzdp3tcftscshrmbgghgrc7kzqhxn4455pycpdkm"),
 	},
+}, {
+	Network:     "calibrationnet",
+	Version:     9,
+	ManifestCid: MustParseCid("bafy2bzacectdlm3f5m252s6ken3ncnndvwj4a7hqyev2m7iba4lubkt72r2rm"),
+	Actors: map[string]cid.Cid{
+		"account":          MustParseCid("bafk2bzaceag3pkyy5youg2i4qyrhqms2jobycddkhd4eebtlkeqj4wu4sw7ji"),
+		"cron":             MustParseCid("bafk2bzaceaii7ryi4sjqqrrwyqaxulzqgiigkjqfa3f3ghhisleaqrinyal4o"),
+		"init":             MustParseCid("bafk2bzacebpi37hcleu7cpqgs5zdg4zsw7qpydt5w25jny2dyyscsyidquvcw"),
+		"multisig":         MustParseCid("bafk2bzacebt522zme5n3q25mecukjvomfyyzduxormgnprl73mdb5ekda2ywy"),
+		"paymentchannel":   MustParseCid("bafk2bzacebdxmtmloxfyhxkoj4p4ze5gojqohvqjmdmv2t54xyrgfnjs52p26"),
+		"reward":           MustParseCid("bafk2bzacebt7vup2pdxie67goc5ourij32z4ofejlrbv7styzm47ytvmmxx5a"),
+		"storagemarket":    MustParseCid("bafk2bzacebeohybortft5jiihfvfly5oiygvf2hulnuif7vivklind32hbr3c"),
+		"storageminer":     MustParseCid("bafk2bzaceallhekg5fhibexjurqcwrx6uzacbyq5eofic6jbdiu3vhepq6xsw"),
+		"storagepower":     MustParseCid("bafk2bzaceassqqdljvblivbcrkv62a36gxjwf7dmkgez6ezq2oydx3dv2b3z2"),
+		"system":           MustParseCid("bafk2bzacebxsecqifqvqxavnmsuwigmnc5rcuxoezspmykeqinkbgrkfukmgs"),
+		"verifiedregistry": MustParseCid("bafk2bzacedrmxfk5qifi7ujcb7mm73mwoy5e7nbndxhbqauuyol6cunf6popm"),
+	},
 }, {
 	Network:     "caterpillarnet",
 	Version:     8,
@@ -57,6 +91,23 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 		"system":           MustParseCid("bafk2bzacebu47th3xerlngqavlipb6cfu2utljkxxzgadc3totogto2tmx2jc"),
 		"verifiedregistry": MustParseCid("bafk2bzaceci3niq3rmbcmepgn27zvlgci6d5t4dvthx3pbmmx3wcu5elova6i"),
 	},
+}, {
+	Network:     "caterpillarnet",
+	Version:     9,
+	ManifestCid: MustParseCid("bafy2bzaceaknnblpmv3qqeqlgdycpps2vkgbgkflugbqz47k6zwrqte54hqy2"),
+	Actors: map[string]cid.Cid{
+		"account":          MustParseCid("bafk2bzaceckajm6tcdpvfsrvagduv6cuefuo3modkpfupum6ibspulgvporrw"),
+		"cron":             MustParseCid("bafk2bzacea74zhlldpihms2ohp3wvefojifvudbblmeju3izwrpcpxemp43fo"),
+		"init":             MustParseCid("bafk2bzacea5nqh4l76vm4ys6vgrqyt77tquk66il3w5ho3zxugka22fipdkdw"),
+		"multisig":         MustParseCid("bafk2bzacea3ftmx2roav4psh63r5n2sol6x3kwpmfwe473pexhj6lg7pruu2k"),
+		"paymentchannel":   MustParseCid("bafk2bzacea2gqhnycejjfuinywoz4j7pmsvxn3i2y5epdx5vqpo23phx6jeac"),
+		"reward":           MustParseCid("bafk2bzaceamdxogvus7zawqtju3vesmj6ffpf3ta6dlwvvstyfbf23sow2tes"),
+		"storagemarket":    MustParseCid("bafk2bzacecvh6qhfe3ityehpibxmmw43l7recwtru3l4n3yrdj5bmnv4efnrc"),
+		"storageminer":     MustParseCid("bafk2bzaced6ya5wlvplkvnvmhuc2uh5zq7oft6pyyoouwouuoreuk6grn6vsw"),
+		"storagepower":     MustParseCid("bafk2bzacedbfoblr65vbqwx37cym6v3zhbgrau46vma4c2lbh6toz6q5dyrtc"),
+		"system":           MustParseCid("bafk2bzaced7z2lu5hp4qpsesbmubq3chrad3sl6kgveldafksh4w252ijjaw2"),
+		"verifiedregistry": MustParseCid("bafk2bzacec6fdlcvlnaiktdptogae3bgofhzo24lhn4hs6rsa2zbnknfpsct6"),
+	},
 }, {
 	Network:     "devnet",
 	Version:     8,
@@ -74,6 +125,23 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 		"system":           MustParseCid("bafk2bzacecf7eta2stfd3cnuxzervd33imbvlaqq6b5tsho7pxmhifrybreru"),
 		"verifiedregistry": MustParseCid("bafk2bzaceaajgtglewgitshgdi2nzrvq7eihjtyqj5yiamesqun2hujl3xev2"),
 	},
+}, {
+	Network:     "devnet",
+	Version:     9,
+	ManifestCid: MustParseCid("bafy2bzacecogd7hb3dap7d4abyikdgt6b2zs46s4qyilsgxk4rsienbvvstp4"),
+	Actors: map[string]cid.Cid{
+		"account":          MustParseCid("bafk2bzacecrgqcwn2j6c44tnfx32om4upg56b7gdwslfgeuyfsm6iwkiuhm2w"),
+		"cron":             MustParseCid("bafk2bzaceblepwha5i5cpac37arpcxjcebm5k7p4e5ed7bwcq7ie6fryxgno4"),
+		"init":             MustParseCid("bafk2bzacebjr3xwsdzfie47rtffibt3irr7a5qqstt7fsbwnolzz3z6oy5frw"),
+		"multisig":         MustParseCid("bafk2bzacec66uf5u7o7q3u3pufrtufya6ntwrreysrwwlzqdjbtj36ajuamz4"),
+		"paymentchannel":   MustParseCid("bafk2bzacebouukxeahifxfoymmvibngg6abftd7luiq5x2c7zxulvqyerwoeg"),
+		"reward":           MustParseCid("bafk2bzacecj4pry2xb6w4kd4xxvs3dqd2endfynnwrkf66zakmt376tafy6ym"),
+		"storagemarket":    MustParseCid("bafk2bzacecmij6fgaasgu7zm354fltmvhwmepzwadhl4euxhb64yg4pgfkxgw"),
+		"storageminer":     MustParseCid("bafk2bzaceb5qyhchlvqvcq5cvrc5tuyjrrz6qkgbbxx4vr4wxhtoa4bfp56bg"),
+		"storagepower":     MustParseCid("bafk2bzaced377v52henedhmudi2isuxe73mp2lrv2zyyxfp4r4uppz3sy2ipw"),
+		"system":           MustParseCid("bafk2bzacebb3gdmg2fqexijivo7dkxj6g6gu7ou2jsw6opszr5gfmwt4fvorw"),
+		"verifiedregistry": MustParseCid("bafk2bzacec5xfb3a56mlclymhvs2br3qzx7sagqda4dvl4xz3fgiymkkbz5tk"),
+	},
 }, {
 	Network:     "mainnet",
 	Version:     8,
@@ -91,6 +159,23 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 		"system":           MustParseCid("bafk2bzacedwq5uppsw7vp55zpj7jdieizirmldceehu6wvombw3ixq2tcq57w"),
 		"verifiedregistry": MustParseCid("bafk2bzaceb3zbkjz3auizmoln2unmxep7dyfcmsre64vnqfhdyh7rkqfoxlw4"),
 	},
+}, {
+	Network:     "mainnet",
+	Version:     9,
+	ManifestCid: MustParseCid("bafy2bzaceb7nrn4icde3yt7s2kpllliodblq2i3iuslh3l6swgjqyusual72u"),
+	Actors: map[string]cid.Cid{
+		"account":          MustParseCid("bafk2bzacedchhykcbvptlrnts62xa2h7fewsyvb4imktudll7tbogkwj4ccpy"),
+		"cron":             MustParseCid("bafk2bzaced2e2ivtglevf5tnqay6eqdl3m5ilx4az766ojbhmy2kbxmkogbni"),
+		"init":             MustParseCid("bafk2bzaceapzqn6e3cfpf4ntln4dkyrqe5egygmjczefduy7fuabxh2npqq6o"),
+		"multisig":         MustParseCid("bafk2bzacebxihi4x4lzelmmdtxiz5gub3cjgthrgxuj5mxl2ihxaz6qhp5qbu"),
+		"paymentchannel":   MustParseCid("bafk2bzacecv34ahs6k552won2vj4r62gur5i7inqnycg7y4dg27ktlq76stlk"),
+		"reward":           MustParseCid("bafk2bzacea4er76nlnzz3muuvdxs4dbygejztezanpafi42idj57n75sumu7o"),
+		"storagemarket":    MustParseCid("bafk2bzacedd3pjxcwgxfli4a5udw3ebbo4gom43epizbldvto7ovduqk6udj6"),
+		"storageminer":     MustParseCid("bafk2bzacebtuopxp2464fxulzfyr4iw7bj4pelfxlatudaf6q5qtahokeo3lo"),
+		"storagepower":     MustParseCid("bafk2bzacec3dmcwtizgy6qhxg6fgb4ykc2pamxxh2n5oyeqvebjf3ikzd2sui"),
+		"system":           MustParseCid("bafk2bzacecrm36gf2gusruvhfc6ukf36qflsr5pdbquvfyysprdyudsatoufm"),
+		"verifiedregistry": MustParseCid("bafk2bzaceah3mwu6tb4jmffohpoz6nyoxpgno2hmxz455rycw4o2hrat3yg3u"),
+	},
 }, {
 	Network:     "testing",
 	Version:     8,
@@ -108,6 +193,23 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 		"system":           MustParseCid("bafk2bzaced6kjkbv7lrb2qwq5we2hqaxc6ztch5p52g27qtjy45zdemsk4b7m"),
 		"verifiedregistry": MustParseCid("bafk2bzacectzxvtoselhnzsair5nv6k5vokvegnht6z2lfee4p3xexo4kg4m6"),
 	},
+}, {
+	Network:     "testing",
+	Version:     9,
+	ManifestCid: MustParseCid("bafy2bzaceas5fka4rtyvnqwvvwdi2qqhbnu7hnyz37chhpflh4rdbwnoupols"),
+	Actors: map[string]cid.Cid{
+		"account":          MustParseCid("bafk2bzacecsmbmd42fsg33wd3u5vxyp4n4a4mac423fduauishjhiyv2bqz7w"),
+		"cron":             MustParseCid("bafk2bzacedajlwnkefucjxzasudkfojrz3bnhnnirebvacxrtly5y2kfb7qiu"),
+		"init":             MustParseCid("bafk2bzaceat7cvf7s2vnmqoldzafkpjygi6wv3sfnclenfbahk4htsgu4aqsa"),
+		"multisig":         MustParseCid("bafk2bzacebh7pwb7eoyigwgjs2r32ebtxwbwgxqd4cvou4katq4axmhd7lxsq"),
+		"paymentchannel":   MustParseCid("bafk2bzaceb6nbmyo2girdpnwywby7yzelx7g6s4snbq3kb3mmd2pp33qefkww"),
+		"reward":           MustParseCid("bafk2bzacedttgmuddtlrp2ie4i2vzuueury455knof5muqtqlslscxrk7ucl6"),
+		"storagemarket":    MustParseCid("bafk2bzacebjtz7fg5n2sm4wg464fyg5vcvux5hqrsdwki675hsquhh3rpe5rg"),
+		"storageminer":     MustParseCid("bafk2bzacea2c2genvtsktr3f6yjlny57gwqyt5dbqrfws56wn5ssyl2tevjjg"),
+		"storagepower":     MustParseCid("bafk2bzaceasmr5xc3c7jgihqcb5pdv557kfor72wlmzmh4syvedknelebhsve"),
+		"system":           MustParseCid("bafk2bzaceabj3whuno364pjqn7sphwquiqpl2w5le6aa6quwytgyxshvelcwq"),
+		"verifiedregistry": MustParseCid("bafk2bzaceamn6gzihkytyqym4dqs7wqmo2fz6kpge4tphumvutsp2ys4bwaha"),
+	},
 }, {
 	Network:     "testing-fake-proofs",
 	Version:     8,
@@ -125,4 +227,21 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 		"system":           MustParseCid("bafk2bzaced6kjkbv7lrb2qwq5we2hqaxc6ztch5p52g27qtjy45zdemsk4b7m"),
 		"verifiedregistry": MustParseCid("bafk2bzacectzxvtoselhnzsair5nv6k5vokvegnht6z2lfee4p3xexo4kg4m6"),
 	},
+}, {
+	Network:     "testing-fake-proofs",
+	Version:     9,
+	ManifestCid: MustParseCid("bafy2bzacebr7bo4shotwq4cct3ablt6hlcozg4h2e52av2q3md5di7sibrqj2"),
+	Actors: map[string]cid.Cid{
+		"account":          MustParseCid("bafk2bzacecsmbmd42fsg33wd3u5vxyp4n4a4mac423fduauishjhiyv2bqz7w"),
+		"cron":             MustParseCid("bafk2bzacedajlwnkefucjxzasudkfojrz3bnhnnirebvacxrtly5y2kfb7qiu"),
+		"init":             MustParseCid("bafk2bzaceat7cvf7s2vnmqoldzafkpjygi6wv3sfnclenfbahk4htsgu4aqsa"),
+		"multisig":         MustParseCid("bafk2bzacebh7pwb7eoyigwgjs2r32ebtxwbwgxqd4cvou4katq4axmhd7lxsq"),
+		"paymentchannel":   MustParseCid("bafk2bzaceb6nbmyo2girdpnwywby7yzelx7g6s4snbq3kb3mmd2pp33qefkww"),
+		"reward":           MustParseCid("bafk2bzacedttgmuddtlrp2ie4i2vzuueury455knof5muqtqlslscxrk7ucl6"),
+		"storagemarket":    MustParseCid("bafk2bzacebjtz7fg5n2sm4wg464fyg5vcvux5hqrsdwki675hsquhh3rpe5rg"),
+		"storageminer":     MustParseCid("bafk2bzacedaxvye4reegpy2fecjuzwt4nly2myhplnvkzhssquo2jjw7vfmuy"),
+		"storagepower":     MustParseCid("bafk2bzaceaoteuuvld3bzy3t3kzjp5iabgkvxjc5cxq2ziwankpxqqixkqqjw"),
+		"system":           MustParseCid("bafk2bzaceabj3whuno364pjqn7sphwquiqpl2w5le6aa6quwytgyxshvelcwq"),
+		"verifiedregistry": MustParseCid("bafk2bzaceamn6gzihkytyqym4dqs7wqmo2fz6kpge4tphumvutsp2ys4bwaha"),
+	},
 }}

build/builtin_actors_test.go

@@ -3,9 +3,12 @@ package build_test
 import (
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
+	actorstypes "github.com/filecoin-project/go-state-types/actors"
+
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/stretchr/testify/require"
 )
 
 // Test that the embedded metadata is correct.
@@ -18,16 +21,16 @@ func TestEmbeddedMetadata(t *testing.T) {
 
 // Test that we're registering the manifest correctly.
 func TestRegistration(t *testing.T) {
-	manifestCid, found := actors.GetManifest(actors.Version8)
+	manifestCid, found := actors.GetManifest(actorstypes.Version8)
 	require.True(t, found)
 	require.True(t, manifestCid.Defined())
 
 	for _, key := range actors.GetBuiltinActorsKeys() {
-		actorCid, found := actors.GetActorCodeID(actors.Version8, key)
+		actorCid, found := actors.GetActorCodeID(actorstypes.Version8, key)
 		require.True(t, found)
 		name, version, found := actors.GetActorMetaByCode(actorCid)
 		require.True(t, found)
-		require.Equal(t, actors.Version8, version)
+		require.Equal(t, actorstypes.Version8, version)
 		require.Equal(t, key, name)
 	}
 }

build/openrpc.go

@@ -52,3 +52,11 @@ func OpenRPCDiscoverJSON_Worker() apitypes.OpenRPCDocument {
 	}
 	return mustReadGzippedOpenRPCDocument(data)
 }
+
+func OpenRPCDiscoverJSON_Gateway() apitypes.OpenRPCDocument {
+	data, err := openrpcfs.ReadFile("openrpc/gateway.json.gz")
+	if err != nil {
+		panic(err)
+	}
+	return mustReadGzippedOpenRPCDocument(data)
+}

build/openrpc_test.go

@@ -1,4 +1,4 @@
-//stm: #unit
+// stm: #unit
 package build
 
 import (
@@ -16,6 +16,7 @@ func TestOpenRPCDiscoverJSON_Version(t *testing.T) {
 		OpenRPCDiscoverJSON_Full,
 		OpenRPCDiscoverJSON_Miner,
 		OpenRPCDiscoverJSON_Worker,
+		OpenRPCDiscoverJSON_Gateway,
 	} {
 		doc := docFn()
 		if got, ok := doc["openrpc"]; !ok || got != openRPCDocVersion {

build/params_2k.go

@@ -10,9 +10,9 @@ import (
 	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/go-state-types/abi"
+	actorstypes "github.com/filecoin-project/go-state-types/actors"
 	"github.com/filecoin-project/go-state-types/network"
 
-	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
 )
 
@@ -20,7 +20,7 @@ const BootstrappersFile = ""
 const GenesisFile = ""
 
 var NetworkBundle = "devnet"
-var BundleOverrides map[actors.Version]string
+var BundleOverrides map[actorstypes.Version]string
 
 const GenesisNetworkVersion = network.Version16
 
@@ -53,8 +53,11 @@ var UpgradeHyperdriveHeight = abi.ChainEpoch(-16)
 var UpgradeChocolateHeight = abi.ChainEpoch(-17)
 
 var UpgradeOhSnapHeight = abi.ChainEpoch(-18)
+
 var UpgradeSkyrHeight = abi.ChainEpoch(-19)
 
+var UpgradeV17Height = abi.ChainEpoch(99999999999999)
+
 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 	0: DrandMainnet,
 }
@@ -106,6 +109,7 @@ func init() {
 	UpgradeChocolateHeight = getUpgradeHeight("LOTUS_CHOCOLATE_HEIGHT", UpgradeChocolateHeight)
 	UpgradeOhSnapHeight = getUpgradeHeight("LOTUS_OHSNAP_HEIGHT", UpgradeOhSnapHeight)
 	UpgradeSkyrHeight = getUpgradeHeight("LOTUS_SKYR_HEIGHT", UpgradeSkyrHeight)
+	UpgradeV17Height = getUpgradeHeight("LOTUS_V17_HEIGHT", UpgradeV17Height)
 
 	BuildType |= Build2k
 

build/params_butterfly.go

@@ -4,23 +4,25 @@
 package build
 
 import (
+	"github.com/ipfs/go-cid"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	actorstypes "github.com/filecoin-project/go-state-types/actors"
 	"github.com/filecoin-project/go-state-types/network"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/actors/policy"
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
-	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/lotus/chain/actors/policy"
 )
 
 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 	0: DrandMainnet,
 }
 
-const GenesisNetworkVersion = network.Version15
+const GenesisNetworkVersion = network.Version16
 
 var NetworkBundle = "butterflynet"
-var BundleOverrides map[actors.Version]string
+var BundleOverrides map[actorstypes.Version]string
 
 const BootstrappersFile = "butterflynet.pi"
 const GenesisFile = "butterflynet.car"
@@ -45,10 +47,9 @@ const UpgradeNorwegianHeight = -14
 const UpgradeTurboHeight = -15
 const UpgradeHyperdriveHeight = -16
 const UpgradeChocolateHeight = -17
-
 const UpgradeOhSnapHeight = -18
-
-const UpgradeSkyrHeight = abi.ChainEpoch(50)
+const UpgradeSkyrHeight = -19
+const UpgradeV17Height = abi.ChainEpoch(99999999999999)
 
 var SupportedProofTypes = []abi.RegisteredSealProof{
 	abi.RegisteredSealProof_StackedDrg512MiBV1,

build/params_calibnet.go

@@ -4,13 +4,18 @@
 package build
 
 import (
+	"os"
+	"strconv"
+
+	"github.com/ipfs/go-cid"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	actorstypes "github.com/filecoin-project/go-state-types/actors"
 	"github.com/filecoin-project/go-state-types/network"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/actors/policy"
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
-	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/lotus/chain/actors/policy"
 )
 
 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
@@ -20,7 +25,7 @@ var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 const GenesisNetworkVersion = network.Version0
 
 var NetworkBundle = "calibrationnet"
-var BundleOverrides map[actors.Version]string
+var BundleOverrides map[actorstypes.Version]string
 
 const BootstrappersFile = "calibnet.pi"
 const GenesisFile = "calibnet.car"
@@ -64,6 +69,8 @@ const UpgradeOhSnapHeight = 682006
 // 2022-06-16T17:30:00Z
 const UpgradeSkyrHeight = 1044660
 
+var UpgradeV17Height = abi.ChainEpoch(99999999999999)
+
 var SupportedProofTypes = []abi.RegisteredSealProof{
 	abi.RegisteredSealProof_StackedDrg32GiBV1,
 	abi.RegisteredSealProof_StackedDrg64GiBV1,
@@ -82,13 +89,28 @@ func init() {
 
 	Devnet = true
 
+	// NOTE: DO NOT change this unless you REALLY know what you're doing. This is not consensus critical, however,
+	//set this value too high may impacts your block submission; set this value too low may cause you miss
+	//parent tipsets for blocking forming and mining.
+	if len(os.Getenv("PROPAGATION_DELAY_SECS")) != 0 {
+		pds, err := strconv.ParseUint(os.Getenv("PROPAGATION_DELAY_SECS"), 10, 64)
+		if err != nil {
+			log.Warnw("Error setting PROPAGATION_DELAY_SECS, %v, proceed with default value %s", err,
+				PropagationDelaySecs)
+		} else {
+			PropagationDelaySecs = pds
+			log.Warnw(" !!WARNING!! propagation delay is set to be %s second, "+
+				"this value impacts your message republish interval and block forming - monitor with caution!!", PropagationDelaySecs)
+		}
+	}
+
 	BuildType = BuildCalibnet
 
 }
 
 const BlockDelaySecs = uint64(builtin2.EpochDurationSeconds)
 
-const PropagationDelaySecs = uint64(6)
+var PropagationDelaySecs = uint64(10)
 
 // BootstrapPeerThreshold is the minimum number peers we need to track for a sync worker to start
 const BootstrapPeerThreshold = 4

build/params_interop.go

@@ -7,22 +7,24 @@ import (
 	"os"
 	"strconv"
 
+	"github.com/ipfs/go-cid"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	actorstypes "github.com/filecoin-project/go-state-types/actors"
 	"github.com/filecoin-project/go-state-types/network"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/actors/policy"
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
-	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/lotus/chain/actors/policy"
 )
 
 var NetworkBundle = "caterpillarnet"
-var BundleOverrides map[actors.Version]string
+var BundleOverrides map[actorstypes.Version]string
 
 const BootstrappersFile = "interopnet.pi"
 const GenesisFile = "interopnet.car"
 
-const GenesisNetworkVersion = network.Version15
+const GenesisNetworkVersion = network.Version16
 
 var UpgradeBreezeHeight = abi.ChainEpoch(-1)
 
@@ -32,26 +34,22 @@ var UpgradeSmokeHeight = abi.ChainEpoch(-1)
 var UpgradeIgnitionHeight = abi.ChainEpoch(-2)
 var UpgradeRefuelHeight = abi.ChainEpoch(-3)
 var UpgradeTapeHeight = abi.ChainEpoch(-4)
-
 var UpgradeAssemblyHeight = abi.ChainEpoch(-5)
 var UpgradeLiftoffHeight = abi.ChainEpoch(-6)
-
 var UpgradeKumquatHeight = abi.ChainEpoch(-7)
 var UpgradeCalicoHeight = abi.ChainEpoch(-9)
 var UpgradePersianHeight = abi.ChainEpoch(-10)
 var UpgradeOrangeHeight = abi.ChainEpoch(-11)
 var UpgradeClausHeight = abi.ChainEpoch(-12)
-
 var UpgradeTrustHeight = abi.ChainEpoch(-13)
-
 var UpgradeNorwegianHeight = abi.ChainEpoch(-14)
-
 var UpgradeTurboHeight = abi.ChainEpoch(-15)
-
 var UpgradeHyperdriveHeight = abi.ChainEpoch(-16)
 var UpgradeChocolateHeight = abi.ChainEpoch(-17)
 var UpgradeOhSnapHeight = abi.ChainEpoch(-18)
-var UpgradeSkyrHeight = abi.ChainEpoch(100)
+var UpgradeSkyrHeight = abi.ChainEpoch(-19)
+
+const UpgradeV17Height = abi.ChainEpoch(99999999999999)
 
 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 	0: DrandMainnet,
@@ -105,6 +103,7 @@ func init() {
 	UpgradeChocolateHeight = getUpgradeHeight("LOTUS_CHOCOLATE_HEIGHT", UpgradeChocolateHeight)
 	UpgradeOhSnapHeight = getUpgradeHeight("LOTUS_OHSNAP_HEIGHT", UpgradeOhSnapHeight)
 	UpgradeSkyrHeight = getUpgradeHeight("LOTUS_SKYR_HEIGHT", UpgradeSkyrHeight)
+	UpgradeV17Height = getUpgradeHeight("LOTUS_V17_HEIGHT", UpgradeV17Height)
 
 	BuildType |= BuildInteropnet
 	SetAddressNetwork(address.Testnet)

build/params_mainnet.go

@@ -6,12 +6,12 @@ package build
 import (
 	"math"
 	"os"
-
-	"github.com/filecoin-project/go-state-types/network"
-	"github.com/filecoin-project/lotus/chain/actors"
+	"strconv"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	actorstypes "github.com/filecoin-project/go-state-types/actors"
+	"github.com/filecoin-project/go-state-types/network"
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
 )
 
@@ -23,7 +23,7 @@ var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 var NetworkBundle = "mainnet"
 
 // NOTE: DO NOT change this unless you REALLY know what you're doing. This is consensus critical.
-var BundleOverrides map[actors.Version]string
+var BundleOverrides map[actorstypes.Version]string
 
 const GenesisNetworkVersion = network.Version0
 
@@ -77,7 +77,9 @@ const UpgradeChocolateHeight = 1231620
 const UpgradeOhSnapHeight = 1594680
 
 // 2022-07-06T14:00:00Z
-var UpgradeSkyrHeight = abi.ChainEpoch(1960320)
+const UpgradeSkyrHeight = 1960320
+
+var UpgradeV17Height = abi.ChainEpoch(99999999999999)
 
 var SupportedProofTypes = []abi.RegisteredSealProof{
 	abi.RegisteredSealProof_StackedDrg32GiBV1,
@@ -86,14 +88,30 @@ var SupportedProofTypes = []abi.RegisteredSealProof{
 var ConsensusMinerMinPower = abi.NewStoragePower(10 << 40)
 var MinVerifiedDealSize = abi.NewStoragePower(1 << 20)
 var PreCommitChallengeDelay = abi.ChainEpoch(150)
+var PropagationDelaySecs = uint64(10)
 
 func init() {
 	if os.Getenv("LOTUS_USE_TEST_ADDRESSES") != "1" {
 		SetAddressNetwork(address.Mainnet)
 	}
 
-	if os.Getenv("LOTUS_DISABLE_SKYR") == "1" {
-		UpgradeSkyrHeight = math.MaxInt64
+	if os.Getenv("LOTUS_DISABLE_V17") == "1" {
+		UpgradeV17Height = math.MaxInt64
+	}
+
+	// NOTE: DO NOT change this unless you REALLY know what you're doing. This is not consensus critical, however,
+	//set this value too high may impacts your block submission; set this value too low may cause you miss
+	//parent tipsets for blocking forming and mining.
+	if len(os.Getenv("PROPAGATION_DELAY_SECS")) != 0 {
+		pds, err := strconv.ParseUint(os.Getenv("PROPAGATION_DELAY_SECS"), 10, 64)
+		if err != nil {
+			log.Warnw("Error setting PROPAGATION_DELAY_SECS, %v, proceed with default value %s", err,
+				PropagationDelaySecs)
+		} else {
+			PropagationDelaySecs = pds
+			log.Warnw(" !!WARNING!! propagation delay is set to be %s second, "+
+				"this value impacts your message republish interval and block forming - monitor with caution!!", PropagationDelaySecs)
+		}
 	}
 
 	Devnet = false
@@ -103,8 +121,6 @@ func init() {
 
 const BlockDelaySecs = uint64(builtin2.EpochDurationSeconds)
 
-const PropagationDelaySecs = uint64(6)
-
 // BootstrapPeerThreshold is the minimum number peers we need to track for a sync worker to start
 const BootstrapPeerThreshold = 4
 

build/params_shared_funcs.go

@@ -1,10 +1,10 @@
 package build
 
 import (
-	"github.com/filecoin-project/go-address"
 	"github.com/ipfs/go-cid"
+	"github.com/libp2p/go-libp2p/core/protocol"
 
-	"github.com/libp2p/go-libp2p-core/protocol"
+	"github.com/filecoin-project/go-address"
 
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 )

build/params_shared_vals.go

@@ -10,7 +10,6 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/network"
-
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
 
 	"github.com/filecoin-project/lotus/chain/actors/policy"
@@ -34,7 +33,7 @@ const NewestNetworkVersion = network.Version{{.latestNetworkVersion}}
 
 /* inline-gen start */
 
-const NewestNetworkVersion = network.Version16
+const NewestNetworkVersion = network.Version17
 
 /* inline-gen end */
 

build/params_testground.go

@@ -5,19 +5,18 @@
 //
 // Its purpose is to unlock various degrees of flexibility and parametrization
 // when writing Testground plans for Lotus.
-//
 package build
 
 import (
 	"math/big"
 
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/network"
 	"github.com/ipfs/go-cid"
 
+	"github.com/filecoin-project/go-state-types/abi"
+	actorstypes "github.com/filecoin-project/go-state-types/actors"
+	"github.com/filecoin-project/go-state-types/network"
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
 
-	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
 )
 
@@ -108,6 +107,7 @@ var (
 	UpgradeChocolateHeight  abi.ChainEpoch = -16
 	UpgradeOhSnapHeight     abi.ChainEpoch = -17
 	UpgradeSkyrHeight       abi.ChainEpoch = -18
+	UpgradeV17Height        abi.ChainEpoch = -19
 
 	DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 		0: DrandMainnet,
@@ -115,10 +115,10 @@ var (
 
 	GenesisNetworkVersion = network.Version0
 	NetworkBundle         = "devnet"
-	BundleOverrides       map[actors.Version]string
+	BundleOverrides       map[actorstypes.Version]string
 
-	NewestNetworkVersion       = network.Version15
-	ActorUpgradeNetworkVersion = network.Version15
+	NewestNetworkVersion       = network.Version16
+	ActorUpgradeNetworkVersion = network.Version16
 
 	Devnet      = true
 	ZeroAddress = MustParseAddress("f3yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaby2smx7a")

build/version.go

@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }
 
 // BuildVersion is the local build version
-const BuildVersion = "1.16.0"
+const BuildVersion = "1.17.2"
 
 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

chain/actors/actor_cids.go

@@ -0,0 +1,329 @@
+package actors
+
+import (
+	"github.com/ipfs/go-cid"
+	"golang.org/x/xerrors"
+
+	actorstypes "github.com/filecoin-project/go-state-types/actors"
+	builtin0 "github.com/filecoin-project/specs-actors/actors/builtin"
+	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
+	builtin3 "github.com/filecoin-project/specs-actors/v3/actors/builtin"
+	builtin4 "github.com/filecoin-project/specs-actors/v4/actors/builtin"
+	builtin5 "github.com/filecoin-project/specs-actors/v5/actors/builtin"
+	builtin6 "github.com/filecoin-project/specs-actors/v6/actors/builtin"
+	builtin7 "github.com/filecoin-project/specs-actors/v7/actors/builtin"
+)
+
+// GetActorCodeID looks up a builtin actor's code CID by actor version and canonical actor name.
+func GetActorCodeID(av actorstypes.Version, name string) (cid.Cid, bool) {
+
+	// Actors V8 and above
+	if av >= actorstypes.Version8 {
+		if cids, ok := GetActorCodeIDsFromManifest(av); ok {
+			c, ok := cids[name]
+			return c, ok
+		}
+	}
+
+	// Actors V7 and lower
+	switch name {
+
+	case AccountKey:
+		switch av {
+
+		case actorstypes.Version0:
+			return builtin0.AccountActorCodeID, true
+
+		case actorstypes.Version2:
+			return builtin2.AccountActorCodeID, true
+
+		case actorstypes.Version3:
+			return builtin3.AccountActorCodeID, true
+
+		case actorstypes.Version4:
+			return builtin4.AccountActorCodeID, true
+
+		case actorstypes.Version5:
+			return builtin5.AccountActorCodeID, true
+
+		case actorstypes.Version6:
+			return builtin6.AccountActorCodeID, true
+
+		case actorstypes.Version7:
+			return builtin7.AccountActorCodeID, true
+		}
+
+	case CronKey:
+		switch av {
+
+		case actorstypes.Version0:
+			return builtin0.CronActorCodeID, true
+
+		case actorstypes.Version2:
+			return builtin2.CronActorCodeID, true
+
+		case actorstypes.Version3:
+			return builtin3.CronActorCodeID, true
+
+		case actorstypes.Version4:
+			return builtin4.CronActorCodeID, true
+
+		case actorstypes.Version5:
+			return builtin5.CronActorCodeID, true
+
+		case actorstypes.Version6:
+			return builtin6.CronActorCodeID, true
+
+		case actorstypes.Version7:
+			return builtin7.CronActorCodeID, true
+		}
+
+	case InitKey:
+		switch av {
+
+		case actorstypes.Version0:
+			return builtin0.InitActorCodeID, true
+
+		case actorstypes.Version2:
+			return builtin2.InitActorCodeID, true
+
+		case actorstypes.Version3:
+			return builtin3.InitActorCodeID, true
+
+		case actorstypes.Version4:
+			return builtin4.InitActorCodeID, true
+
+		case actorstypes.Version5:
+			return builtin5.InitActorCodeID, true
+
+		case actorstypes.Version6:
+			return builtin6.InitActorCodeID, true
+
+		case actorstypes.Version7:
+			return builtin7.InitActorCodeID, true
+		}
+
+	case MarketKey:
+		switch av {
+
+		case actorstypes.Version0:
+			return builtin0.StorageMarketActorCodeID, true
+
+		case actorstypes.Version2:
+			return builtin2.StorageMarketActorCodeID, true
+
+		case actorstypes.Version3:
+			return builtin3.StorageMarketActorCodeID, true
+
+		case actorstypes.Version4:
+			return builtin4.StorageMarketActorCodeID, true
+
+		case actorstypes.Version5:
+			return builtin5.StorageMarketActorCodeID, true
+
+		case actorstypes.Version6:
+			return builtin6.StorageMarketActorCodeID, true
+
+		case actorstypes.Version7:
+			return builtin7.StorageMarketActorCodeID, true
+		}
+
+	case MinerKey:
+		switch av {
+
+		case actorstypes.Version0:
+			return builtin0.StorageMinerActorCodeID, true
+
+		case actorstypes.Version2:
+			return builtin2.StorageMinerActorCodeID, true
+
+		case actorstypes.Version3:
+			return builtin3.StorageMinerActorCodeID, true
+
+		case actorstypes.Version4:
+			return builtin4.StorageMinerActorCodeID, true
+
+		case actorstypes.Version5:
+			return builtin5.StorageMinerActorCodeID, true
+
+		case actorstypes.Version6:
+			return builtin6.StorageMinerActorCodeID, true
+
+		case actorstypes.Version7:
+			return builtin7.StorageMinerActorCodeID, true
+		}
+
+	case MultisigKey:
+		switch av {
+
+		case actorstypes.Version0:
+			return builtin0.MultisigActorCodeID, true
+
+		case actorstypes.Version2:
+			return builtin2.MultisigActorCodeID, true
+
+		case actorstypes.Version3:
+			return builtin3.MultisigActorCodeID, true
+
+		case actorstypes.Version4:
+			return builtin4.MultisigActorCodeID, true
+
+		case actorstypes.Version5:
+			return builtin5.MultisigActorCodeID, true
+
+		case actorstypes.Version6:
+			return builtin6.MultisigActorCodeID, true
+
+		case actorstypes.Version7:
+			return builtin7.MultisigActorCodeID, true
+		}
+
+	case PaychKey:
+		switch av {
+
+		case actorstypes.Version0:
+			return builtin0.PaymentChannelActorCodeID, true
+
+		case actorstypes.Version2:
+			return builtin2.PaymentChannelActorCodeID, true
+
+		case actorstypes.Version3:
+			return builtin3.PaymentChannelActorCodeID, true
+
+		case actorstypes.Version4:
+			return builtin4.PaymentChannelActorCodeID, true
+
+		case actorstypes.Version5:
+			return builtin5.PaymentChannelActorCodeID, true
+
+		case actorstypes.Version6:
+			return builtin6.PaymentChannelActorCodeID, true
+
+		case actorstypes.Version7:
+			return builtin7.PaymentChannelActorCodeID, true
+		}
+
+	case PowerKey:
+		switch av {
+
+		case actorstypes.Version0:
+			return builtin0.StoragePowerActorCodeID, true
+
+		case actorstypes.Version2:
+			return builtin2.StoragePowerActorCodeID, true
+
+		case actorstypes.Version3:
+			return builtin3.StoragePowerActorCodeID, true
+
+		case actorstypes.Version4:
+			return builtin4.StoragePowerActorCodeID, true
+
+		case actorstypes.Version5:
+			return builtin5.StoragePowerActorCodeID, true
+
+		case actorstypes.Version6:
+			return builtin6.StoragePowerActorCodeID, true
+
+		case actorstypes.Version7:
+			return builtin7.StoragePowerActorCodeID, true
+		}
+
+	case RewardKey:
+		switch av {
+
+		case actorstypes.Version0:
+			return builtin0.RewardActorCodeID, true
+
+		case actorstypes.Version2:
+			return builtin2.RewardActorCodeID, true
+
+		case actorstypes.Version3:
+			return builtin3.RewardActorCodeID, true
+
+		case actorstypes.Version4:
+			return builtin4.RewardActorCodeID, true
+
+		case actorstypes.Version5:
+			return builtin5.RewardActorCodeID, true
+
+		case actorstypes.Version6:
+			return builtin6.RewardActorCodeID, true
+
+		case actorstypes.Version7:
+			return builtin7.RewardActorCodeID, true
+		}
+
+	case SystemKey:
+		switch av {
+
+		case actorstypes.Version0:
+			return builtin0.SystemActorCodeID, true
+
+		case actorstypes.Version2:
+			return builtin2.SystemActorCodeID, true
+
+		case actorstypes.Version3:
+			return builtin3.SystemActorCodeID, true
+
+		case actorstypes.Version4:
+			return builtin4.SystemActorCodeID, true
+
+		case actorstypes.Version5:
+			return builtin5.SystemActorCodeID, true
+
+		case actorstypes.Version6:
+			return builtin6.SystemActorCodeID, true
+
+		case actorstypes.Version7:
+			return builtin7.SystemActorCodeID, true
+		}
+
+	case VerifregKey:
+		switch av {
+
+		case actorstypes.Version0:
+			return builtin0.VerifiedRegistryActorCodeID, true
+
+		case actorstypes.Version2:
+			return builtin2.VerifiedRegistryActorCodeID, true
+
+		case actorstypes.Version3:
+			return builtin3.VerifiedRegistryActorCodeID, true
+
+		case actorstypes.Version4:
+			return builtin4.VerifiedRegistryActorCodeID, true
+
+		case actorstypes.Version5:
+			return builtin5.VerifiedRegistryActorCodeID, true
+
+		case actorstypes.Version6:
+			return builtin6.VerifiedRegistryActorCodeID, true
+
+		case actorstypes.Version7:
+			return builtin7.VerifiedRegistryActorCodeID, true
+		}
+	}
+
+	return cid.Undef, false
+}
+
+// GetActorCodeIDs looks up all builtin actor's code CIDs by actor version.
+func GetActorCodeIDs(av actorstypes.Version) (map[string]cid.Cid, error) {
+	cids, ok := GetActorCodeIDsFromManifest(av)
+	if ok {
+		return cids, nil
+	}
+
+	actorsKeys := GetBuiltinActorsKeys()
+	synthCids := make(map[string]cid.Cid)
+
+	for _, key := range actorsKeys {
+		c, ok := GetActorCodeID(av, key)
+		if !ok {
+			return nil, xerrors.Errorf("could not find builtin actor cids for Actors version %d", av)
+		}
+		synthCids[key] = c
+	}
+
+	return synthCids, nil
+}

chain/actors/adt/diff_adt.go

@@ -3,8 +3,9 @@ package adt
 import (
 	"bytes"
 
-	"github.com/filecoin-project/go-state-types/abi"
 	typegen "github.com/whyrusleeping/cbor-gen"
+
+	"github.com/filecoin-project/go-state-types/abi"
 )
 
 // AdtArrayDiff generalizes adt.Array diffing by accepting a Deferred type that can unmarshalled to its corresponding struct
@@ -25,7 +26,7 @@ type AdtArrayDiff interface {
 // - All values that exist in preArr and not in curArr are passed to AdtArrayDiff.Remove()
 // - All values that exist in curArr nnd not in prevArr are passed to adtArrayDiff.Add()
 // - All values that exist in preArr and in curArr are passed to AdtArrayDiff.Modify()
-//  - It is the responsibility of AdtArrayDiff.Modify() to determine if the values it was passed have been modified.
+//   - It is the responsibility of AdtArrayDiff.Modify() to determine if the values it was passed have been modified.
 func DiffAdtArray(preArr, curArr Array, out AdtArrayDiff) error {
 	notNew := make(map[int64]struct{}, curArr.Length())
 	prevVal := new(typegen.Deferred)

chain/actors/adt/diff_adt_test.go

@@ -1,4 +1,4 @@
-//stm: #unit
+// stm: #unit
 package adt
 
 import (
@@ -6,14 +6,12 @@ import (
 	"context"
 	"testing"
 
+	cbornode "github.com/ipfs/go-ipld-cbor"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-
-	cbornode "github.com/ipfs/go-ipld-cbor"
 	typegen "github.com/whyrusleeping/cbor-gen"
 
 	"github.com/filecoin-project/go-state-types/abi"
-
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
 	adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt"
 

chain/actors/adt/store.go

@@ -3,8 +3,9 @@ package adt
 import (
 	"context"
 
-	adt "github.com/filecoin-project/specs-actors/actors/util/adt"
 	cbor "github.com/ipfs/go-ipld-cbor"
+
+	"github.com/filecoin-project/specs-actors/actors/util/adt"
 )
 
 type Store interface {

chain/actors/aerrors/error.go

@@ -3,8 +3,9 @@ package aerrors
 import (
 	"fmt"
 
-	"github.com/filecoin-project/go-state-types/exitcode"
 	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-state-types/exitcode"
 )
 
 func IsFatal(err ActorError) bool {

chain/actors/aerrors/error_test.go

@@ -1,14 +1,15 @@
-//stm: #unit
+// stm: #unit
 package aerrors_test
 
 import (
 	"testing"
 
-	"github.com/filecoin-project/go-state-types/exitcode"
-	. "github.com/filecoin-project/lotus/chain/actors/aerrors"
-
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-state-types/exitcode"
+
+	. "github.com/filecoin-project/lotus/chain/actors/aerrors"
 )
 
 func TestFatalError(t *testing.T) {