New interop network info

.circleci/config.yml

@@ -1,17 +1,15 @@
 version: 2.1
 orbs:
   go: gotest/tools@0.0.13
-  aws-cli: circleci/aws-cli@1.3.2
-  packer: salaxander/packer@0.0.3
 
 executors:
   golang:
     docker:
-      - image: circleci/golang:1.16.4
+      - image: circleci/golang:1.14.2
     resource_class: 2xlarge
   ubuntu:
     docker:
-      - image: ubuntu:20.04
+      - image: ubuntu:19.10
 
 commands:
   install-deps:
@@ -36,7 +34,7 @@ commands:
           condition: << parameters.linux >>
           steps:
             - run: sudo apt-get update
-            - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
+            - run: sudo apt-get install ocl-icd-opencl-dev
       - run: git submodule sync
       - run: git submodule update --init
   download-params:
@@ -47,7 +45,7 @@ commands:
             - 'v25-2k-lotus-params'
           paths:
             - /var/tmp/filecoin-proof-parameters/
-      - run:  ./lotus fetch-params 2048
+      - run:  ./lotus fetch-params --proving-params 2048
       - save_cache:
           name: Save parameters cache
           key: 'v25-2k-lotus-params'
@@ -95,10 +93,10 @@ jobs:
       - store_artifacts:
           path: lotus
       - store_artifacts:
-          path: lotus-miner
+          path: lotus-storage-miner
       - store_artifacts:
-          path: lotus-worker
-      - run: mkdir linux && mv lotus lotus-miner lotus-worker linux/
+          path: lotus-seal-worker
+      - run: mkdir linux && mv lotus lotus-storage-miner lotus-seal-worker linux/
       - persist_to_workspace:
           root: "."
           paths:
@@ -112,10 +110,10 @@ jobs:
       - run:
           command: make debug
 
-  test:
+  test: &test
     description: |
       Run tests with gotestsum.
-    parameters: &test-params
+    parameters:
       executor:
         type: executor
         default: golang
@@ -123,20 +121,20 @@ jobs:
         type: string
         default: "-timeout 30m"
         description: Flags passed to go test.
-      target:
+      packages:
         type: string
         default: "./..."
         description: Import paths of packages to be tested.
-      proofs-log-test:
+      winpost-test:
         type: string
         default: "0"
-      suite:
+      test-suite-name:
         type: string
         default: unit
         description: Test suite name to report to CircleCI.
       gotestsum-format:
         type: string
-        default: standard-verbose
+        default: short
         description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
       coverage:
         type: string
@@ -144,7 +142,7 @@ jobs:
         description: Coverage flag. Set to the empty string to disable.
       codecov-upload:
         type: boolean
-        default: true
+        default: false
         description: |
           Upload coverage report to https://codecov.io/. Requires the codecov API token to be
           set as an environment variable for private projects.
@@ -158,28 +156,21 @@ jobs:
       - download-params
       - go/install-gotestsum:
           gobin: $HOME/.local/bin
-          version: 0.5.2
       - run:
           name: go test
           environment:
-            TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >>
-            SKIP_CONFORMANCE: "1"
+            GOTESTSUM_JUNITFILE: /tmp/test-reports/<< parameters.test-suite-name >>/junit.xml
+            GOTESTSUM_FORMAT: << parameters.gotestsum-format >>
+            LOTUS_TEST_WINDOW_POST: << parameters.winpost-test >>
           command: |
-            mkdir -p /tmp/test-reports/<< parameters.suite >>
-            mkdir -p /tmp/test-artifacts
-            gotestsum \
-              --format << parameters.gotestsum-format >> \
-              --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
-              --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \
-              -- \
+            mkdir -p /tmp/test-reports/<< parameters.test-suite-name >>
+            gotestsum -- \
               << parameters.coverage >> \
               << parameters.go-test-flags >> \
-              << parameters.target >>
+              << parameters.packages >>
           no_output_timeout: 30m
       - store_test_results:
           path: /tmp/test-reports
-      - store_artifacts:
-          path: /tmp/test-artifacts/<< parameters.suite >>.json
       - when:
           condition: << parameters.codecov-upload >>
           steps:
@@ -190,160 +181,10 @@ jobs:
                 command: |
                   bash <(curl -s https://codecov.io/bash)
 
-  test-conformance:
-    description: |
-      Run tests using a corpus of interoperable test vectors for Filecoin 
-      implementations to test their correctness and compliance with the Filecoin
-      specifications.
-    parameters:
-      <<: *test-params
-      vectors-branch:
-        type: string
-        default: ""
-        description: |
-          Branch on github.com/filecoin-project/test-vectors to checkout and
-          test with. If empty (the default) the commit defined by the git
-          submodule is used.
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          command: make deps lotus
-          no_output_timeout: 30m
-      - download-params
-      - when:
-          condition:
-            not:
-              equal: [ "", << parameters.vectors-branch >> ]
-          steps:
-            - run:
-                name: checkout vectors branch
-                command: |
-                  cd extern/test-vectors
-                  git fetch
-                  git checkout origin/<< parameters.vectors-branch >>
-      - go/install-gotestsum:
-          gobin: $HOME/.local/bin
-          version: 0.5.2
-      - run:
-          name: install statediff globally
-          command: |
-            ## statediff is optional; we succeed even if compilation fails.
-            mkdir -p /tmp/statediff
-            git clone https://github.com/filecoin-project/statediff.git /tmp/statediff
-            cd /tmp/statediff
-            go install ./cmd/statediff || exit 0
-      - run:
-          name: go test
-          environment:
-            SKIP_CONFORMANCE: "0"
-          command: |
-            mkdir -p /tmp/test-reports
-            mkdir -p /tmp/test-artifacts
-            gotestsum \
-              --format pkgname-and-test-fails \
-              --junitfile /tmp/test-reports/junit.xml \
-              -- \
-              -v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/
-            go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html
-          no_output_timeout: 30m
-      - store_test_results:
-          path: /tmp/test-reports
-      - store_artifacts:
-          path: /tmp/test-artifacts/conformance-coverage.html
-  build-ntwk-calibration:
-    description: |
-      Compile lotus binaries for the calibration network
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: make calibnet
-      - run: mkdir linux-calibrationnet && mv lotus lotus-miner lotus-worker linux-calibrationnet
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux-calibrationnet
-  build-ntwk-butterfly:
-    description: |
-      Compile lotus binaries for the butterfly network
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: make butterflynet
-      - run: mkdir linux-butterflynet && mv lotus lotus-miner lotus-worker linux-butterflynet
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux-butterflynet
-  build-ntwk-nerpa:
-    description: |
-      Compile lotus binaries for the nerpa network
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: make nerpanet
-      - run: mkdir linux-nerpanet && mv lotus lotus-miner lotus-worker linux-nerpanet
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux-nerpanet
-  build-lotus-soup:
-    description: |
-      Compile `lotus-soup` Testground test plan
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: cd extern/filecoin-ffi && make
-      - run:
-          name: "go get lotus@master"
-          command: cd testplans/lotus-soup && go mod edit -replace=github.com/filecoin-project/lotus=../.. && go mod tidy
-      - run:
-          name: "build lotus-soup testplan"
-          command: pushd testplans/lotus-soup && go build -tags=testground .
-  trigger-testplans:
-    description: |
-      Trigger `lotus-soup` test cases on TaaS
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          name: "download testground"
-          command: wget https://gist.github.com/nonsense/5fbf3167cac79945f658771aed32fc44/raw/2e17eb0debf7ec6bdf027c1bdafc2c92dd97273b/testground-d3e9603 -O ~/testground-cli && chmod +x ~/testground-cli
-      - run:
-          name: "prepare .env.toml"
-          command: pushd testplans/lotus-soup && mkdir -p $HOME/testground && cp env-ci.toml $HOME/testground/.env.toml && echo 'endpoint="https://ci.testground.ipfs.team"' >> $HOME/testground/.env.toml && echo 'user="circleci"' >> $HOME/testground/.env.toml
-      - run:
-          name: "prepare testground home dir and link test plans"
-          command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup && ln -s $(pwd)/testplans/graphsync $HOME/testground/plans/graphsync
-      - run:
-          name: "go get lotus@master"
-          command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master
-      - run:
-          name: "trigger deals baseline testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/baseline-k8s-3-1.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-      - run:
-          name: "trigger payment channel stress testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-      - run:
-          name: "trigger graphsync testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/graphsync/_compositions/stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-
+  test-short:
+    <<: *test
+  test-window-post:
+    <<: *test
 
   build-macos:
     description: build darwin lotus binary
@@ -357,8 +198,8 @@ jobs:
       - run:
           name: Install go
           command: |
-            curl -O https://dl.google.com/go/go1.16.4.darwin-amd64.pkg && \
-            sudo installer -pkg go1.16.4.darwin-amd64.pkg -target /
+            curl -O https://dl.google.com/go/go1.14.2.darwin-amd64.pkg && \
+            sudo installer -pkg go1.14.2.darwin-amd64.pkg -target /
       - run:
           name: Install pkg-config
           command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
@@ -372,15 +213,6 @@ jobs:
           command: |
             curl --location https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 --output /usr/local/bin/jq
             chmod +x /usr/local/bin/jq
-      - run:
-          name: Install hwloc
-          command: |
-            mkdir ~/hwloc
-            curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-            cd ~/hwloc
-            tar -xvzpf hwloc-2.4.1.tar.gz
-            cd hwloc-2.4.1
-            ./configure && make && sudo make install
       - restore_cache:
           name: restore cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
@@ -391,10 +223,10 @@ jobs:
       - store_artifacts:
           path: lotus
       - store_artifacts:
-          path: lotus-miner
+          path: lotus-storage-miner
       - store_artifacts:
-          path: lotus-worker
-      - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
+          path: lotus-seal-worker
+      - run: mkdir darwin && mv lotus lotus-storage-miner lotus-seal-worker darwin/
       - persist_to_workspace:
           root: "."
           paths:
@@ -406,41 +238,6 @@ jobs:
             - "~/.rustup"
             - "~/.cargo"
 
-  build-appimage:
-    machine:
-      image: ubuntu-2004:202104-01
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - run:
-          name: install appimage-builder
-          command: |
-            # docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html
-            sudo apt update
-            sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace
-            sudo curl -Lo /usr/local/bin/appimagetool https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage
-            sudo chmod +x /usr/local/bin/appimagetool
-            sudo pip3 install appimage-builder
-      - run:
-          name: install lotus dependencies
-          command: sudo apt install ocl-icd-opencl-dev libhwloc-dev
-      - run:
-          name: build appimage
-          command: |
-            sed -i "s/version: latest/version: ${CIRCLE_TAG:-latest}/" AppImageBuilder.yml
-            make appimage
-      - run:
-          name: prepare workspace
-          command: |
-            mkdir appimage
-            mv Lotus-*.AppImage appimage
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - appimage
-
-
   gofmt:
     executor: golang
     steps:
@@ -449,41 +246,6 @@ jobs:
       - run:
           command: "! go fmt ./... 2>&1 | read"
 
-  gen-check:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run: make deps
-      - run: go install golang.org/x/tools/cmd/goimports
-      - run: go install github.com/hannahhoward/cbor-gen-for
-      - run: make gen
-      - run: git --no-pager diff
-      - run: git --no-pager diff --quiet
-      - run: make docsgen-cli
-      - run: git --no-pager diff
-      - run: git --no-pager diff --quiet
-
-  docs-check:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run: go install golang.org/x/tools/cmd/goimports
-      - run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full
-      - run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner
-      - run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker
-      - run: make deps
-      - run: make docsgen
-      - run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full
-      - run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner
-      - run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker
-      - run: git --no-pager diff
-      - run: diff ../pre-openrpc-full ../post-openrpc-full
-      - run: diff ../pre-openrpc-miner ../post-openrpc-miner
-      - run: diff ../pre-openrpc-worker ../post-openrpc-worker
-      - run: git --no-pager diff --quiet
-
   lint: &lint
     description: |
       Run golangci-lint.
@@ -520,6 +282,9 @@ jobs:
           command: |
             $HOME/.local/bin/golangci-lint run -v --timeout 2m \
               --concurrency << parameters.concurrency >> << parameters.args >>
+  lint-changes:
+    <<: *lint
+
   lint-all:
     <<: *lint
 
@@ -543,547 +308,52 @@ jobs:
           name: Publish release
           command: ./scripts/publish-release.sh
 
-  publish-snapcraft:
-    description: build and push snapcraft
-    machine:
-      image: ubuntu-2004:202104-01
-    resource_class: 2xlarge
-    parameters:
-      channel:
-        type: string
-        default: "edge"
-        description: snapcraft channel
-    steps:
-      - checkout
-      - run:
-          name: install snapcraft
-          command: sudo snap install snapcraft --classic
-      - run:
-          name: create snapcraft config file
-          command: |
-            mkdir -p ~/.config/snapcraft
-            echo "$SNAPCRAFT_LOGIN_FILE" | base64 -d > ~/.config/snapcraft/snapcraft.cfg
-      - run:
-          name: build snap
-          command: snapcraft --use-lxd
-      - run:
-          name: publish snap
-          command: snapcraft push *.snap --release << parameters.channel >>
-
-  build-and-push-image:
-    description: build and push docker images to public AWS ECR registry
-    executor: aws-cli/default
-    parameters:
-      profile-name:
-        type: string
-        default: "default"
-        description: AWS profile name to be configured.
-
-      aws-access-key-id:
-        type: env_var_name
-        default: AWS_ACCESS_KEY_ID
-        description: >
-          AWS access key id for IAM role. Set this to the name of
-          the environment variable you will set to hold this
-          value, i.e. AWS_ACCESS_KEY.
-
-      aws-secret-access-key:
-        type: env_var_name
-        default: AWS_SECRET_ACCESS_KEY
-        description: >
-          AWS secret key for IAM role. Set this to the name of
-          the environment variable you will set to hold this
-          value, i.e. AWS_SECRET_ACCESS_KEY.
-
-      region:
-        type: env_var_name
-        default: AWS_REGION
-        description: >
-          Name of env var storing your AWS region information,
-          defaults to AWS_REGION
-
-      account-url:
-        type: env_var_name
-        default: AWS_ECR_ACCOUNT_URL
-        description: >
-          Env var storing Amazon ECR account URL that maps to an AWS account,
-          e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com
-          defaults to AWS_ECR_ACCOUNT_URL
-
-      dockerfile:
-        type: string
-        default: Dockerfile
-        description: Name of dockerfile to use. Defaults to Dockerfile.
-
-      path:
-        type: string
-        default: .
-        description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory).
-
-      extra-build-args:
-        type: string
-        default: ""
-        description: >
-          Extra flags to pass to docker build. For examples, see
-          https://docs.docker.com/engine/reference/commandline/build
-
-      repo:
-        type: string
-        description: Name of an Amazon ECR repository
-
-      tag:
-        type: string
-        default: "latest"
-        description: A comma-separated string containing docker image tags to build and push (default = latest)
-
-    steps:
-      - run:
-          name: Confirm that environment variables are set
-          command: |
-            if [ -z "$AWS_ACCESS_KEY_ID" ]; then
-              echo "No AWS_ACCESS_KEY_ID is set. Skipping build-and-push job ..."
-              circleci-agent step halt
-            fi
-
-      - aws-cli/setup:
-          profile-name: <<parameters.profile-name>>
-          aws-access-key-id: <<parameters.aws-access-key-id>>
-          aws-secret-access-key: <<parameters.aws-secret-access-key>>
-          aws-region: <<parameters.region>>
-
-      - run:
-          name: Log into Amazon ECR
-          command: |
-            aws ecr-public get-login-password --region $<<parameters.region>> --profile <<parameters.profile-name>> | docker login --username AWS --password-stdin $<<parameters.account-url>>
-
-      - checkout
-
-      - setup_remote_docker:
-          version: 19.03.13
-          docker_layer_caching: false
-
-      - run:
-          name: Build docker image
-          command: |
-            registry_id=$(echo $<<parameters.account-url>> | sed "s;\..*;;g")
-
-            docker_tag_args=""
-            IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
-            for tag in "${DOCKER_TAGS[@]}"; do
-              docker_tag_args="$docker_tag_args -t $<<parameters.account-url>>/<<parameters.repo>>:$tag"
-            done
-
-            docker build \
-              <<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
-              -f <<parameters.path>>/<<parameters.dockerfile>> \
-              $docker_tag_args \
-              <<parameters.path>>
-
-      - run:
-          name: Push image to Amazon ECR
-          command: |
-            IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
-            for tag in "${DOCKER_TAGS[@]}"; do
-              docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
-            done
-
-  publish-packer-mainnet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
-  publish-packer-calibrationnet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
-  publish-packer-butterflynet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG"
-  publish-packer-nerpanet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG"
-  publish-dockerhub:
-    description: publish to dockerhub
-    machine:
-      image: ubuntu-2004:202010-01
-    parameters:
-      tag:
-        type: string
-        default: latest
-    steps:
-      - checkout
-      - run:
-          name: dockerhub login
-          command: echo $DOCKERHUB_PASSWORD | docker login --username $DOCKERHUB_USERNAME --password-stdin
-      - run:
-          name: docker build
-          command: |
-            docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus .
-            docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus .
-            if [[ ! -z $CIRCLE_SHA1 ]]; then
-              docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus .
-              docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus .
-            fi
-            if [[ ! -z $CIRCLE_TAG ]]; then
-              docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus .
-              docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus .
-            fi
-      - run:
-          name: docker push
-          command: |
-            docker push filecoin/lotus:<< parameters.tag >>
-            docker push filecoin/lotus-all-in-one:<< parameters.tag >>
-            if [[ ! -z $CIRCLE_SHA1 ]]; then
-              docker push filecoin/lotus:$CIRCLE_SHA1
-              docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1
-            fi
-            if [[ ! -z $CIRCLE_TAG ]]; then
-              docker push filecoin/lotus:$CIRCLE_TAG
-              docker push filecoin/lotus-all-in-one:$CIRCLE_TAG
-            fi
 
 workflows:
   version: 2.1
   ci:
     jobs:
-      - lint-all:
-          concurrency: "16"   # expend all docker 2xlarge CPUs.
+      - lint-changes:
+          args: "--new-from-rev origin/next" 
       - mod-tidy-check
       - gofmt
-      - gen-check
-      - docs-check
       - test:
-          name: test-itest-api
-          suite: itest-api
-          target: "./itests/api_test.go"
-      
-      - test:
-          name: test-itest-batch_deal
-          suite: itest-batch_deal
-          target: "./itests/batch_deal_test.go"
-      
-      - test:
-          name: test-itest-ccupgrade
-          suite: itest-ccupgrade
-          target: "./itests/ccupgrade_test.go"
-      
-      - test:
-          name: test-itest-cli
-          suite: itest-cli
-          target: "./itests/cli_test.go"
-      
-      - test:
-          name: test-itest-deadlines
-          suite: itest-deadlines
-          target: "./itests/deadlines_test.go"
-      
-      - test:
-          name: test-itest-deals_concurrent
-          suite: itest-deals_concurrent
-          target: "./itests/deals_concurrent_test.go"
-      
-      - test:
-          name: test-itest-deals_offline
-          suite: itest-deals_offline
-          target: "./itests/deals_offline_test.go"
-      
-      - test:
-          name: test-itest-deals_padding
-          suite: itest-deals_padding
-          target: "./itests/deals_padding_test.go"
-      
-      - test:
-          name: test-itest-deals_power
-          suite: itest-deals_power
-          target: "./itests/deals_power_test.go"
-      
-      - test:
-          name: test-itest-deals_pricing
-          suite: itest-deals_pricing
-          target: "./itests/deals_pricing_test.go"
-      
-      - test:
-          name: test-itest-deals_publish
-          suite: itest-deals_publish
-          target: "./itests/deals_publish_test.go"
-      
-      - test:
-          name: test-itest-deals
-          suite: itest-deals
-          target: "./itests/deals_test.go"
-      
-      - test:
-          name: test-itest-gateway
-          suite: itest-gateway
-          target: "./itests/gateway_test.go"
-      
-      - test:
-          name: test-itest-get_messages_in_ts
-          suite: itest-get_messages_in_ts
-          target: "./itests/get_messages_in_ts_test.go"
-      
-      - test:
-          name: test-itest-multisig
-          suite: itest-multisig
-          target: "./itests/multisig_test.go"
-      
-      - test:
-          name: test-itest-nonce
-          suite: itest-nonce
-          target: "./itests/nonce_test.go"
-      
-      - test:
-          name: test-itest-paych_api
-          suite: itest-paych_api
-          target: "./itests/paych_api_test.go"
-      
-      - test:
-          name: test-itest-paych_cli
-          suite: itest-paych_cli
-          target: "./itests/paych_cli_test.go"
-      
-      - test:
-          name: test-itest-sdr_upgrade
-          suite: itest-sdr_upgrade
-          target: "./itests/sdr_upgrade_test.go"
-      
-      - test:
-          name: test-itest-sector_finalize_early
-          suite: itest-sector_finalize_early
-          target: "./itests/sector_finalize_early_test.go"
-      
-      - test:
-          name: test-itest-sector_miner_collateral
-          suite: itest-sector_miner_collateral
-          target: "./itests/sector_miner_collateral_test.go"
-      
-      - test:
-          name: test-itest-sector_pledge
-          suite: itest-sector_pledge
-          target: "./itests/sector_pledge_test.go"
-      
-      - test:
-          name: test-itest-sector_terminate
-          suite: itest-sector_terminate
-          target: "./itests/sector_terminate_test.go"
-      
-      - test:
-          name: test-itest-tape
-          suite: itest-tape
-          target: "./itests/tape_test.go"
-      
-      - test:
-          name: test-itest-verifreg
-          suite: itest-verifreg
-          target: "./itests/verifreg_test.go"
-      
-      - test:
-          name: test-itest-wdpost_dispute
-          suite: itest-wdpost_dispute
-          target: "./itests/wdpost_dispute_test.go"
-      
-      - test:
-          name: test-itest-wdpost
-          suite: itest-wdpost
-          target: "./itests/wdpost_test.go"
-      
-      - test:
-          name: test-unit-cli
-          suite: utest-unit-cli
-          target: "./cli/... ./cmd/... ./api/..."
-      - test:
-          name: test-unit-node
-          suite: utest-unit-node
-          target: "./node/..."
-      - test:
-          name: test-unit-rest
-          suite: utest-unit-rest
-          target: "./api/... ./blockstore/... ./build/... ./chain/... ./cli/... ./cmd/... ./conformance/... ./extern/... ./gateway/... ./journal/... ./lib/... ./markets/... ./node/... ./paychmgr/... ./storage/... ./tools/..."
-      - test:
-          name: test-unit-storage
-          suite: utest-unit-storage
-          target: "./storage/... ./extern/..."
-      - test:
-          go-test-flags: "-run=TestMulticoreSDR"
-          suite: multicore-sdr-check
-          target: "./extern/sector-storage/ffiwrapper"
-          proofs-log-test: "1"
-      - test-conformance:
-          suite: conformance
-          codecov-upload: false
-          target: "./conformance"
-      - test-conformance:
-          name: test-conformance-bleeding-edge
-          codecov-upload: false
-          suite: conformance-bleeding-edge
-          target: "./conformance"
-          vectors-branch: master
-      - trigger-testplans:
+          codecov-upload: true
+      - test-window-post:
+          go-test-flags: "-run=TestWindowedPost"
+          winpost-test: "1"
+      - test-short:
+          go-test-flags: "--timeout 10m --short"
           filters:
-            branches:
+            tags:
               only:
-                - master
+                - /^v\d+\.\d+\.\d+$/
       - build-debug
       - build-all:
+          requires:
+            - test-short
           filters:
             tags:
               only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-ntwk-calibration:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-ntwk-butterfly:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-ntwk-nerpa:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-lotus-soup
+                - /^v\d+\.\d+\.\d+$/
       - build-macos:
+          requires:
+            - test-short
           filters:
             branches:
               ignore:
                 - /.*/
             tags:
               only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-appimage:
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+                - /^v\d+\.\d+\.\d+$/
       - publish:
           requires:
             - build-all
             - build-macos
-            - build-appimage
           filters:
             branches:
               ignore:
                 - /.*/
             tags:
               only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-and-push-image:
-          dockerfile: Dockerfile.lotus
-          path: .
-          repo: lotus-dev
-          tag: '${CIRCLE_SHA1:0:8}'
-      - publish-packer-mainnet:
-          requires:
-            - build-all
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-calibrationnet:
-          requires:
-            - build-ntwk-calibration
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-butterflynet:
-          requires:
-            - build-ntwk-butterfly
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-nerpanet:
-          requires:
-            - build-ntwk-nerpa
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-snapcraft:
-          name: publish-snapcraft-stable
-          channel: stable
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-dockerhub:
-          name: publish-dockerhub
-          tag: stable
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-
-  nightly:
-    triggers:
-      - schedule:
-          cron: "0 0 * * *"
-          filters:
-            branches:
-              only:
-                - master
-    jobs:
-      - publish-snapcraft:
-          name: publish-snapcraft-nightly
-          channel: edge
-      - publish-dockerhub:
-          name: publish-dockerhub-nightly
-          tag: nightly
+                - /^v\d+\.\d+\.\d+$/

.circleci/gen.go

@@ -1,136 +0,0 @@
-package main
-
-import (
-	"embed"
-	"fmt"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-	"text/template"
-)
-
-//go:generate go run ./gen.go ..
-
-//go:embed template.yml
-var templateFile embed.FS
-
-type (
-	dirs  = []string
-	suite = string
-)
-
-// groupedUnitTests maps suite names to top-level directories that should be
-// included in that suite. The program adds an implicit group "rest" that
-// includes all other top-level directories.
-var groupedUnitTests = map[suite]dirs{
-	"unit-node":    {"node"},
-	"unit-storage": {"storage", "extern"},
-	"unit-cli":     {"cli", "cmd", "api"},
-}
-
-func main() {
-	if len(os.Args) != 2 {
-		panic("expected path to repo as argument")
-	}
-
-	repo := os.Args[1]
-
-	tmpl := template.New("template.yml")
-	tmpl.Delims("[[", "]]")
-	tmpl.Funcs(template.FuncMap{
-		"stripSuffix": func(in string) string {
-			return strings.TrimSuffix(in, "_test.go")
-		},
-	})
-	tmpl = template.Must(tmpl.ParseFS(templateFile, "*"))
-
-	// list all itests.
-	itests, err := filepath.Glob(filepath.Join(repo, "./itests/*_test.go"))
-	if err != nil {
-		panic(err)
-	}
-
-	// strip the dir from all entries.
-	for i, f := range itests {
-		itests[i] = filepath.Base(f)
-	}
-
-	// calculate the exclusion set of unit test directories to exclude because
-	// they are already included in a grouped suite.
-	var excluded = map[string]struct{}{}
-	for _, ss := range groupedUnitTests {
-		for _, s := range ss {
-			e, err := filepath.Abs(filepath.Join(repo, s))
-			if err != nil {
-				panic(err)
-			}
-			excluded[e] = struct{}{}
-		}
-	}
-
-	// all unit tests top-level dirs that are not itests, nor included in other suites.
-	var rest = map[string]struct{}{}
-	err = filepath.Walk(repo, func(path string, f os.FileInfo, err error) error {
-		// include all tests that aren't in the itests directory.
-		if strings.Contains(path, "itests") {
-			return filepath.SkipDir
-		}
-		// exclude all tests included in other suites
-		if f.IsDir() {
-			if _, ok := excluded[path]; ok {
-				return filepath.SkipDir
-			}
-		}
-		if strings.HasSuffix(path, "_test.go") {
-			rel, err := filepath.Rel(repo, path)
-			if err != nil {
-				panic(err)
-			}
-			// take the first directory
-			rest[strings.Split(rel, string(os.PathSeparator))[0]] = struct{}{}
-		}
-		return err
-	})
-	if err != nil {
-		panic(err)
-	}
-
-	// add other directories to a 'rest' suite.
-	for k := range rest {
-		groupedUnitTests["unit-rest"] = append(groupedUnitTests["unit-rest"], k)
-	}
-
-	// map iteration guarantees no order, so sort the array in-place.
-	sort.Strings(groupedUnitTests["unit-rest"])
-
-	// form the input data.
-	type data struct {
-		ItestFiles []string
-		UnitSuites map[string]string
-	}
-	in := data{
-		ItestFiles: itests,
-		UnitSuites: func() map[string]string {
-			ret := make(map[string]string)
-			for name, dirs := range groupedUnitTests {
-				for i, d := range dirs {
-					dirs[i] = fmt.Sprintf("./%s/...", d) // turn into package
-				}
-				ret[name] = strings.Join(dirs, " ")
-			}
-			return ret
-		}(),
-	}
-
-	out, err := os.Create("./config.yml")
-	if err != nil {
-		panic(err)
-	}
-	defer out.Close()
-
-	// execute the template.
-	if err := tmpl.Execute(out, in); err != nil {
-		panic(err)
-	}
-}

.circleci/template.yml

@@ -1,954 +0,0 @@
-version: 2.1
-orbs:
-  go: gotest/tools@0.0.13
-  aws-cli: circleci/aws-cli@1.3.2
-  packer: salaxander/packer@0.0.3
-
-executors:
-  golang:
-    docker:
-      - image: circleci/golang:1.16.4
-    resource_class: 2xlarge
-  ubuntu:
-    docker:
-      - image: ubuntu:20.04
-
-commands:
-  install-deps:
-    steps:
-      - go/install-ssh
-      - go/install: {package: git}
-  prepare:
-    parameters:
-      linux:
-        default: true
-        description: is a linux build environment?
-        type: boolean
-      darwin:
-        default: false
-        description: is a darwin build environment?
-        type: boolean
-    steps:
-      - checkout
-      - git_fetch_all_tags
-      - checkout
-      - when:
-          condition: << parameters.linux >>
-          steps:
-            - run: sudo apt-get update
-            - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
-      - run: git submodule sync
-      - run: git submodule update --init
-  download-params:
-    steps:
-      - restore_cache:
-          name: Restore parameters cache
-          keys:
-            - 'v25-2k-lotus-params'
-          paths:
-            - /var/tmp/filecoin-proof-parameters/
-      - run:  ./lotus fetch-params 2048
-      - save_cache:
-          name: Save parameters cache
-          key: 'v25-2k-lotus-params'
-          paths:
-            - /var/tmp/filecoin-proof-parameters/
-  install_ipfs:
-    steps:
-      - run: |
-          apt update
-          apt install -y wget
-          wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz
-          wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512
-          if [ "$(sha512sum go-ipfs_v0.4.22_linux-amd64.tar.gz)" != "$(cat go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512)" ]
-          then
-            echo "ipfs failed checksum check"
-            exit 1
-          fi
-          tar -xf go-ipfs_v0.4.22_linux-amd64.tar.gz
-          mv go-ipfs/ipfs /usr/local/bin/ipfs
-          chmod +x /usr/local/bin/ipfs
-  git_fetch_all_tags:
-    steps:
-      - run:
-          name: fetch all tags
-          command: |
-            git fetch --all
-
-jobs:
-  mod-tidy-check:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - go/mod-tidy-check
-
-  build-all:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run: sudo apt-get update
-      - run: sudo apt-get install npm
-      - run:
-          command: make buildall
-      - store_artifacts:
-          path: lotus
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir linux && mv lotus lotus-miner lotus-worker linux/
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux
-
-  build-debug:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          command: make debug
-
-  test:
-    description: |
-      Run tests with gotestsum.
-    parameters: &test-params
-      executor:
-        type: executor
-        default: golang
-      go-test-flags:
-        type: string
-        default: "-timeout 30m"
-        description: Flags passed to go test.
-      target:
-        type: string
-        default: "./..."
-        description: Import paths of packages to be tested.
-      proofs-log-test:
-        type: string
-        default: "0"
-      suite:
-        type: string
-        default: unit
-        description: Test suite name to report to CircleCI.
-      gotestsum-format:
-        type: string
-        default: standard-verbose
-        description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
-      coverage:
-        type: string
-        default: -coverprofile=coverage.txt -coverpkg=github.com/filecoin-project/lotus/...
-        description: Coverage flag. Set to the empty string to disable.
-      codecov-upload:
-        type: boolean
-        default: true
-        description: |
-          Upload coverage report to https://codecov.io/. Requires the codecov API token to be
-          set as an environment variable for private projects.
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          command: make deps lotus
-          no_output_timeout: 30m
-      - download-params
-      - go/install-gotestsum:
-          gobin: $HOME/.local/bin
-          version: 0.5.2
-      - run:
-          name: go test
-          environment:
-            TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >>
-            SKIP_CONFORMANCE: "1"
-          command: |
-            mkdir -p /tmp/test-reports/<< parameters.suite >>
-            mkdir -p /tmp/test-artifacts
-            gotestsum \
-              --format << parameters.gotestsum-format >> \
-              --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
-              --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \
-              -- \
-              << parameters.coverage >> \
-              << parameters.go-test-flags >> \
-              << parameters.target >>
-          no_output_timeout: 30m
-      - store_test_results:
-          path: /tmp/test-reports
-      - store_artifacts:
-          path: /tmp/test-artifacts/<< parameters.suite >>.json
-      - when:
-          condition: << parameters.codecov-upload >>
-          steps:
-            - go/install: {package: bash}
-            - go/install: {package: curl}
-            - run:
-                shell: /bin/bash -eo pipefail
-                command: |
-                  bash <(curl -s https://codecov.io/bash)
-
-  test-conformance:
-    description: |
-      Run tests using a corpus of interoperable test vectors for Filecoin 
-      implementations to test their correctness and compliance with the Filecoin
-      specifications.
-    parameters:
-      <<: *test-params
-      vectors-branch:
-        type: string
-        default: ""
-        description: |
-          Branch on github.com/filecoin-project/test-vectors to checkout and
-          test with. If empty (the default) the commit defined by the git
-          submodule is used.
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          command: make deps lotus
-          no_output_timeout: 30m
-      - download-params
-      - when:
-          condition:
-            not:
-              equal: [ "", << parameters.vectors-branch >> ]
-          steps:
-            - run:
-                name: checkout vectors branch
-                command: |
-                  cd extern/test-vectors
-                  git fetch
-                  git checkout origin/<< parameters.vectors-branch >>
-      - go/install-gotestsum:
-          gobin: $HOME/.local/bin
-          version: 0.5.2
-      - run:
-          name: install statediff globally
-          command: |
-            ## statediff is optional; we succeed even if compilation fails.
-            mkdir -p /tmp/statediff
-            git clone https://github.com/filecoin-project/statediff.git /tmp/statediff
-            cd /tmp/statediff
-            go install ./cmd/statediff || exit 0
-      - run:
-          name: go test
-          environment:
-            SKIP_CONFORMANCE: "0"
-          command: |
-            mkdir -p /tmp/test-reports
-            mkdir -p /tmp/test-artifacts
-            gotestsum \
-              --format pkgname-and-test-fails \
-              --junitfile /tmp/test-reports/junit.xml \
-              -- \
-              -v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/
-            go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html
-          no_output_timeout: 30m
-      - store_test_results:
-          path: /tmp/test-reports
-      - store_artifacts:
-          path: /tmp/test-artifacts/conformance-coverage.html
-  build-ntwk-calibration:
-    description: |
-      Compile lotus binaries for the calibration network
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: make calibnet
-      - run: mkdir linux-calibrationnet && mv lotus lotus-miner lotus-worker linux-calibrationnet
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux-calibrationnet
-  build-ntwk-butterfly:
-    description: |
-      Compile lotus binaries for the butterfly network
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: make butterflynet
-      - run: mkdir linux-butterflynet && mv lotus lotus-miner lotus-worker linux-butterflynet
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux-butterflynet
-  build-ntwk-nerpa:
-    description: |
-      Compile lotus binaries for the nerpa network
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: make nerpanet
-      - run: mkdir linux-nerpanet && mv lotus lotus-miner lotus-worker linux-nerpanet
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux-nerpanet
-  build-lotus-soup:
-    description: |
-      Compile `lotus-soup` Testground test plan
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: cd extern/filecoin-ffi && make
-      - run:
-          name: "go get lotus@master"
-          command: cd testplans/lotus-soup && go mod edit -replace=github.com/filecoin-project/lotus=../.. && go mod tidy
-      - run:
-          name: "build lotus-soup testplan"
-          command: pushd testplans/lotus-soup && go build -tags=testground .
-  trigger-testplans:
-    description: |
-      Trigger `lotus-soup` test cases on TaaS
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          name: "download testground"
-          command: wget https://gist.github.com/nonsense/5fbf3167cac79945f658771aed32fc44/raw/2e17eb0debf7ec6bdf027c1bdafc2c92dd97273b/testground-d3e9603 -O ~/testground-cli && chmod +x ~/testground-cli
-      - run:
-          name: "prepare .env.toml"
-          command: pushd testplans/lotus-soup && mkdir -p $HOME/testground && cp env-ci.toml $HOME/testground/.env.toml && echo 'endpoint="https://ci.testground.ipfs.team"' >> $HOME/testground/.env.toml && echo 'user="circleci"' >> $HOME/testground/.env.toml
-      - run:
-          name: "prepare testground home dir and link test plans"
-          command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup && ln -s $(pwd)/testplans/graphsync $HOME/testground/plans/graphsync
-      - run:
-          name: "go get lotus@master"
-          command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master
-      - run:
-          name: "trigger deals baseline testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/baseline-k8s-3-1.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-      - run:
-          name: "trigger payment channel stress testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-      - run:
-          name: "trigger graphsync testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/graphsync/_compositions/stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-
-
-  build-macos:
-    description: build darwin lotus binary
-    macos:
-      xcode: "10.0.0"
-    working_directory: ~/go/src/github.com/filecoin-project/lotus
-    steps:
-      - prepare:
-          linux: false
-          darwin: true
-      - run:
-          name: Install go
-          command: |
-            curl -O https://dl.google.com/go/go1.16.4.darwin-amd64.pkg && \
-            sudo installer -pkg go1.16.4.darwin-amd64.pkg -target /
-      - run:
-          name: Install pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
-      - run: go version
-      - run:
-          name: Install Rust
-          command: |
-            curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - run:
-          name: Install jq
-          command: |
-            curl --location https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 --output /usr/local/bin/jq
-            chmod +x /usr/local/bin/jq
-      - run:
-          name: Install hwloc
-          command: |
-            mkdir ~/hwloc
-            curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-            cd ~/hwloc
-            tar -xvzpf hwloc-2.4.1.tar.gz
-            cd hwloc-2.4.1
-            ./configure && make && sudo make install
-      - restore_cache:
-          name: restore cargo cache
-          key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-      - install-deps
-      - run:
-          command: make build
-          no_output_timeout: 30m
-      - store_artifacts:
-          path: lotus
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - darwin
-      - save_cache:
-          name: save cargo cache
-          key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-          paths:
-            - "~/.rustup"
-            - "~/.cargo"
-
-  build-appimage:
-    machine:
-      image: ubuntu-2004:202104-01
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - run:
-          name: install appimage-builder
-          command: |
-            # docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html
-            sudo apt update
-            sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace
-            sudo curl -Lo /usr/local/bin/appimagetool https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage
-            sudo chmod +x /usr/local/bin/appimagetool
-            sudo pip3 install appimage-builder
-      - run:
-          name: install lotus dependencies
-          command: sudo apt install ocl-icd-opencl-dev libhwloc-dev
-      - run:
-          name: build appimage
-          command: |
-            sed -i "s/version: latest/version: ${CIRCLE_TAG:-latest}/" AppImageBuilder.yml
-            make appimage
-      - run:
-          name: prepare workspace
-          command: |
-            mkdir appimage
-            mv Lotus-*.AppImage appimage
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - appimage
-
-
-  gofmt:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          command: "! go fmt ./... 2>&1 | read"
-
-  gen-check:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run: make deps
-      - run: go install golang.org/x/tools/cmd/goimports
-      - run: go install github.com/hannahhoward/cbor-gen-for
-      - run: make gen
-      - run: git --no-pager diff
-      - run: git --no-pager diff --quiet
-      - run: make docsgen-cli
-      - run: git --no-pager diff
-      - run: git --no-pager diff --quiet
-
-  docs-check:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run: go install golang.org/x/tools/cmd/goimports
-      - run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full
-      - run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner
-      - run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker
-      - run: make deps
-      - run: make docsgen
-      - run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full
-      - run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner
-      - run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker
-      - run: git --no-pager diff
-      - run: diff ../pre-openrpc-full ../post-openrpc-full
-      - run: diff ../pre-openrpc-miner ../post-openrpc-miner
-      - run: diff ../pre-openrpc-worker ../post-openrpc-worker
-      - run: git --no-pager diff --quiet
-
-  lint: &lint
-    description: |
-      Run golangci-lint.
-    parameters:
-      executor:
-        type: executor
-        default: golang
-      golangci-lint-version:
-        type: string
-        default: 1.27.0
-      concurrency:
-        type: string
-        default: '2'
-        description: |
-          Concurrency used to run linters. Defaults to 2 because NumCPU is not
-          aware of container CPU limits.
-      args:
-        type: string
-        default: ''
-        description: |
-          Arguments to pass to golangci-lint
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          command: make deps
-          no_output_timeout: 30m
-      - go/install-golangci-lint:
-          gobin: $HOME/.local/bin
-          version: << parameters.golangci-lint-version >>
-      - run:
-          name: Lint
-          command: |
-            $HOME/.local/bin/golangci-lint run -v --timeout 2m \
-              --concurrency << parameters.concurrency >> << parameters.args >>
-  lint-all:
-    <<: *lint
-
-  publish:
-    description: publish binary artifacts
-    executor: ubuntu
-    steps:
-      - run:
-          name: Install git jq curl
-          command: apt update && apt install -y git jq curl
-      - checkout
-      - git_fetch_all_tags
-      - checkout
-      - install_ipfs
-      - attach_workspace:
-          at: "."
-      - run:
-          name: Create bundles
-          command: ./scripts/build-bundle.sh
-      - run:
-          name: Publish release
-          command: ./scripts/publish-release.sh
-
-  publish-snapcraft:
-    description: build and push snapcraft
-    machine:
-      image: ubuntu-2004:202104-01
-    resource_class: 2xlarge
-    parameters:
-      channel:
-        type: string
-        default: "edge"
-        description: snapcraft channel
-    steps:
-      - checkout
-      - run:
-          name: install snapcraft
-          command: sudo snap install snapcraft --classic
-      - run:
-          name: create snapcraft config file
-          command: |
-            mkdir -p ~/.config/snapcraft
-            echo "$SNAPCRAFT_LOGIN_FILE" | base64 -d > ~/.config/snapcraft/snapcraft.cfg
-      - run:
-          name: build snap
-          command: snapcraft --use-lxd
-      - run:
-          name: publish snap
-          command: snapcraft push *.snap --release << parameters.channel >>
-
-  build-and-push-image:
-    description: build and push docker images to public AWS ECR registry
-    executor: aws-cli/default
-    parameters:
-      profile-name:
-        type: string
-        default: "default"
-        description: AWS profile name to be configured.
-
-      aws-access-key-id:
-        type: env_var_name
-        default: AWS_ACCESS_KEY_ID
-        description: >
-          AWS access key id for IAM role. Set this to the name of
-          the environment variable you will set to hold this
-          value, i.e. AWS_ACCESS_KEY.
-
-      aws-secret-access-key:
-        type: env_var_name
-        default: AWS_SECRET_ACCESS_KEY
-        description: >
-          AWS secret key for IAM role. Set this to the name of
-          the environment variable you will set to hold this
-          value, i.e. AWS_SECRET_ACCESS_KEY.
-
-      region:
-        type: env_var_name
-        default: AWS_REGION
-        description: >
-          Name of env var storing your AWS region information,
-          defaults to AWS_REGION
-
-      account-url:
-        type: env_var_name
-        default: AWS_ECR_ACCOUNT_URL
-        description: >
-          Env var storing Amazon ECR account URL that maps to an AWS account,
-          e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com
-          defaults to AWS_ECR_ACCOUNT_URL
-
-      dockerfile:
-        type: string
-        default: Dockerfile
-        description: Name of dockerfile to use. Defaults to Dockerfile.
-
-      path:
-        type: string
-        default: .
-        description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory).
-
-      extra-build-args:
-        type: string
-        default: ""
-        description: >
-          Extra flags to pass to docker build. For examples, see
-          https://docs.docker.com/engine/reference/commandline/build
-
-      repo:
-        type: string
-        description: Name of an Amazon ECR repository
-
-      tag:
-        type: string
-        default: "latest"
-        description: A comma-separated string containing docker image tags to build and push (default = latest)
-
-    steps:
-      - run:
-          name: Confirm that environment variables are set
-          command: |
-            if [ -z "$AWS_ACCESS_KEY_ID" ]; then
-              echo "No AWS_ACCESS_KEY_ID is set. Skipping build-and-push job ..."
-              circleci-agent step halt
-            fi
-
-      - aws-cli/setup:
-          profile-name: <<parameters.profile-name>>
-          aws-access-key-id: <<parameters.aws-access-key-id>>
-          aws-secret-access-key: <<parameters.aws-secret-access-key>>
-          aws-region: <<parameters.region>>
-
-      - run:
-          name: Log into Amazon ECR
-          command: |
-            aws ecr-public get-login-password --region $<<parameters.region>> --profile <<parameters.profile-name>> | docker login --username AWS --password-stdin $<<parameters.account-url>>
-
-      - checkout
-
-      - setup_remote_docker:
-          version: 19.03.13
-          docker_layer_caching: false
-
-      - run:
-          name: Build docker image
-          command: |
-            registry_id=$(echo $<<parameters.account-url>> | sed "s;\..*;;g")
-
-            docker_tag_args=""
-            IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
-            for tag in "${DOCKER_TAGS[@]}"; do
-              docker_tag_args="$docker_tag_args -t $<<parameters.account-url>>/<<parameters.repo>>:$tag"
-            done
-
-            docker build \
-              <<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
-              -f <<parameters.path>>/<<parameters.dockerfile>> \
-              $docker_tag_args \
-              <<parameters.path>>
-
-      - run:
-          name: Push image to Amazon ECR
-          command: |
-            IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
-            for tag in "${DOCKER_TAGS[@]}"; do
-              docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
-            done
-
-  publish-packer-mainnet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
-  publish-packer-calibrationnet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
-  publish-packer-butterflynet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG"
-  publish-packer-nerpanet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG"
-  publish-dockerhub:
-    description: publish to dockerhub
-    machine:
-      image: ubuntu-2004:202010-01
-    parameters:
-      tag:
-        type: string
-        default: latest
-    steps:
-      - checkout
-      - run:
-          name: dockerhub login
-          command: echo $DOCKERHUB_PASSWORD | docker login --username $DOCKERHUB_USERNAME --password-stdin
-      - run:
-          name: docker build
-          command: |
-            docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus .
-            docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus .
-            if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then
-              docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus .
-              docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus .
-            fi
-            if [["[[ ! -z $CIRCLE_TAG ]]"]]; then
-              docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus .
-              docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus .
-            fi
-      - run:
-          name: docker push
-          command: |
-            docker push filecoin/lotus:<< parameters.tag >>
-            docker push filecoin/lotus-all-in-one:<< parameters.tag >>
-            if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then
-              docker push filecoin/lotus:$CIRCLE_SHA1
-              docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1
-            fi
-            if [["[[ ! -z $CIRCLE_TAG ]]"]]; then
-              docker push filecoin/lotus:$CIRCLE_TAG
-              docker push filecoin/lotus-all-in-one:$CIRCLE_TAG
-            fi
-
-workflows:
-  version: 2.1
-  ci:
-    jobs:
-      - lint-all:
-          concurrency: "16"   # expend all docker 2xlarge CPUs.
-      - mod-tidy-check
-      - gofmt
-      - gen-check
-      - docs-check
-
-      [[- range $file := .ItestFiles -]]
-      [[ with $name := $file | stripSuffix ]]
-      - test:
-          name: test-itest-[[ $name ]]
-          suite: itest-[[ $name ]]
-          target: "./itests/[[ $file ]]"
-      [[ end ]]
-      [[- end -]]
-
-      [[range $suite, $pkgs := .UnitSuites]]
-      - test:
-          name: test-[[ $suite ]]
-          suite: utest-[[ $suite ]]
-          target: "[[ $pkgs ]]"
-      [[- end]]
-      - test:
-          go-test-flags: "-run=TestMulticoreSDR"
-          suite: multicore-sdr-check
-          target: "./extern/sector-storage/ffiwrapper"
-          proofs-log-test: "1"
-      - test-conformance:
-          suite: conformance
-          codecov-upload: false
-          target: "./conformance"
-      - test-conformance:
-          name: test-conformance-bleeding-edge
-          codecov-upload: false
-          suite: conformance-bleeding-edge
-          target: "./conformance"
-          vectors-branch: master
-      - trigger-testplans:
-          filters:
-            branches:
-              only:
-                - master
-      - build-debug
-      - build-all:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-ntwk-calibration:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-ntwk-butterfly:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-ntwk-nerpa:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-lotus-soup
-      - build-macos:
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-appimage:
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish:
-          requires:
-            - build-all
-            - build-macos
-            - build-appimage
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-and-push-image:
-          dockerfile: Dockerfile.lotus
-          path: .
-          repo: lotus-dev
-          tag: '${CIRCLE_SHA1:0:8}'
-      - publish-packer-mainnet:
-          requires:
-            - build-all
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-calibrationnet:
-          requires:
-            - build-ntwk-calibration
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-butterflynet:
-          requires:
-            - build-ntwk-butterfly
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-nerpanet:
-          requires:
-            - build-ntwk-nerpa
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-snapcraft:
-          name: publish-snapcraft-stable
-          channel: stable
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-dockerhub:
-          name: publish-dockerhub
-          tag: stable
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-
-  nightly:
-    triggers:
-      - schedule:
-          cron: "0 0 * * *"
-          filters:
-            branches:
-              only:
-                - master
-    jobs:
-      - publish-snapcraft:
-          name: publish-snapcraft-nightly
-          channel: edge
-      - publish-dockerhub:
-          name: publish-dockerhub-nightly
-          tag: nightly

.codecov.yml

@@ -1,19 +1,3 @@
 comment: off
 ignore:
-  - "**/cbor_gen.go"
-  - "api/test/**/*"
-  - "api/test/*"
-  - "gen/**/*"
-  - "gen/*"
-  - "cmd/lotus-shed/*"
-  - "cmd/tvx/*"
-  - "cmd/lotus-pcr/*"
-  - "cmd/tvx/*"
-  - "cmd/lotus-chainwatch/*"
-  - "cmd/lotus-health/*"
-  - "cmd/lotus-fountain/*"
-  - "cmd/lotus-townhall/*"
-  - "cmd/lotus-stats/*"
-  - "cmd/lotus-pcr/*"
-github_checks:
-  annotations: false
+  - "cbor_gen.go"

.github/CODEOWNERS

@@ -1,6 +0,0 @@
-# Reference
-# https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/creating-a-repository-on-github/about-code-owners
-
-# Global owners
-# Ensure maintainers team is a requested reviewer for non-draft PRs
-*                       @filecoin-project/lotus-maintainers

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,27 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Run '...'
+2. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Version (run `lotus --version`):**
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,92 +0,0 @@
-name: "Bug Report"
-description: "File a bug report to help us improve"
-labels: [need/triage, kind/bug]
-body:
-- type: checkboxes
-  attributes:
-    label: Checklist
-    description: Please check off the following boxes before continuing to file a bug report!
-    options:
-      - label: This is **not** a security-related bug/issue. If it is, please follow please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
-        required: true
-      - label: This is **not** a question or a support request. If you have any lotus related questions, please ask in the [lotus forum](https://github.com/filecoin-project/lotus/discussions).
-        required: true
-      - label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead.
-        required: true
-      - label: This is **not** an enhancement request. If it is, please file a [improvement suggestion](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Fenhancement&template=enhancement.yml) instead.
-        required: true
-      - label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion.
-        required: true
-      - label: I am running the [`Latest release`](https://github.com/filecoin-project/lotus/releases), or the most recent RC(release canadiate) for the upcoming release or the dev branch(master), or have an issue updating to any of these.
-        required: true
-      - label: I did not make any code changes to lotus.
-        required: false
-- type: dropdown 
-  id: component-and-area
-  validations:
-    required: true
-  attributes:
-    label: Lotus component
-    description: Please select the lotus component you are filing a bug for
-    options:
-    - lotus daemon - chain sync
-    - lotus miner - mining and block production
-    - lotus miner/worker - sealing
-    - lotus miner - proving(WindowPoSt)
-    - lotus miner/market - storage deal
-    - lotus miner/market - retrieval deal
-    - lotus client
-    - lotus JSON-RPC API
-    - lotus message management (mpool)
-    - Other
-- type: textarea
-  id: version
-  attributes:
-    label: Lotus Version
-    description: Enter the output of `lotus version` and `lotus-miner version` if applicable.
-    placeholder: |
-      e.g. 
-      Daemon:1.11.0-rc2+debug+git.0519cd371.dirty+api1.3.0 
-      Local: lotus version 1.11.0-rc2+debug+git.0519cd371.dirty
-  validations:
-    reuiqred: true
-- type: textarea
-  id: Description
-  attributes:
-    label: Describe the Bug
-    description: |
-      This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information:
-      * What you were doding when you experienced the bug?
-      * Any *error* messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas).
-      * What is the expected behaviour?
-      * For sealing issues, include the output of `lotus-miner sectors status --log <sectorId>` for the failed sector(s).
-      * For proving issues, include the output of `lotus-miner proving` info.
-      * For deal making issues, include the output of `lotus client list-deals -v` and/or `lotus-miner storage-deals|retrieval-deals|data-transfers list [-v]` commands for the deal(s) in question.
-    render: bash
-  validations:
-    required: true
-- type: textarea
-  id: extraInfo
-  attributes:
-    label: Logging Information
-    description: |
-      Please provide debug logs of the problem, remember you can get set log level control for:
-      * lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/#log-level-control).
-      * lotus-miner:`lotus-miner log list` to get all log systems available and set level by `lotus-miner log set-level
-      If you don't provide detailed logs when you raise the issue it will almost certainly be the first request I make before furthur diagnosing the problem.
-      render: bash
-  validations:
-    required: true
-- type: textarea
-  id: RepoSteps
-  attributes:
-    label: Repo Steps
-    description: "Steps to reproduce the behavior"
-    value: |
-      1. Run '...'
-      2. Do '...'
-      3. See error '...'
-      ...
-    render: bash
-  validations:
-    required: false

.github/ISSUE_TEMPLATE/enhancement.yml

@@ -1,44 +0,0 @@
-name: Enhancement
-description:  Suggest an improvement to an existing lotus feature.
-labels: [need/triage, kind/enhancement]
-body:
-- type: checkboxes
-  attributes:
-    label: Checklist
-    description: Please check off the following boxes before continuing to create an improvement suggestion!
-    options:
-      - label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is,  please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md).
-        required: true
-      - label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead.
-        required: true
-      - label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`.
-        required: true
-      - label: I **have** a specific, actionable, and well motivated improvement to propose.
-        required: true
-- type: dropdown 
-  id: component
-  validations:
-    required: true
-  attributes:
-    label: Lotus component
-    description: Please select the lotus component you are propoing improvement for
-    options:
-      - lotus daemon - chain sync
-      - lotus miner - mining and block production
-      - lotus miner/worker - sealing
-      - lotus miner - proving(WindowPoSt)
-      - lotus miner/market - storage deal
-      - lotus miner/market - retrieval deal
-      - lotus client
-      - lotus JSON-RPC API
-      - lotus message management (mpool)
-      - Other
-- type: textarea
-  id: request
-  attributes:
-    label: Improvement Suggestion
-    description: A clear and concise description of what the motivation or the current problem is and what is the suggested improvement?
-    placeholder: Ex. Currently lotus... However, as a storage provider, I'd like...
-  validations:
-    required: true
-

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,63 +0,0 @@
-name: Feature request
-description:  Suggest an idea for lotus
-labels: [need/triage, kind/feature]
-body:
-- type: checkboxes
-  attributes:
-    label: Checklist
-    description: Please check off the following boxes before continuing to create a new feature request!
-    options:
-      - label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is,  please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md).
-        required: true
-      - label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`.
-        required: true
-      - label: I **have** a specific, actionable, and well motivated feature request to propose.
-        required: true
-- type: dropdown 
-  id: component
-  validations:
-    required: true
-  attributes:
-    label: Lotus component
-    description: Please select the lotus component you are requesting a new feature for
-    options:
-      - lotus daemon - chain sync
-      - lotus miner - mining and block production
-      - lotus miner/worker - sealing
-      - lotus miner - proving(WindowPoSt)
-      - lotus miner/market - storage deal
-      - lotus miner/market - retrieval deal
-      - lotus client
-      - lotus JSON-RPC API
-      - lotus message management (mpool)
-      - Other
-- type: textarea
-  id: request
-  attributes:
-    label: What is the motivation behind this feature request? Is your feature request related to a problem? Please describe.
-    description: A clear and concise description of what the motivation or the problem is. 
-    placeholder: Ex. I'm always frustrated when [...]
-  validations:
-    required: true
-- type: textarea
-  id: solution
-  attributes:
-    label: Describe the solution you'd like
-    description: A clear and concise description of what you want to happen. 
-  validations:
-    required: true
-- type: textarea
-  id: alternates
-  attributes:
-    label: Describe alternatives you've considered
-    description: A clear and concise description of any alternative solutions or features you've considered.
-  validations:
-    required: false
-- type: textarea
-  id: extra
-  attributes:
-    label: Additional context
-    description: Add any other context, design docs or screenshots about the feature request here.
-  validations:
-    required: false
-

.github/ISSUE_TEMPLATE/m1_bug_report_deal.yml

@@ -1,91 +0,0 @@
-name: "M1 Bug Report For Deal Making"
-description: "File a bug report around deal making for the M1 releases"
-labels: [need/triage, kind/bug, M1-release]
-body:
-- type: checkboxes
-  id: checklist
-  attributes:
-    label: Checklist
-    description: Please check off the following boxes before continuing to file a bug report!
-    options:
-      - label: This is **not** a question or a support request. If you have any lotus related questions, please ask in the [lotus forum](https://github.com/filecoin-project/lotus/discussions).
-        required: true
-      - label: I **am** reporting a bug w.r.t one of the [M1 tags](https://github.com/filecoin-project/lotus/discussions/6852#discussioncomment-1043951). If not, choose another issue option [here](https://github.com/filecoin-project/lotus/issues/new/choose).
-        required: true
-      - label: I **am** reporting a bug around deal making. If not, create a [M1 Bug Report For Non Deal Making Issue](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Fbug%2CM1-release&template=m1_bug_report_non_deal.yml).
-        required: true
-      - label: I have my log level set as instructed [here](https://github.com/filecoin-project/lotus/discussions/6852#discussioncomment-1043678) and have logs available for troubleshooting.
-        required: true
-      - label: The deal is coming from one of the M1 clients(communitcated in the coordination slack channel).
-        required: true
-      - label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion.
-        required: true
-- type: dropdown 
-  id: lotus-componets
-  validations:
-    required: true
-  attributes:
-    label: Lotus Component
-    description: Please select the lotus component you are filing a bug for
-    options:
-    - lotus miner market subsystem - storage deal
-    - lotus miner market subsystem - retrieval deal
-    - lotus miner - storage deal
-    - lotus miner - retrieval deal
-- type: textarea
-  id: version
-  attributes:
-    label: Lotus Tag and  Version
-    description: Enter the  lotus tag, output of `lotus version` and `lotus-miner version`.
-  validations:
-    reuiqred: true
-- type: textarea
-  id: Description
-  attributes:
-    label: Describe the Bug
-    description: |
-      This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information:
-      * What you were doding when you experienced the bug?
-      * Any *error* messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas).
-      * What is the expected behaviour?
-    render: bash
-  validations:
-    required: true
-- type: textarea
-  id: deal-status
-  attributes:
-    label: Deal Status
-    description: What's the status of the deal?
-    placeholder: |
-      Please share the output of `lotus-miner storage-deals|retrieval-deals list [-v]` commands for the deal(s) in question.
-  validations:
-    required: true
-- type: textarea
-  id: data-transfer-status
-  attributes:
-    label: Data Transfer Status
-    description: What's the status of the data transfer?
-    placeholder: |
-      Please share the output of `lotus-miner data-transfers list -v` commands for the deal(s) in question.
-  validations:
-    required: true
-- type: textarea
-  id: logging
-  attributes:
-    label: Logging Information
-    description: Please link to the whole of the miner logs on your side of the transaction. You can upload the logs to a [gist](https://gist.github.com).
-  validations:
-    required: true
-- type: textarea
-  id: RepoSteps
-  attributes:
-    label: Repo Steps (optional)
-    description: "Steps to reproduce the behavior"
-    value: |
-      1. Run '...'
-      2. Do '...'
-      3. See error '...'
-      ...
-    render: bash
-  validations:
-    required: false

.github/ISSUE_TEMPLATE/m1_bug_report_non_deal.yml

@@ -1,81 +0,0 @@
-name: "M1 Bug Report For Non Deal Making Issue"
-description: "File a bug report around non deal making issue for the M1 releases"
-labels: [need/triage, kind/bug, M1-release]
-body:
-- type: checkboxes
-  id: checklist
-  attributes:
-    label: Checklist
-    description: Please check off the following boxes before continuing to file a bug report!
-    options:
-      - label: This is **not** a question or a support request. If you have any lotus related questions, please ask in the [lotus forum](https://github.com/filecoin-project/lotus/discussions).
-        required: true
-      - label: I **am** reporting a bug w.r.t one of the [M1 tags](https://github.com/filecoin-project/lotus/discussions/6852#discussioncomment-1043951). If not, choose another issue option [here](https://github.com/filecoin-project/lotus/issues/new/choose).
-        required: true
-      - label: I am **not** reporting a bug around deal making. If yes, create a [M1 Bug Report For Deal Making](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Fbug%2CM1-release&template=m1_bug_report_deal.yml).
-        required: true
-      - label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion.
-        required: true
-- type: dropdown 
-  id: component-and-area
-  validations:
-    required: true
-  attributes:
-    label: Lotus component
-    description: Please select the lotus component you are filing a bug for
-    options:
-    - lotus daemon - chain sync **with** splitstore enabled
-    - lotus daemon - chain sync **without** splitstore enabled
-    - lotus miner - mining and block production
-    - lotus miner/worker - sealing
-    - lotus miner - proving(WindowPoSt)
-    - lotus client
-    - lotus JSON-RPC API
-    - lotus message management (mpool)
-    - Other
-- type: textarea
-  id: version
-  attributes:
-    label: Lotus Tag and  Version
-    description: Enter the  lotus tag, output of `lotus version` and `lotus-miner version`.
-  validations:
-    reuiqred: true
-- type: textarea
-  id: Description
-  attributes:
-    label: Describe the Bug
-    description: |
-      This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information:
-      * What you were doding when you experienced the bug?
-      * Any *error* messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas).
-      * What is the expected behaviour?
-      * For sealing issues, include the output of `lotus-miner sectors status --log <sectorId>` for the failed sector(s).
-      * For proving issues, include the output of `lotus-miner proving` info.
-    render: bash
-  validations:
-    required: true
-- type: textarea
-  id: extraInfo
-  attributes:
-    label: Logging Information
-    description: |
-      Please provide debug logs of the problem, remember you can get set log level control for:
-      * lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/#log-level-control).
-      * lotus-miner:`lotus-miner log list` to get all log systems available and set level by `lotus-miner log set-level
-      If you don't provide detailed logs when you raise the issue it will almost certainly be the first request I make before furthur diagnosing the problem.
-    render: bash
-  validations:
-    required: true
-- type: textarea
-  id: RepoSteps
-  attributes:
-    label: Repo Steps
-    description: "Steps to reproduce the behavior"
-    value: |
-      1. Run '...'
-      2. Do '...'
-      3. See error '...'
-      ...
-    render: bash
-  validations:
-    required: false

.github/ISSUE_TEMPLATE/sealingfailed.md

@@ -0,0 +1,34 @@
+---
+name: Sealing Issues
+about: Create a report for help with sealing (commit) failures.
+title: ''
+labels: 'sealing'
+assignees: ''
+
+---
+
+Please provide all the information requested here to help us troubleshoot "commit failed" issues.
+If the information requested is missing, we will probably have to just ask you to provide it anyway,
+before we can help debug.
+
+**Describe the problem**
+
+A brief description of the problem you encountered while proving (sealing) a sector.
+
+Including what commands you ran, and a description of your setup, is very helpful.
+
+**Sectors list**
+
+The output of `./lotus-storage-miner sectors list`.
+
+**Sectors status**
+
+The output of `./lotus-storage-miner sectors status --log <sectorId>` for the failed sector(s).
+
+**Lotus storage miner logs**
+
+Please go through the logs of your storage miner, and include screenshots of any error-like messages you find.
+
+**Version**
+
+The output of `./lotus --version`.

.github/labels.yml

@@ -1,248 +0,0 @@
-###
-### Special magic GitHub labels
-### https://help.github.com/en/github/building-a-strong-community/encouraging-helpful-contributions-to-your-project-with-labels
-#
-- name: "good first issue"
-  color: 7057ff
-  description: "Good for newcomers"
-- name: "help wanted"
-  color: 008672
-  description: "Extra attention is needed"
-
-###
-### Goals
-#
-- name: goal/incentives
-  color: ff004d
-  description: "Incentinet"
-
-###
-### Areas
-#
-- name: area/ux
-  color: 00A4E0
-  description: "Area: UX"
-- name: area/chain/vm
-  color: 00A4E2
-  description: "Area: Chain/VM"
-- name: area/chain/sync
-  color: 00A4E4
-  description: "Area: Chain/Sync"
-- name: area/chain/misc
-  color: 00A4E6
-  description: "Area: Chain/Misc"
-- name: area/markets
-  color: 00A4E8
-  description: "Area: Markets"  
-- name: area/sealing/fsm
-  color: 0bb1ed
-  description: "Area: Sealing/FSM"
-- name: area/sealing/storage
-  color: 0EB4F0
-  description: "Area: Sealing/Storage"
-- name: area/proving
-  color: 0EB4F0
-  description: "Area: Proving"
-- name: area/mining
-  color: 10B6F2
-  description: "Area: Mining"
-- name: area/client/storage
-  color: 13B9F5
-  description: "Area: Client/Storage"
-- name: area/client/retrieval
-  color: 15BBF7
-  description: "Area: Client/Retrieval"
-- name: area/wallet
-  color: 15BBF7
-  description: "Area: Wallet"
-- name: area/payment-channel
-  color: ff6767
-  description: "Area: Payment Channel"
-- name: area/multisig
-  color: fff0ff
-  description: "Area: Multisig"
-- name: area/networking
-  color: 273f8a
-  description: "Area: Networking"
-
-###
-### Kinds
-#
-- name: kind/bug
-  color: c92712
-  description: "Kind: Bug"
-- name: kind/chore
-  color: fcf0b5
-  description: "Kind: Chore"
-- name: kind/feature
-  color: FFF3B8
-  description: "Kind: Feature"
-- name: kind/improvement
-  color: FFF5BA
-  description: "Kind: Improvement"
-- name: kind/test
-  color: FFF8BD
-  description: "Kind: Test"
-- name: kind/question
-  color: FFFDC2
-  description: "Kind: Question"
-- name: kind/enhancement
-  color: FFFFC5
-  description: "Kind: Enhancement"
-- name: kind/discussion
-  color: FFFFC7
-  description: "Kind: Discussion"
-
-###
-### Difficulties
-#
-- name: dif/trivial
-  color: b2b7ff
-  description: "Can be confidently tackled by newcomers, who are widely unfamiliar with lotus"
-- name: dif/easy
-  color: 7886d7
-  description: "An existing lotus user should be able to pick this up"
-- name: dif/medium
-  color: 6574cd
-  description: "Prior development experience with lotus is likely helpful"
-- name: dif/hard
-  color: 5661b3
-  description: "Suggests that having worked on the specific component affected by this issue is important"
-- name: dif/expert
-  color: 2f365f
-  description: "Requires extensive knowledge of the history, implications, ramifications of the issue"
-
-###
-### Efforts
-#
-- name: effort/minutes
-  color: e8fffe
-  description: "Effort: Minutes"
-- name: effort/hours
-  color: a0f0ed
-  description: "Effort: Hours"
-- name: effort/day
-  color: 64d5ca
-  description: "Effort: One Day"
-- name: effort/days
-  color: 4dc0b5
-  description: "Effort: Multiple Days"
-- name: effort/week
-  color: 38a89d
-  description: "Effort: One Week"
-- name: effort/weeks
-  color: 20504f
-  description: "Effort: Multiple Weeks"
-
-###
-### Impacts
-#
-- name: impact/regression
-  color: f1f5f8
-  description: "Impact: Regression"
-- name: impact/api-breakage
-  color: ECF0F3
-  description: "Impact: API Breakage"
-- name: impact/quality
-  color: E7EBEE
-  description: "Impact: Quality"
-- name: impact/dx
-  color: E2E6E9
-  description: "Impact: Developer Experience"
-- name: impact/test-flakiness
-  color: DDE1E4
-  description: "Impact: Test Flakiness"
-- name: impact/consensus
-  color: b20014
-  description: "Impact: Consensus"
-
-###
-### Topics
-#
-- name: topic/interoperability
-  color: bf0f73
-  description: "Topic: Interoperability"
-- name: topic/specs
-  color: CC1C80
-  description: "Topic: Specs"
-- name: topic/docs
-  color: D9298D
-  description: "Topic: Documentation"
-- name: topic/architecture
-  color: E53599
-  description: "Topic: Architecture"
-
-###
-### Priorities
-###
-- name: P0
-  color: dd362a
-  description: "P0: Critical Blocker"
-- name: P1
-  color: ce8048
-  description: "P1: Must be resolved"
-- name: P2
-  color: dbd81a
-  description: "P2: Should be resolved"
-- name: P3
-  color: 9fea8f
-  description: "P3: Might get resolved"
-
-###
-### Hints
-#
-#- name: hint/good-first-issue
-#  color: 7057ff
-#  description: "Hint: Good First Issue"
-#- name: hint/help-wanted
-#  color: 008672
-#  description: "Hint: Help Wanted"
-- name: hint/needs-decision
-  color: 33B9A5
-  description: "Hint: Needs Decision"
-- name: hint/needs-triage
-  color: 1AA08C
-  description: "Hint: Needs Triage"
-- name: hint/needs-analysis
-  color: 26AC98
-  description: "Hint: Needs Analysis"
-- name: hint/needs-author-input
-  color: 33B9A5
-  description: "Hint: Needs Author Input"
-- name: hint/needs-team-input
-  color: 40C6B2
-  description: "Hint: Needs Team Input"
-- name: hint/needs-community-input
-  color: 4DD3BF
-  description: "Hint: Needs Community Input"
-- name: hint/needs-review
-  color: 5AE0CC
-  description: "Hint: Needs Review"
-
-###
-### Statuses
-#
-- name: status/done
-  color: edb3a6
-  description: "Status: Done"
-- name: status/deferred
-  color: E0A699
-  description: "Status: Deferred"
-- name: status/in-progress
-  color: D49A8D
-  description: "Status: In Progress"
-- name: status/blocked
-  color: C78D80
-  description: "Status: Blocked"
-- name: status/inactive
-  color: BA8073
-  description: "Status: Inactive"
-- name: status/waiting
-  color: AD7366
-  description: "Status: Waiting"
-- name: status/rotten
-  color: 7A4033
-  description: "Status: Rotten"
-- name: status/discarded
-  color: 6D3326
-  description: "Status: Discarded / Won't fix"

.github/workflows/codeql-analysis.yml

@@ -1,69 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ master ]
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'go' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
-        # Learn more:
-        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-
-    - uses: actions/setup-go@v1
-      with:
-        go-version: '1.16.4'
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.github/workflows/label-syncer.yml

@@ -1,17 +0,0 @@
-
-name: Label syncer
-on:
-  push:
-    paths:
-      - '.github/labels.yml'
-    branches:
-      - master
-jobs:
-  build:
-    name: Sync labels
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@1.0.0
-      - uses: micnncim/action-label-syncer@v1.0.0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/stale.yml

@@ -1,33 +0,0 @@
-name: Close and mark stale issue
-
-on:
-  schedule:
-  - cron: '0 12 * * *'
-
-jobs:
-  stale:
-
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-
-    steps:
-    - uses: actions/stale@v3
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 24 hours.'
-        close-issue-message: 'This issue was closed because it is missing author input.'
-        stale-pr-message: 'Thank you for submitting the PR and contributing to lotus! Lotus maintainers need more of your input before merging it, please address the suggested changes or reply to the comments or this PR will be closed in 48 hours. You are always more than welcome to reopen the PR later as well!'
-        close-pr-message: 'This PR was closed because it is missing author input. Please feel free to reopen the PR when you get to it! Thank you for your interest in contributing to lotus!'
-        stale-issue-label: 'kind/stale'
-        stale-pr-label: 'kind/stale'
-        any-of-labels: 'need/author-input '
-        days-before-issue-stale: 3
-        days-before-issue-close: 1
-        days-before-pr-stale: 5
-        days-before-pr-close: 2
-        remove-stale-when-updated: true
-        enable-statistics: true
-       
-

.github/workflows/testground-on-push.yml

@@ -1,29 +0,0 @@
----
-name: Testground PR Checker
-
-on: [push]
-
-jobs:
-  testground:
-    runs-on: ubuntu-latest
-    name: ${{ matrix.composition_file }}
-    strategy:
-      matrix:
-        include:
-          - backend_addr: ci.testground.ipfs.team
-            backend_proto: https
-            plan_directory: testplans/lotus-soup
-            composition_file: testplans/lotus-soup/_compositions/baseline-k8s-3-1.toml
-          - backend_addr: ci.testground.ipfs.team
-            backend_proto: https
-            plan_directory: testplans/lotus-soup
-            composition_file: testplans/lotus-soup/_compositions/paych-stress-k8s.toml
-    steps:
-      - uses: actions/checkout@v2
-      - name: testground run
-        uses: coryschwartz/testground-github-action@v1.1
-        with:
-          backend_addr: ${{ matrix.backend_addr }}
-          backend_proto: ${{ matrix.backend_proto }}
-          plan_directory: ${{ matrix.plan_directory }}
-          composition_file: ${{ matrix.composition_file }}

.gitignore

@@ -1,31 +1,19 @@
-/AppDir
-/appimage-builder-cache
-*.AppImage
 /lotus
-/lotus-miner
-/lotus-worker
+/lotus-storage-miner
+/lotus-seal-worker
 /lotus-seed
 /lotus-health
-/lotus-chainwatch
 /lotus-shed
-/lotus-sim
-/lotus-pond
-/lotus-townhall
-/lotus-fountain
-/lotus-stats
-/lotus-bench
-/lotus-gateway
-/lotus-pcr
-/lotus-wallet
-/lotus-keygen
-/docgen-md
-/docgen-openrpc
+/pond
+/townhall
+/fountain
+/stats
+/bench
 /bench.json
 /lotuspond/front/node_modules
 /lotuspond/front/build
 /cmd/lotus-townhall/townhall/node_modules
 /cmd/lotus-townhall/townhall/build
-/cmd/lotus-townhall/townhall/package-lock.json
 extern/filecoin-ffi/rust/target
 **/*.a
 **/*.pc
@@ -36,6 +24,7 @@ build/paramfetch.sh
 /vendor
 /blocks.dot
 /blocks.svg
+/chainwatch
 /chainwatch.db
 /bundle
 /darwin
@@ -46,4 +35,3 @@ build/paramfetch.sh
 bin/ipget
 bin/tmp/*
 .idea
-scratchpad

.gitmodules

@@ -1,9 +1,7 @@
 [submodule "extern/filecoin-ffi"]
 	path = extern/filecoin-ffi
 	url = https://github.com/filecoin-project/filecoin-ffi.git
+	branch = master
 [submodule "extern/serialization-vectors"]
 	path = extern/serialization-vectors
-	url = https://github.com/filecoin-project/serialization-vectors.git
-[submodule "extern/test-vectors"]
-	path = extern/test-vectors
-	url = https://github.com/filecoin-project/test-vectors.git
+	url = https://github.com/filecoin-project/serialization-vectors

.golangci.yml

@@ -16,12 +16,6 @@ linters:
     - deadcode
     - scopelint
 
-# We don't want to skip builtin/
-skip-dirs-use-default: false
-skip-dirs:
-  - vendor$
-  - testdata$
-  - examples$
 
 issues:
   exclude:
@@ -29,14 +23,6 @@ issues:
     - "Potential file inclusion via variable"
     - "should have( a package)? comment"
     - "Error return value of `logging.SetLogLevel` is not checked"
-    - "comment on exported"
-    - "(func|method) \\w+ should be \\w+"
-    - "(type|var|struct field|(method|func) parameter) `\\w+` should be `\\w+`"
-    - "(G306|G301|G307|G108|G302|G204|G104)"
-    - "don't use ALL_CAPS in Go names"
-    - "string .* has .* occurrences, make it a constant"
-    - "a blank import should be only in a main or test package, or have a comment justifying it"
-    - "package comment should be of the form"
 
   exclude-use-default: false
   exclude-rules:
@@ -60,19 +46,6 @@ issues:
       linters:
         - gosec
 
-    - path: chain/vectors/gen/.*
-      linters:
-        - gosec
-
-    - path: cmd/lotus-bench/.*
-      linters:
-        - gosec
-
-    - path: api/test/.*
-      text: "context.Context should be the first parameter"
-      linters:
-        - golint
-
 linters-settings:
   goconst:
     min-occurrences: 6

AppDir/usr/share/icons/icon.svg

@@ -1 +0,0 @@
-<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0" y="0" viewBox="0 0 127 127" xml:space="preserve" enable-background="new 0 0 127 127"><style type="text/css">.st0{fill:#00d2d6}.st1{fill:#fff}</style><g><path class="st0" d="M63.5,127C28.5,127.1-0.2,98.4,0,63.2C0.2,28.3,28.6-0.2,63.9,0c34.8,0.2,63.3,28.7,63.1,64 C126.7,98.7,98.5,127.1,63.5,127z M71.4,57.6c5.5,0.8,11,1.5,16.5,2.3c0.5-1.7,0.9-3.1,1.3-4.7c-5.7-0.8-11.2-1.7-17.1-2.5 c2-7,3.7-13.7,5.8-20.2c0.7-2.2,2.3-4.2,3.9-5.9c2.1-2.2,5-1.7,6.8,0.7c0.7,1,1.4,2.1,2.3,2.9c1.1,1.1,2.8,1.6,4,0.6 c0.8-0.7,0.7-2.4,0.8-3.6c0-0.5-0.6-1.1-1-1.6c-2-2.3-4.7-3.1-7.5-3.2c-6.3-0.3-10.9,3-14.5,7.8c-3.5,4.8-5.1,10.5-6.8,16.2 c-0.5,1.6-0.9,3.3-1.4,5.1c-6.2-0.9-12.1-1.7-18.2-2.6c-0.2,1.6-0.4,3.2-0.6,4.8c6,0.9,11.8,1.8,17.8,2.7c-0.8,3.4-1.5,6.5-2.3,9.7 c-5.8-0.8-11.4-1.6-17-2.4c-0.2,1.8-0.4,3.2-0.6,4.8c5.6,0.9,11,1.7,16.5,2.5c0,0.6,0.1,1,0,1.3c-1.7,7.4-3.4,14.8-5.3,22.2 c-0.9,3.5-2.4,6.9-5.3,9.3c-2.4,2-5,1.7-6.8-0.8c-0.8-1.1-1.5-2.5-2.6-3.3c-0.8-0.6-2.5-0.9-3.1-0.5c-0.9,0.7-1.5,2.2-1.4,3.3 c0.1,1,1,2.2,1.9,2.8c3,2.3,6.5,2.6,10,1.9c5.9-1.2,10.1-4.9,12.7-10.1c2-4.1,3.6-8.5,5-12.9c1.3-4,2.2-8,3.3-12.2 c5.8,0.8,11.5,1.7,17.3,2.5c0.5-1.7,0.9-3.2,1.4-4.8c-6.1-0.9-11.9-1.7-17.7-2.6C70.1,64,70.7,60.9,71.4,57.6z"/><path class="st1" d="M71.4,57.6c-0.7,3.3-1.3,6.4-2,9.8c5.9,0.9,11.7,1.7,17.7,2.6c-0.5,1.6-0.9,3.1-1.4,4.8 c-5.8-0.8-11.5-1.7-17.3-2.5c-1.1,4.2-2,8.3-3.3,12.2c-1.4,4.4-3,8.7-5,12.9c-2.6,5.2-6.8,8.9-12.7,10.1c-3.5,0.7-7,0.4-10-1.9 c-0.9-0.7-1.8-1.8-1.9-2.8c-0.1-1.1,0.5-2.7,1.4-3.3c0.6-0.5,2.3-0.1,3.1,0.5c1.1,0.8,1.8,2.1,2.6,3.3c1.8,2.5,4.4,2.9,6.8,0.8 c2.9-2.5,4.4-5.8,5.3-9.3c1.9-7.3,3.6-14.8,5.3-22.2c0.1-0.3,0-0.7,0-1.3c-5.4-0.8-10.8-1.7-16.5-2.5c0.2-1.6,0.4-3,0.6-4.8 c5.6,0.8,11.1,1.6,17,2.4c0.8-3.2,1.5-6.4,2.3-9.7c-6-0.9-11.7-1.8-17.8-2.7c0.2-1.6,0.4-3.2,0.6-4.8c6.1,0.9,12,1.7,18.2,2.6 c0.5-1.8,0.9-3.5,1.4-5.1c1.7-5.6,3.2-11.3,6.8-16.2c3.6-4.9,8.1-8.1,14.5-7.8c2.8,0.1,5.5,0.9,7.5,3.2c0.4,0.5,1,1.1,1,1.6 c-0.1,1.2,0,2.9-0.8,3.6c-1.1,1.1-2.8,0.5-4-0.6c-0.9-0.9-1.6-1.9-2.3-2.9c-1.8-2.4-4.7-2.9-6.8-0.7c-1.6,1.7-3.2,3.7-3.9,5.9 C75.7,39.4,74,46,72,53c5.9,0.9,11.4,1.7,17.1,2.5c-0.5,1.6-0.9,3.1-1.3,4.7C82.3,59.1,76.9,58.3,71.4,57.6z"/></g></svg>

AppImageBuilder.yml

@@ -1,73 +0,0 @@
-version: 1
-AppDir:
-  path: ./AppDir
-  app_info:
-    id: io.filecoin.lotus
-    name: Lotus
-    icon: icon
-    version: latest
-    exec: usr/bin/lotus
-    exec_args: $@
-  apt:
-    arch: amd64
-    allow_unauthenticated: true
-    sources:
-    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal main restricted
-    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-updates main restricted
-    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal universe
-    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-updates universe
-    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal multiverse
-    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-updates multiverse
-    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-backports main restricted
-        universe multiverse
-    - sourceline: deb http://security.ubuntu.com/ubuntu focal-security main restricted
-    - sourceline: deb http://security.ubuntu.com/ubuntu focal-security universe
-    - sourceline: deb http://security.ubuntu.com/ubuntu focal-security multiverse
-    - sourceline: deb https://cli-assets.heroku.com/apt ./
-    - sourceline: deb http://ppa.launchpad.net/openjdk-r/ppa/ubuntu focal main
-    - sourceline: deb http://ppa.launchpad.net/git-core/ppa/ubuntu focal main
-    - sourceline: deb http://archive.canonical.com/ubuntu focal partner
-    include:
-    - ocl-icd-libopencl1
-    - libhwloc15
-    exclude: []
-  files:
-    include:
-    - /usr/lib/x86_64-linux-gnu/libgcc_s.so.1
-    - /usr/lib/x86_64-linux-gnu/libpthread-2.31.so
-    - /usr/lib/x86_64-linux-gnu/libm-2.31.so
-    - /usr/lib/x86_64-linux-gnu/libdl-2.31.so
-    - /usr/lib/x86_64-linux-gnu/libc-2.31.so
-    - /usr/lib/x86_64-linux-gnu/libudev.so.1.6.17
-    exclude:
-    - usr/share/man
-    - usr/share/doc/*/README.*
-    - usr/share/doc/*/changelog.*
-    - usr/share/doc/*/NEWS.*
-    - usr/share/doc/*/TODO.*
-  test:
-    fedora:
-      image: appimagecrafters/tests-env:fedora-30
-      command: ./AppRun
-      use_host_x: true
-    debian:
-      image: appimagecrafters/tests-env:debian-stable
-      command: ./AppRun
-      use_host_x: true
-    arch:
-      image: appimagecrafters/tests-env:archlinux-latest
-      command: ./AppRun
-      use_host_x: true
-    centos:
-      image: appimagecrafters/tests-env:centos-7
-      command: ./AppRun
-      use_host_x: true
-    ubuntu:
-      image: appimagecrafters/tests-env:ubuntu-xenial
-      command: ./AppRun
-      use_host_x: true
-AppImage:
-  arch: x86_64
-  update-information: guess
-  sign-key: None
-

Dockerfile.lotus

@@ -1,228 +0,0 @@
-FROM golang:1.16.4 AS builder-deps
-MAINTAINER Lotus Development Team
-
-RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
-
-ARG RUST_VERSION=nightly
-ENV XDG_CACHE_HOME="/tmp"
-
-ENV RUSTUP_HOME=/usr/local/rustup \
-    CARGO_HOME=/usr/local/cargo \
-    PATH=/usr/local/cargo/bin:$PATH
-
-RUN wget "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init"; \
-    chmod +x rustup-init; \
-    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION; \
-    rm rustup-init; \
-    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
-    rustup --version; \
-    cargo --version; \
-    rustc --version;
-
-
-FROM builder-deps AS builder-local
-MAINTAINER Lotus Development Team
-
-COPY ./ /opt/filecoin
-WORKDIR /opt/filecoin
-RUN make clean deps
-
-
-FROM builder-local AS builder
-MAINTAINER Lotus Development Team
-
-WORKDIR /opt/filecoin
-
-ARG RUSTFLAGS=""
-ARG GOFLAGS=""
-
-RUN make lotus lotus-miner lotus-worker lotus-shed lotus-wallet lotus-gateway
-
-
-FROM ubuntu:20.04 AS base
-MAINTAINER Lotus Development Team
-
-# Base resources
-COPY --from=builder /etc/ssl/certs                           /etc/ssl/certs
-COPY --from=builder /lib/x86_64-linux-gnu/libdl.so.2         /lib/
-COPY --from=builder /lib/x86_64-linux-gnu/librt.so.1         /lib/
-COPY --from=builder /lib/x86_64-linux-gnu/libgcc_s.so.1      /lib/
-COPY --from=builder /lib/x86_64-linux-gnu/libutil.so.1       /lib/
-COPY --from=builder /usr/lib/x86_64-linux-gnu/libltdl.so.7   /lib/
-COPY --from=builder /usr/lib/x86_64-linux-gnu/libnuma.so.1   /lib/
-COPY --from=builder /usr/lib/x86_64-linux-gnu/libhwloc.so.5  /lib/
-COPY --from=builder /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /lib/
-
-RUN useradd -r -u 532 -U fc
-
-
-###
-FROM base AS lotus
-MAINTAINER Lotus Development Team
-
-COPY --from=builder /opt/filecoin/lotus /usr/local/bin/
-COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/
-COPY scripts/docker-lotus-entrypoint.sh /
-
-ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV LOTUS_PATH /var/lib/lotus
-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831
-ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
-ENV DOCKER_LOTUS_IMPORT_WALLET ""
-
-RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters
-RUN chown fc: /var/lib/lotus /var/tmp/filecoin-proof-parameters
-
-VOLUME /var/lib/lotus
-VOLUME /var/tmp/filecoin-proof-parameters
-
-USER fc
-
-EXPOSE 1234
-
-ENTRYPOINT ["/docker-lotus-entrypoint.sh"]
-
-CMD ["-help"]
-
-###
-FROM base AS lotus-wallet
-MAINTAINER Lotus Development Team
-
-COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/
-
-ENV WALLET_PATH /var/lib/lotus-wallet
-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831
-
-RUN mkdir /var/lib/lotus-wallet
-RUN chown fc: /var/lib/lotus-wallet
-
-VOLUME /var/lib/lotus-wallet
-
-USER fc
-
-EXPOSE 1777
-
-ENTRYPOINT ["/usr/local/bin/lotus-wallet"]
-
-CMD ["-help"]
-
-###
-FROM base AS lotus-gateway
-MAINTAINER Lotus Development Team
-
-COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/
-
-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831
-ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
-
-USER fc
-
-EXPOSE 1234
-
-ENTRYPOINT ["/usr/local/bin/lotus-gateway"]
-
-CMD ["-help"]
-
-
-###
-FROM base AS lotus-miner
-MAINTAINER Lotus Development Team
-
-COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/
-COPY scripts/docker-lotus-miner-entrypoint.sh /
-
-ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
-ENV LOTUS_MINER_PATH /var/lib/lotus-miner
-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831
-ENV DOCKER_LOTUS_MINER_INIT true
-
-RUN mkdir /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters
-RUN chown fc: /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters
-
-VOLUME /var/lib/lotus-miner
-VOLUME /var/tmp/filecoin-proof-parameters
-
-USER fc
-
-EXPOSE 2345
-
-ENTRYPOINT ["/docker-lotus-miner-entrypoint.sh"]
-
-CMD ["-help"]
-
-
-###
-FROM base AS lotus-worker
-MAINTAINER Lotus Development Team
-
-COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/
-
-ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV MINER_API_INFO /ip4/127.0.0.1/tcp/2345/http
-ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831
-
-RUN mkdir /var/lib/lotus-worker
-RUN chown fc: /var/lib/lotus-worker
-
-VOLUME /var/lib/lotus-worker
-
-USER fc
-
-EXPOSE 3456
-
-ENTRYPOINT ["/usr/local/bin/lotus-worker"]
-
-CMD ["-help"]
-
-
-###
-from base as lotus-all-in-one
-
-ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831
-ENV LOTUS_MINER_PATH /var/lib/lotus-miner
-ENV LOTUS_PATH /var/lib/lotus
-ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
-ENV MINER_API_INFO /ip4/127.0.0.1/tcp/2345/http
-ENV WALLET_PATH /var/lib/lotus-wallet
-ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
-ENV DOCKER_LOTUS_MINER_INIT true
-
-COPY --from=builder /opt/filecoin/lotus      /usr/local/bin/
-COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/
-COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/
-COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/
-COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/
-COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/
-
-RUN mkdir /var/tmp/filecoin-proof-parameters
-RUN mkdir /var/lib/lotus
-RUN mkdir /var/lib/lotus-miner
-RUN mkdir /var/lib/lotus-worker
-RUN mkdir /var/lib/lotus-wallet
-RUN chown fc: /var/tmp/filecoin-proof-parameters
-RUN chown fc: /var/lib/lotus
-RUN chown fc: /var/lib/lotus-miner
-RUN chown fc: /var/lib/lotus-worker
-RUN chown fc: /var/lib/lotus-wallet
-
-
-VOLUME /var/tmp/filecoin-proof-parameters
-VOLUME /var/lib/lotus
-VOLUME /var/lib/lotus-miner
-VOLUME /var/lib/lotus-worker
-VOLUME /var/lib/lotus-wallet
-
-EXPOSE 1234
-EXPOSE 2345
-EXPOSE 3456
-EXPOSE 1777

Makefile

@@ -5,10 +5,10 @@ all: build
 
 unexport GOFLAGS
 
-GOVERSION:=$(shell go version | cut -d' ' -f 3 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
-ifeq ($(shell expr $(GOVERSION) \< 1016000), 1)
-$(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell expr $(GOVERSION) % 1000000 / 1000).$(shell expr $(GOVERSION) % 1000))
-$(error Update Golang to version to at least 1.16.0)
+GOVERSION:=$(shell go version | cut -d' ' -f 3 | cut -d. -f 2)
+ifeq ($(shell expr $(GOVERSION) \< 13), 1)
+$(warning Your Golang version is go 1.$(GOVERSION))
+$(error Update Golang to version $(shell grep '^go' go.mod))
 endif
 
 # git modules that need to be loaded
@@ -41,13 +41,8 @@ MODULES+=$(FFI_PATH)
 BUILD_DEPS+=build/.filecoin-install
 CLEAN+=build/.filecoin-install
 
-ffi-version-check:
-	@[[ "$$(awk '/const Version/{print $$5}' extern/filecoin-ffi/version.go)" -eq 3 ]] || (echo "FFI version mismatch, update submodules"; exit 1)
-BUILD_DEPS+=ffi-version-check
-
-.PHONY: ffi-version-check
-
 $(MODULES): build/.update-modules ;
+
 # dummy file that marks the last time modules were updated
 build/.update-modules:
 	git submodule update --init --recursive
@@ -62,83 +57,71 @@ CLEAN+=build/.update-modules
 deps: $(BUILD_DEPS)
 .PHONY: deps
 
-build-devnets: build lotus-seed lotus-shed lotus-wallet lotus-gateway
-.PHONY: build-devnets
-
 debug: GOFLAGS+=-tags=debug
-debug: build-devnets
+debug: lotus lotus-storage-miner lotus-seal-worker lotus-seed
 
 2k: GOFLAGS+=-tags=2k
-2k: build-devnets
-
-calibnet: GOFLAGS+=-tags=calibnet
-calibnet: build-devnets
-
-nerpanet: GOFLAGS+=-tags=nerpanet
-nerpanet: build-devnets
-
-butterflynet: GOFLAGS+=-tags=butterflynet
-butterflynet: build-devnets
-
-interopnet: GOFLAGS+=-tags=interopnet
-interopnet: build-devnets
+2k: lotus lotus-storage-miner lotus-seal-worker lotus-seed
 
 lotus: $(BUILD_DEPS)
 	rm -f lotus
 	go build $(GOFLAGS) -o lotus ./cmd/lotus
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus -i ./build
 
 .PHONY: lotus
 BINS+=lotus
 
-lotus-miner: $(BUILD_DEPS)
-	rm -f lotus-miner
-	go build $(GOFLAGS) -o lotus-miner ./cmd/lotus-miner
-.PHONY: lotus-miner
-BINS+=lotus-miner
+lotus-storage-miner: $(BUILD_DEPS)
+	rm -f lotus-storage-miner
+	go build $(GOFLAGS) -o lotus-storage-miner ./cmd/lotus-storage-miner
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus-storage-miner -i ./build
+.PHONY: lotus-storage-miner
+BINS+=lotus-storage-miner
 
-lotus-worker: $(BUILD_DEPS)
-	rm -f lotus-worker
-	go build $(GOFLAGS) -o lotus-worker ./cmd/lotus-seal-worker
-.PHONY: lotus-worker
-BINS+=lotus-worker
+lotus-seal-worker: $(BUILD_DEPS)
+	rm -f lotus-seal-worker
+	go build $(GOFLAGS) -o lotus-seal-worker ./cmd/lotus-seal-worker
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus-seal-worker -i ./build
+.PHONY: lotus-seal-worker
+BINS+=lotus-seal-worker
 
 lotus-shed: $(BUILD_DEPS)
 	rm -f lotus-shed
 	go build $(GOFLAGS) -o lotus-shed ./cmd/lotus-shed
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus-shed -i ./build
 .PHONY: lotus-shed
 BINS+=lotus-shed
 
-lotus-gateway: $(BUILD_DEPS)
-	rm -f lotus-gateway
-	go build $(GOFLAGS) -o lotus-gateway ./cmd/lotus-gateway
-.PHONY: lotus-gateway
-BINS+=lotus-gateway
-
-build: lotus lotus-miner lotus-worker
+build: lotus lotus-storage-miner lotus-seal-worker
 	@[[ $$(type -P "lotus") ]] && echo "Caution: you have \
 an existing lotus binary in your PATH. This may cause problems if you don't run 'sudo make install'" || true
 
 .PHONY: build
 
-install: install-daemon install-miner install-worker
-
-install-daemon:
+install:
 	install -C ./lotus /usr/local/bin/lotus
+	install -C ./lotus-storage-miner /usr/local/bin/lotus-storage-miner
+	install -C ./lotus-seal-worker /usr/local/bin/lotus-seal-worker
 
-install-miner:
-	install -C ./lotus-miner /usr/local/bin/lotus-miner
+install-services: install
+	mkdir -p /usr/local/lib/systemd/system
+	install -C -m 0644 ./scripts/lotus-daemon.service /usr/local/lib/systemd/system/lotus-daemon.service
+	install -C -m 0644 ./scripts/lotus-miner.service /usr/local/lib/systemd/system/lotus-miner.service
+	systemctl daemon-reload
+	@echo
+	@echo "lotus and lotus-miner services installed. Don't forget to 'systemctl enable lotus|lotus-miner' for it to be enabled on startup."
 
-install-worker:
-	install -C ./lotus-worker /usr/local/bin/lotus-worker
-
-install-app:
-	install -C ./$(APP) /usr/local/bin/$(APP)
+clean-services:
+	rm -f /usr/local/lib/systemd/system/lotus-daemon.service
+	rm -f /usr/local/lib/systemd/system/lotus-miner.service
+	systemctl daemon-reload
 
 # TOOLS
 
 lotus-seed: $(BUILD_DEPS)
 	rm -f lotus-seed
 	go build $(GOFLAGS) -o lotus-seed ./cmd/lotus-seed
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus-seed -i ./build
 
 .PHONY: lotus-seed
 BINS+=lotus-seed
@@ -149,164 +132,71 @@ benchmarks:
 	@curl -X POST 'http://benchmark.kittyhawk.wtf/benchmark' -d '@bench.json' -u "${benchmark_http_cred}"
 .PHONY: benchmarks
 
-lotus-pond: 2k
-	go build -o lotus-pond ./lotuspond
-.PHONY: lotus-pond
-BINS+=lotus-pond
-
-lotus-pond-front:
+pond: 2k
+	go build -o pond ./lotuspond
 	(cd lotuspond/front && npm i && CI=false npm run build)
-.PHONY: lotus-pond-front
+.PHONY: pond
+BINS+=pond
 
-lotus-pond-app: lotus-pond-front lotus-pond
-.PHONY: lotus-pond-app
-
-lotus-townhall:
-	rm -f lotus-townhall
-	go build -o lotus-townhall ./cmd/lotus-townhall
-.PHONY: lotus-townhall
-BINS+=lotus-townhall
-
-lotus-townhall-front:
+townhall:
+	rm -f townhall
+	go build -o townhall ./cmd/lotus-townhall
 	(cd ./cmd/lotus-townhall/townhall && npm i && npm run build)
-.PHONY: lotus-townhall-front
+	go run github.com/GeertJohan/go.rice/rice append --exec townhall -i ./cmd/lotus-townhall -i ./build
+.PHONY: townhall
+BINS+=townhall
 
-lotus-townhall-app: lotus-touch lotus-townhall-front
-.PHONY: lotus-townhall-app
+fountain:
+	rm -f fountain
+	go build -o fountain ./cmd/lotus-fountain
+	go run github.com/GeertJohan/go.rice/rice append --exec fountain -i ./cmd/lotus-fountain -i ./build
+.PHONY: fountain
+BINS+=fountain
 
-lotus-fountain:
-	rm -f lotus-fountain
-	go build -o lotus-fountain ./cmd/lotus-fountain
-.PHONY: lotus-fountain
-BINS+=lotus-fountain
+chainwatch:
+	rm -f chainwatch
+	go build -o chainwatch ./cmd/lotus-chainwatch
+	go run github.com/GeertJohan/go.rice/rice append --exec chainwatch -i ./cmd/lotus-chainwatch -i ./build
+.PHONY: chainwatch
+BINS+=chainwatch
 
-lotus-chainwatch:
-	rm -f lotus-chainwatch
-	go build $(GOFLAGS) -o lotus-chainwatch ./cmd/lotus-chainwatch
-.PHONY: lotus-chainwatch
-BINS+=lotus-chainwatch
+bench:
+	rm -f bench
+	go build -o bench ./cmd/lotus-bench
+	go run github.com/GeertJohan/go.rice/rice append --exec bench -i ./build
+.PHONY: bench
+BINS+=bench
 
-lotus-bench:
-	rm -f lotus-bench
-	go build -o lotus-bench ./cmd/lotus-bench
-.PHONY: lotus-bench
-BINS+=lotus-bench
+stats:
+	rm -f stats
+	go build -o stats ./tools/stats
+	go run github.com/GeertJohan/go.rice/rice append --exec stats -i ./build
+.PHONY: stats
+BINS+=stats
 
-lotus-stats:
-	rm -f lotus-stats
-	go build $(GOFLAGS) -o lotus-stats ./cmd/lotus-stats
-.PHONY: lotus-stats
-BINS+=lotus-stats
-
-lotus-pcr:
-	rm -f lotus-pcr
-	go build $(GOFLAGS) -o lotus-pcr ./cmd/lotus-pcr
-.PHONY: lotus-pcr
-BINS+=lotus-pcr
-
-lotus-health:
+health:
 	rm -f lotus-health
 	go build -o lotus-health ./cmd/lotus-health
-.PHONY: lotus-health
-BINS+=lotus-health
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus-health -i ./build
 
-lotus-wallet:
-	rm -f lotus-wallet
-	go build -o lotus-wallet ./cmd/lotus-wallet
-.PHONY: lotus-wallet
-BINS+=lotus-wallet
-
-lotus-keygen:
-	rm -f lotus-keygen
-	go build -o lotus-keygen ./cmd/lotus-keygen
-.PHONY: lotus-keygen
-BINS+=lotus-keygen
-
-testground:
-	go build -tags testground -o /dev/null ./cmd/lotus
-.PHONY: testground
-BINS+=testground
-
-
-tvx:
-	rm -f tvx
-	go build -o tvx ./cmd/tvx
-.PHONY: tvx
-BINS+=tvx
-
-install-chainwatch: lotus-chainwatch
-	install -C ./lotus-chainwatch /usr/local/bin/lotus-chainwatch
-
-lotus-sim: $(BUILD_DEPS)
-	rm -f lotus-sim
-	go build $(GOFLAGS) -o lotus-sim ./cmd/lotus-sim
-.PHONY: lotus-sim
-BINS+=lotus-sim
-
-# SYSTEMD
-
-install-daemon-service: install-daemon
-	mkdir -p /etc/systemd/system
-	mkdir -p /var/log/lotus
-	install -C -m 0644 ./scripts/lotus-daemon.service /etc/systemd/system/lotus-daemon.service
-	systemctl daemon-reload
-	@echo
-	@echo "lotus-daemon service installed. Don't forget to run 'sudo systemctl start lotus-daemon' to start it and 'sudo systemctl enable lotus-daemon' for it to be enabled on startup."
-
-install-miner-service: install-miner install-daemon-service
-	mkdir -p /etc/systemd/system
-	mkdir -p /var/log/lotus
-	install -C -m 0644 ./scripts/lotus-miner.service /etc/systemd/system/lotus-miner.service
-	systemctl daemon-reload
-	@echo
-	@echo "lotus-miner service installed. Don't forget to run 'sudo systemctl start lotus-miner' to start it and 'sudo systemctl enable lotus-miner' for it to be enabled on startup."
-
-install-chainwatch-service: install-chainwatch install-daemon-service
-	mkdir -p /etc/systemd/system
-	mkdir -p /var/log/lotus
-	install -C -m 0644 ./scripts/lotus-chainwatch.service /etc/systemd/system/lotus-chainwatch.service
-	systemctl daemon-reload
-	@echo
-	@echo "chainwatch service installed. Don't forget to run 'sudo systemctl start lotus-chainwatch' to start it and 'sudo systemctl enable lotus-chainwatch' for it to be enabled on startup."
-
-install-main-services: install-miner-service
-
-install-all-services: install-main-services install-chainwatch-service
-
-install-services: install-main-services
-
-clean-daemon-service: clean-miner-service clean-chainwatch-service
-	-systemctl stop lotus-daemon
-	-systemctl disable lotus-daemon
-	rm -f /etc/systemd/system/lotus-daemon.service
-	systemctl daemon-reload
-
-clean-miner-service:
-	-systemctl stop lotus-miner
-	-systemctl disable lotus-miner
-	rm -f /etc/systemd/system/lotus-miner.service
-	systemctl daemon-reload
-
-clean-chainwatch-service:
-	-systemctl stop lotus-chainwatch
-	-systemctl disable lotus-chainwatch
-	rm -f /etc/systemd/system/lotus-chainwatch.service
-	systemctl daemon-reload
-
-clean-main-services: clean-daemon-service
-
-clean-all-services: clean-main-services
-
-clean-services: clean-all-services
+.PHONY: health
+BINS+=health
 
 # MISC
 
 buildall: $(BINS)
 
+completions:
+	./scripts/make-completions.sh lotus
+	./scripts/make-completions.sh lotus-storage-miner
+.PHONY: completions
+
 install-completions:
 	mkdir -p /usr/share/bash-completion/completions /usr/local/share/zsh/site-functions/
 	install -C ./scripts/bash-completion/lotus /usr/share/bash-completion/completions/lotus
+	install -C ./scripts/bash-completion/lotus-storage-miner /usr/share/bash-completion/completions/lotus-storage-miner
 	install -C ./scripts/zsh-completion/lotus /usr/local/share/zsh/site-functions/_lotus
+	install -C ./scripts/zsh-completion/lotus-storage-miner /usr/local/share/zsh/site-functions/_lotus-storage-miner
 
 clean:
 	rm -rf $(CLEAN) $(BINS)
@@ -318,79 +208,13 @@ dist-clean:
 	git submodule deinit --all -f
 .PHONY: dist-clean
 
-type-gen: api-gen
+type-gen:
 	go run ./gen/main.go
-	go generate -x ./...
-	goimports -w api/
 
-method-gen: api-gen
+method-gen:
 	(cd ./lotuspond/front/src/chain && go run ./methodgen.go)
 
-actors-gen:
-	go run ./chain/actors/agen
-	go fmt ./...
-
-api-gen:
-	go run ./gen/api
-	goimports -w api
-	goimports -w api
-.PHONY: api-gen
-
-cfgdoc-gen:
-	go run ./node/config/cfgdocgen > ./node/config/doc_gen.go
-
-appimage: lotus
-	rm -rf appimage-builder-cache || true
-	rm AppDir/io.filecoin.lotus.desktop || true
-	rm AppDir/icon.svg || true
-	rm Appdir/AppRun || true
-	mkdir -p AppDir/usr/bin
-	cp ./lotus AppDir/usr/bin/
-	appimage-builder
-
-docsgen: docsgen-md docsgen-openrpc
-
-docsgen-md-bin: api-gen actors-gen
-	go build $(GOFLAGS) -o docgen-md ./api/docgen/cmd
-docsgen-openrpc-bin: api-gen actors-gen
-	go build $(GOFLAGS) -o docgen-openrpc ./api/docgen-openrpc/cmd
-
-docsgen-md: docsgen-md-full docsgen-md-storage docsgen-md-worker
-
-docsgen-md-full: docsgen-md-bin
-	./docgen-md "api/api_full.go" "FullNode" "api" "./api" > documentation/en/api-v1-unstable-methods.md
-	./docgen-md "api/v0api/full.go" "FullNode" "v0api" "./api/v0api" > documentation/en/api-v0-methods.md
-docsgen-md-storage: docsgen-md-bin
-	./docgen-md "api/api_storage.go" "StorageMiner" "api" "./api" > documentation/en/api-v0-methods-miner.md
-docsgen-md-worker: docsgen-md-bin
-	./docgen-md "api/api_worker.go" "Worker" "api" "./api" > documentation/en/api-v0-methods-worker.md
-
-docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker
-
-docsgen-openrpc-full: docsgen-openrpc-bin
-	./docgen-openrpc "api/api_full.go" "FullNode" "api" "./api" -gzip > build/openrpc/full.json.gz
-docsgen-openrpc-storage: docsgen-openrpc-bin
-	./docgen-openrpc "api/api_storage.go" "StorageMiner" "api" "./api" -gzip > build/openrpc/miner.json.gz
-docsgen-openrpc-worker: docsgen-openrpc-bin
-	./docgen-openrpc "api/api_worker.go" "Worker" "api" "./api" -gzip > build/openrpc/worker.json.gz
-
-.PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin
-
-gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci
-	@echo ">>> IF YOU'VE MODIFIED THE CLI, REMEMBER TO ALSO MAKE docsgen-cli"
-.PHONY: gen
-
-snap: lotus lotus-miner lotus-worker
-	snapcraft
-	# snapcraft upload ./lotus_*.snap
-
-# separate from gen because it needs binaries
-docsgen-cli: lotus lotus-miner lotus-worker
-	python ./scripts/generate-lotus-cli.py
-.PHONY: docsgen-cli
+gen: type-gen method-gen
 
 print-%:
 	@echo $*=$($*)
-
-circleci:
-	go generate -x ./.circleci

README.md

@@ -1,138 +1,16 @@
-<p align="center">
-  <a href="https://docs.filecoin.io/" title="Filecoin Docs">
-    <img src="documentation/images/lotus_logo_h.png" alt="Project Lotus Logo" width="244" />
-  </a>
-</p>
+![Lotus](documentation/images/lotus_logo_h.png)
 
-<h1 align="center">Project Lotus - 莲</h1>
+# Project Lotus - 莲
 
-<p align="center">
-  <a href="https://circleci.com/gh/filecoin-project/lotus"><img src="https://circleci.com/gh/filecoin-project/lotus.svg?style=svg"></a>
-  <a href="https://codecov.io/gh/filecoin-project/lotus"><img src="https://codecov.io/gh/filecoin-project/lotus/branch/master/graph/badge.svg"></a>
-  <a href="https://goreportcard.com/report/github.com/filecoin-project/lotus"><img src="https://goreportcard.com/badge/github.com/filecoin-project/lotus" /></a>  
-  <a href=""><img src="https://img.shields.io/badge/golang-%3E%3D1.16-blue.svg" /></a>
-  <br>
-</p>
+Lotus is an implementation of the Filecoin Distributed Storage Network. For more details about Filecoin, check out the [Filecoin Spec](https://github.com/filecoin-project/specs).
 
-Lotus is an implementation of the Filecoin Distributed Storage Network. For more details about Filecoin, check out the [Filecoin Spec](https://spec.filecoin.io).
+## Development
+
+All work is tracked via issues. An attempt at keeping an up-to-date view on remaining work is in the [lotus testnet github project board](https://github.com/filecoin-project/lotus/projects/1).
 
 ## Building & Documentation
 
-> Note: The default `master` branch is the dev branch, please use with caution. For the latest stable version, checkout the most recent [`Latest release`](https://github.com/filecoin-project/lotus/releases).
- 
-For complete instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/). Basic build instructions can be found further down in this readme.
-
-## Reporting a Vulnerability
-
-Please send an email to security@filecoin.org. See our [security policy](SECURITY.md) for more details.
-
-## Related packages
-
-These repos are independent and reusable modules, but are tightly integrated into Lotus to make up a fully featured Filecoin implementation:
-
-- [go-fil-markets](https://github.com/filecoin-project/go-fil-markets) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/markets-shared-components-5daa144a7046a60001c6e253/board)
-- [specs-actors](https://github.com/filecoin-project/specs-actors) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/actors-5ee6f3aa87591f0016c05685/board)
-
-## Contribute
-
-Lotus is a universally open project and welcomes contributions of all kinds: code, docs, and more. However, before making a contribution, we ask you to heed these recommendations:
-
-1. If the proposal entails a protocol change, please first submit a [Filecoin Improvement Proposal](https://github.com/filecoin-project/FIPs).
-2. If the change is complex and requires prior discussion, [open an issue](github.com/filecoin-project/lotus/issues) or a [discussion](https://github.com/filecoin-project/lotus/discussions) to request feedback before you start working on a pull request. This is to avoid disappointment and sunk costs, in case the change is not actually needed or accepted.
-3. Please refrain from submitting PRs to adapt existing code to subjective preferences. The changeset should contain functional or technical improvements/enhancements, bug fixes, new features, or some other clear material contribution. Simple stylistic changes are likely to be rejected in order to reduce code churn.
-
-When implementing a change:
-
-1. Adhere to the standard Go formatting guidelines, e.g. [Effective Go](https://golang.org/doc/effective_go.html). Run `go fmt`.
-2. Stick to the idioms and patterns used in the codebase. Familiar-looking code has a higher chance of being accepted than eerie code. Pay attention to commonly used variable and parameter names, avoidance of naked returns, error handling patterns, etc.
-3. Comments: follow the advice on the [Commentary](https://golang.org/doc/effective_go.html#commentary) section of Effective Go.
-4. Minimize code churn. Modify only what is strictly necessary. Well-encapsulated changesets will get a quicker response from maintainers.
-5. Lint your code with [`golangci-lint`](https://golangci-lint.run) (CI will reject your PR if unlinted).
-6. Add tests.
-7. Title the PR in a meaningful way and describe the rationale and the thought process in the PR description.
-8. Write clean, thoughtful, and detailed [commit messages](https://chris.beams.io/posts/git-commit/). This is even more important than the PR description, because commit messages are stored _inside_ the Git history. One good rule is: if you are happy posting the commit message as the PR description, then it's a good commit message.
-
-## Basic Build Instructions
-**System-specific Software Dependencies**:
-
-Building Lotus requires some system dependencies, usually provided by your distribution.
-
-Ubuntu/Debian:
-```
-sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl clang build-essential hwloc libhwloc-dev wget -y && sudo apt upgrade -y
-```
-
-Fedora:
-```
-sudo dnf -y install gcc make git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm wget hwloc libhwloc-dev
-```
-
-For other distributions you can find the required dependencies [here.](https://docs.filecoin.io/get-started/lotus/installation/#system-specific) For instructions specific to macOS, you can find them [here.](https://docs.filecoin.io/get-started/lotus/installation/#macos)
-
-#### Go
-
-To build Lotus, you need a working installation of [Go 1.16.4 or higher](https://golang.org/dl/):
-
-```bash
-wget -c https://golang.org/dl/go1.16.4.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
-```
-
-**TIP:**
-You'll need to add `/usr/local/go/bin` to your path. For most Linux distributions you can run something like:
-
-```shell
-echo "export PATH=$PATH:/usr/local/go/bin" >> ~/.bashrc && source ~/.bashrc
-```
-
-See the [official Golang installation instructions](https://golang.org/doc/install) if you get stuck.
-
-### Build and install Lotus
-
-Once all the dependencies are installed, you can build and install the Lotus suite (`lotus`, `lotus-miner`, and `lotus-worker`).
-
-1. Clone the repository:
-
-   ```sh
-   git clone https://github.com/filecoin-project/lotus.git
-   cd lotus/
-   ```
-   
-Note: The default branch `master` is the dev branch where the latest new features, bug fixes and improvement are in. However, if you want to run lotus on Filecoin mainnet and want to run a production-ready lotus, get the latest release[ here](https://github.com/filecoin-project/lotus/releases).
-
-2. To join mainnet, checkout the [latest release](https://github.com/filecoin-project/lotus/releases).
-
-   If you are changing networks from a previous Lotus installation or there has been a network reset, read the [Switch networks guide](https://docs.filecoin.io/get-started/lotus/switch-networks/) before proceeding.
-
-   For networks other than mainnet, look up the current branch or tag/commit for the network you want to join in the [Filecoin networks dashboard](https://network.filecoin.io), then build Lotus for your specific network below.
-
-   ```sh
-   git checkout <tag_or_branch>
-   # For example:
-   git checkout <vX.X.X> # tag for a release
-   ```
-
-   Currently, the latest code on the _master_ branch corresponds to mainnet.
-
-3. If you are in China, see "[Lotus: tips when running in China](https://docs.filecoin.io/get-started/lotus/tips-running-in-china/)".
-4. This build instruction uses the prebuilt proofs binaries. If you want to build the proof binaries from source check the [complete instructions](https://docs.filecoin.io/get-started/lotus/installation/#build-and-install-lotus). Note, if you are building the proof binaries from source, [installing rustup](https://docs.filecoin.io/get-started/lotus/installation/#rustup) is also needed.
-
-5. Build and install Lotus:
-
-   ```sh
-   make clean all #mainnet
-
-   # Or to join a testnet or devnet:
-   make clean calibnet # Calibration with min 32GiB sectors
-   make clean nerpanet # Nerpa with min 512MiB sectors
-
-   sudo make install
-   ```
-
-   This will put `lotus`, `lotus-miner` and `lotus-worker` in `/usr/local/bin`.
-
-   `lotus` will use the `$HOME/.lotus` folder by default for storage (configuration, chain data, wallets, etc). See [advanced options](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/) for information on how to customize the Lotus folder.
-
-6. You should now have Lotus installed. You can now [start the Lotus daemon and sync the chain](https://docs.filecoin.io/get-started/lotus/installation/#start-the-lotus-daemon-and-sync-the-chain).
+For instructions on how to build lotus from source, please visit [https://docs.lotu.sh](https://docs.lotu.sh) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation).
 
 ## License
 

SECURITY.md

@@ -1,23 +0,0 @@
-# Security Policy
-
-## Reporting a Vulnerability
-
-For reporting security vulnerabilities/bugs, please consult our Security Policy and Responsible Disclosure Program information at https://github.com/filecoin-project/community/blob/master/SECURITY.md. Security vulnerabilities should be reported via our [Vulnerability Reporting channels](https://github.com/filecoin-project/community/blob/master/SECURITY.md#vulnerability-reporting) and will be eligible for a [Bug Bounty](https://security.filecoin.io/bug-bounty/).
-
-Please try to provide a clear description of any bugs reported, along with how to reproduce the bug if possible. More detailed bug reports (especially those with a PoC included) will help us move forward much faster. Additionally, please avoid reporting bugs that already have open issues. Take a moment to search the issue list of the related GitHub repositories before writing up a new report.
-
-Here are some examples of bugs we would consider to be security vulnerabilities:
-
-* If you can spend from a `multisig` wallet you do not control the keys for.
-* If you can cause a miner to be slashed without them actually misbehaving.
-* If you can maintain power without submitting windowed posts regularly.
-* If you can craft a message that causes lotus nodes to panic.
-* If you can cause your miner to win significantly more blocks than it should.
-* If you can craft a message that causes a persistent fork in the network.
-* If you can cause the total amount of Filecoin in the network to no longer be 2 billion.
-
-This is not an exhaustive list, but should provide some idea of what we consider as a security vulnerability, . 
-
-## Reporting a non security bug
-
-For non-security bugs, please simply file a GitHub [issue](https://github.com/filecoin-project/lotus/issues/new?template=bug_report.md). 

api/README.md

@@ -1,14 +0,0 @@
-## Lotus API
-
-This package contains all lotus API definitions. Interfaces defined here are
-exposed as JsonRPC 2.0 endpoints by lotus programs.
-
-### Versions
-
-| File             | Alias File        | Interface      | Exposed by         | Version | HTTP Endpoint | Status                       | Docs
-|------------------|-------------------|----------------|--------------------|---------|---------------|------------------------------|------
-| `api_common.go`  | `v0api/latest.go` | `Common`       | lotus; lotus-miner | v0      | `/rpc/v0`     | Latest, Stable               | [Methods](../documentation/en/api-v0-methods.md)
-| `api_full.go`    | `v1api/latest.go` | `FullNode`     | lotus              | v1      | `/rpc/v1`     | Latest, **Work in progress** | [Methods](../documentation/en/api-v1-unstable-methods.md)
-| `api_storage.go` | `v0api/latest.go` | `StorageMiner` | lotus-miner        | v0      | `/rpc/v0`     | Latest, Stable               | [Methods](../documentation/en/api-v0-methods-miner.md)
-| `api_worker.go`  | `v0api/latest.go` | `Worker`       | lotus-worker       | v0      | `/rpc/v0`     | Latest, Stable               | [Methods](../documentation/en/api-v0-methods-worker.md)
-| `v0api/full.go`  |                   | `FullNode`     | lotus              | v0      | `/rpc/v0`     | Stable                       | [Methods](../documentation/en/api-v0-methods.md)

api/api_common.go

@@ -4,61 +4,51 @@ import (
 	"context"
 	"fmt"
 
-	apitypes "github.com/filecoin-project/lotus/api/types"
-
-	"github.com/google/uuid"
+	"github.com/libp2p/go-libp2p-core/network"
+	"github.com/libp2p/go-libp2p-core/peer"
 
 	"github.com/filecoin-project/go-jsonrpc/auth"
+
+	"github.com/filecoin-project/lotus/build"
 )
 
-//                       MODIFYING THE API INTERFACE
-//
-// When adding / changing methods in this file:
-// * Do the change here
-// * Adjust implementation in `node/impl/`
-// * Run `make gen` - this will:
-//  * Generate proxy structs
-//  * Generate mocks
-//  * Generate markdown docs
-//  * Generate openrpc blobs
-
 type Common interface {
-	// MethodGroup: Auth
+	// Auth
+	AuthVerify(ctx context.Context, token string) ([]auth.Permission, error)
+	AuthNew(ctx context.Context, perms []auth.Permission) ([]byte, error)
 
-	AuthVerify(ctx context.Context, token string) ([]auth.Permission, error) //perm:read
-	AuthNew(ctx context.Context, perms []auth.Permission) ([]byte, error)    //perm:admin
+	// network
 
-	// MethodGroup: Log
+	NetConnectedness(context.Context, peer.ID) (network.Connectedness, error)
+	NetPeers(context.Context) ([]peer.AddrInfo, error)
+	NetConnect(context.Context, peer.AddrInfo) error
+	NetAddrsListen(context.Context) (peer.AddrInfo, error)
+	NetDisconnect(context.Context, peer.ID) error
+	NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error)
+	NetPubsubScores(context.Context) ([]PubsubScore, error)
 
-	LogList(context.Context) ([]string, error)         //perm:write
-	LogSetLevel(context.Context, string, string) error //perm:write
-
-	// MethodGroup: Common
+	// ID returns peerID of libp2p node backing this API
+	ID(context.Context) (peer.ID, error)
 
 	// Version provides information about API provider
-	Version(context.Context) (APIVersion, error) //perm:read
+	Version(context.Context) (Version, error)
 
-	// Discover returns an OpenRPC document describing an RPC API.
-	Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) //perm:read
+	LogList(context.Context) ([]string, error)
+	LogSetLevel(context.Context, string, string) error
 
 	// trigger graceful shutdown
-	Shutdown(context.Context) error //perm:admin
-
-	// Session returns a random UUID of api provider session
-	Session(context.Context) (uuid.UUID, error) //perm:read
-
-	Closing(context.Context) (<-chan struct{}, error) //perm:read
+	Shutdown(context.Context) error
 }
 
-// APIVersion provides various build-time information
-type APIVersion struct {
+// Version provides various build-time information
+type Version struct {
 	Version string
 
 	// APIVersion is a binary encoded semver version of the remote implementing
 	// this api
 	//
 	// See APIVersion in build/version.go
-	APIVersion Version
+	APIVersion build.Version
 
 	// TODO: git commit / os / genesis cid?
 
@@ -66,6 +56,6 @@ type APIVersion struct {
 	BlockDelay uint64
 }
 
-func (v APIVersion) String() string {
+func (v Version) String() string {
 	return fmt.Sprintf("%s+api%s", v.Version, v.APIVersion.String())
 }

api/api_gateway.go

@@ -1,63 +0,0 @@
-package api
-
-import (
-	"context"
-
-	"github.com/ipfs/go-cid"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/dline"
-
-	apitypes "github.com/filecoin-project/lotus/api/types"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-//                       MODIFYING THE API INTERFACE
-//
-// NOTE: This is the V1 (Unstable) API - to add methods to the V0 (Stable) API
-// you'll have to add those methods to interfaces in `api/v0api`
-//
-// When adding / changing methods in this file:
-// * Do the change here
-// * Adjust implementation in `node/impl/`
-// * Run `make gen` - this will:
-//  * Generate proxy structs
-//  * Generate mocks
-//  * Generate markdown docs
-//  * Generate openrpc blobs
-
-type Gateway interface {
-	ChainHasObj(context.Context, cid.Cid) (bool, error)
-	ChainHead(ctx context.Context) (*types.TipSet, error)
-	ChainGetBlockMessages(context.Context, cid.Cid) (*BlockMessages, error)
-	ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
-	ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error)
-	ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error)
-	ChainNotify(context.Context) (<-chan []*HeadChange, error)
-	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
-	GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
-	MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
-	MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
-	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
-	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
-	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
-	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
-	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
-	StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error) //perm:read
-	StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
-	StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
-	StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MarketBalance, error)
-	StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*MarketDeal, error)
-	StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error)
-	StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
-	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error)
-	StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error)
-	StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
-	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
-	StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
-	StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
-	WalletBalance(context.Context, address.Address) (types.BigInt, error)
-	Version(context.Context) (APIVersion, error)
-}

api/api_net.go

@@ -1,66 +0,0 @@
-package api
-
-import (
-	"context"
-
-	metrics "github.com/libp2p/go-libp2p-core/metrics"
-	"github.com/libp2p/go-libp2p-core/network"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"github.com/libp2p/go-libp2p-core/protocol"
-)
-
-//                       MODIFYING THE API INTERFACE
-//
-// When adding / changing methods in this file:
-// * Do the change here
-// * Adjust implementation in `node/impl/`
-// * Run `make gen` - this will:
-//  * Generate proxy structs
-//  * Generate mocks
-//  * Generate markdown docs
-//  * Generate openrpc blobs
-
-type Net interface {
-	// MethodGroup: Net
-
-	NetConnectedness(context.Context, peer.ID) (network.Connectedness, error) //perm:read
-	NetPeers(context.Context) ([]peer.AddrInfo, error)                        //perm:read
-	NetConnect(context.Context, peer.AddrInfo) error                          //perm:write
-	NetAddrsListen(context.Context) (peer.AddrInfo, error)                    //perm:read
-	NetDisconnect(context.Context, peer.ID) error                             //perm:write
-	NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error)              //perm:read
-	NetPubsubScores(context.Context) ([]PubsubScore, error)                   //perm:read
-	NetAutoNatStatus(context.Context) (NatInfo, error)                        //perm:read
-	NetAgentVersion(ctx context.Context, p peer.ID) (string, error)           //perm:read
-	NetPeerInfo(context.Context, peer.ID) (*ExtendedPeerInfo, error)          //perm:read
-
-	// NetBandwidthStats returns statistics about the nodes total bandwidth
-	// usage and current rate across all peers and protocols.
-	NetBandwidthStats(ctx context.Context) (metrics.Stats, error) //perm:read
-
-	// NetBandwidthStatsByPeer returns statistics about the nodes bandwidth
-	// usage and current rate per peer
-	NetBandwidthStatsByPeer(ctx context.Context) (map[string]metrics.Stats, error) //perm:read
-
-	// NetBandwidthStatsByProtocol returns statistics about the nodes bandwidth
-	// usage and current rate per protocol
-	NetBandwidthStatsByProtocol(ctx context.Context) (map[protocol.ID]metrics.Stats, error) //perm:read
-
-	// ConnectionGater API
-	NetBlockAdd(ctx context.Context, acl NetBlockList) error    //perm:admin
-	NetBlockRemove(ctx context.Context, acl NetBlockList) error //perm:admin
-	NetBlockList(ctx context.Context) (NetBlockList, error)     //perm:read
-
-	// ID returns peerID of libp2p node backing this API
-	ID(context.Context) (peer.ID, error) //perm:read
-}
-
-type CommonNet interface {
-	Common
-	Net
-}
-
-type NatInfo struct {
-	Reachability network.Reachability
-	PublicAddr   string
-}

api/api_storage.go

@@ -3,211 +3,61 @@ package api
 import (
 	"bytes"
 	"context"
-	"time"
 
-	"github.com/filecoin-project/lotus/chain/actors/builtin"
-
-	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
-	"github.com/libp2p/go-libp2p-core/peer"
 
 	"github.com/filecoin-project/go-address"
-	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-fil-markets/piecestore"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
-	"github.com/filecoin-project/specs-storage/storage"
-
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-	"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
+	"github.com/filecoin-project/sector-storage/stores"
+	"github.com/filecoin-project/sector-storage/storiface"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 )
 
-//                       MODIFYING THE API INTERFACE
-//
-// When adding / changing methods in this file:
-// * Do the change here
-// * Adjust implementation in `node/impl/`
-// * Run `make gen` - this will:
-//  * Generate proxy structs
-//  * Generate mocks
-//  * Generate markdown docs
-//  * Generate openrpc blobs
-
 // StorageMiner is a low-level interface to the Filecoin network storage miner node
 type StorageMiner interface {
 	Common
-	Net
 
-	ActorAddress(context.Context) (address.Address, error) //perm:read
+	ActorAddress(context.Context) (address.Address, error)
 
-	ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error) //perm:read
-	ActorAddressConfig(ctx context.Context) (AddressConfig, error)            //perm:read
+	ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error)
 
-	MiningBase(context.Context) (*types.TipSet, error) //perm:read
+	MiningBase(context.Context) (*types.TipSet, error)
 
 	// Temp api for testing
-	PledgeSector(context.Context) (abi.SectorID, error) //perm:write
+	PledgeSector(context.Context) error
 
 	// Get the status of a given sector by ID
-	SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (SectorInfo, error) //perm:read
-
-	// Add piece to an open sector. If no sectors with enough space are open,
-	// either a new sector will be created, or this call will block until more
-	// sectors can be created.
-	SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storage.Data, d PieceDealInfo) (SectorOffset, error) //perm:admin
-
-	SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error //perm:admin
+	SectorsStatus(context.Context, abi.SectorNumber) (SectorInfo, error)
 
 	// List all staged sectors
-	SectorsList(context.Context) ([]abi.SectorNumber, error) //perm:read
+	SectorsList(context.Context) ([]abi.SectorNumber, error)
 
-	// Get summary info of sectors
-	SectorsSummary(ctx context.Context) (map[SectorState]int, error) //perm:read
+	SectorsRefs(context.Context) (map[string][]SealedRef, error)
 
-	// List sectors in particular states
-	SectorsListInStates(context.Context, []SectorState) ([]abi.SectorNumber, error) //perm:read
+	SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error
 
-	SectorsRefs(context.Context) (map[string][]SealedRef, error) //perm:read
-
-	// SectorStartSealing can be called on sectors in Empty or WaitDeals states
-	// to trigger sealing early
-	SectorStartSealing(context.Context, abi.SectorNumber) error //perm:write
-	// SectorSetSealDelay sets the time that a newly-created sector
-	// waits for more deals before it starts sealing
-	SectorSetSealDelay(context.Context, time.Duration) error //perm:write
-	// SectorGetSealDelay gets the time that a newly-created sector
-	// waits for more deals before it starts sealing
-	SectorGetSealDelay(context.Context) (time.Duration, error) //perm:read
-	// SectorSetExpectedSealDuration sets the expected time for a sector to seal
-	SectorSetExpectedSealDuration(context.Context, time.Duration) error //perm:write
-	// SectorGetExpectedSealDuration gets the expected time for a sector to seal
-	SectorGetExpectedSealDuration(context.Context) (time.Duration, error) //perm:read
-	SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error   //perm:admin
-	// SectorRemove removes the sector from storage. It doesn't terminate it on-chain, which can
-	// be done with SectorTerminate. Removing and not terminating live sectors will cause additional penalties.
-	SectorRemove(context.Context, abi.SectorNumber) error //perm:admin
-	// SectorTerminate terminates the sector on-chain (adding it to a termination batch first), then
-	// automatically removes it from storage
-	SectorTerminate(context.Context, abi.SectorNumber) error //perm:admin
-	// SectorTerminateFlush immediately sends a terminate message with sectors batched for termination.
-	// Returns null if message wasn't sent
-	SectorTerminateFlush(ctx context.Context) (*cid.Cid, error) //perm:admin
-	// SectorTerminatePending returns a list of pending sector terminations to be sent in the next batch message
-	SectorTerminatePending(ctx context.Context) ([]abi.SectorID, error)  //perm:admin
-	SectorMarkForUpgrade(ctx context.Context, id abi.SectorNumber) error //perm:admin
-	// SectorPreCommitFlush immediately sends a PreCommit message with sectors batched for PreCommit.
-	// Returns null if message wasn't sent
-	SectorPreCommitFlush(ctx context.Context) ([]sealiface.PreCommitBatchRes, error) //perm:admin
-	// SectorPreCommitPending returns a list of pending PreCommit sectors to be sent in the next batch message
-	SectorPreCommitPending(ctx context.Context) ([]abi.SectorID, error) //perm:admin
-	// SectorCommitFlush immediately sends a Commit message with sectors aggregated for Commit.
-	// Returns null if message wasn't sent
-	SectorCommitFlush(ctx context.Context) ([]sealiface.CommitBatchRes, error) //perm:admin
-	// SectorCommitPending returns a list of pending Commit sectors to be sent in the next aggregate message
-	SectorCommitPending(ctx context.Context) ([]abi.SectorID, error) //perm:admin
+	StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error)
+	StorageLocal(ctx context.Context) (map[stores.ID]string, error)
+	StorageStat(ctx context.Context, id stores.ID) (stores.FsStat, error)
 
 	// WorkerConnect tells the node to connect to workers RPC
-	WorkerConnect(context.Context, string) error                              //perm:admin retry:true
-	WorkerStats(context.Context) (map[uuid.UUID]storiface.WorkerStats, error) //perm:admin
-	WorkerJobs(context.Context) (map[uuid.UUID][]storiface.WorkerJob, error)  //perm:admin
+	WorkerConnect(context.Context, string) error
+	WorkerStats(context.Context) (map[uint64]storiface.WorkerStats, error)
 
-	//storiface.WorkerReturn
-	ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error                //perm:admin retry:true
-	ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storage.PreCommit1Out, err *storiface.CallError) error //perm:admin retry:true
-	ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storage.SectorCids, err *storiface.CallError) error //perm:admin retry:true
-	ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storage.Commit1Out, err *storiface.CallError) error       //perm:admin retry:true
-	ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storage.Proof, err *storiface.CallError) error          //perm:admin retry:true
-	ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                            //perm:admin retry:true
-	ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                           //perm:admin retry:true
-	ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                               //perm:admin retry:true
-	ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                               //perm:admin retry:true
-	ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err *storiface.CallError) error                        //perm:admin retry:true
-	ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                     //perm:admin retry:true
+	stores.SectorIndex
 
-	// SealingSchedDiag dumps internal sealing scheduler state
-	SealingSchedDiag(ctx context.Context, doSched bool) (interface{}, error) //perm:admin
-	SealingAbort(ctx context.Context, call storiface.CallID) error           //perm:admin
+	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error
+	MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error)
+	MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error)
+	MarketSetPrice(context.Context, types.BigInt) error
 
-	//stores.SectorIndex
-	StorageAttach(context.Context, stores.StorageInfo, fsutil.FsStat) error                                                                                             //perm:admin
-	StorageInfo(context.Context, stores.ID) (stores.StorageInfo, error)                                                                                                 //perm:admin
-	StorageReportHealth(context.Context, stores.ID, stores.HealthReport) error                                                                                          //perm:admin
-	StorageDeclareSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error                                     //perm:admin
-	StorageDropSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType) error                                                      //perm:admin
-	StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]stores.SectorStorageInfo, error) //perm:admin
-	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]stores.StorageInfo, error)           //perm:admin
-	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                                          //perm:admin
-	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                               //perm:admin
-	StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error)                                                                                               //perm:admin
+	DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error
+	DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error)
 
-	StorageLocal(ctx context.Context) (map[stores.ID]string, error)       //perm:admin
-	StorageStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) //perm:admin
-
-	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error                                                                                                        //perm:write
-	MarketListDeals(ctx context.Context) ([]MarketDeal, error)                                                                                                                           //perm:read
-	MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error)                                                                                           //perm:read
-	MarketGetDealUpdates(ctx context.Context) (<-chan storagemarket.MinerDeal, error)                                                                                                    //perm:read
-	MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error)                                                                                                    //perm:read
-	MarketSetAsk(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error //perm:admin
-	MarketGetAsk(ctx context.Context) (*storagemarket.SignedStorageAsk, error)                                                                                                           //perm:read
-	MarketSetRetrievalAsk(ctx context.Context, rask *retrievalmarket.Ask) error                                                                                                          //perm:admin
-	MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error)                                                                                                             //perm:read
-	MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)                                                                                                          //perm:write
-	MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)                                                                                                   //perm:write
-	// MarketRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer
-	MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
-	// MarketCancelDataTransfer cancels a data transfer with the given transfer ID and other peer
-	MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
-	MarketPendingDeals(ctx context.Context) (PendingDealInfo, error)                                                             //perm:write
-	MarketPublishPendingDeals(ctx context.Context) error                                                                         //perm:admin
-
-	// RuntimeSubsystems returns the subsystems that are enabled
-	// in this instance.
-	RuntimeSubsystems(ctx context.Context) (MinerSubsystems, error) //perm:read
-
-	DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error //perm:admin
-	DealsList(ctx context.Context) ([]MarketDeal, error)                         //perm:admin
-	DealsConsiderOnlineStorageDeals(context.Context) (bool, error)               //perm:admin
-	DealsSetConsiderOnlineStorageDeals(context.Context, bool) error              //perm:admin
-	DealsConsiderOnlineRetrievalDeals(context.Context) (bool, error)             //perm:admin
-	DealsSetConsiderOnlineRetrievalDeals(context.Context, bool) error            //perm:admin
-	DealsPieceCidBlocklist(context.Context) ([]cid.Cid, error)                   //perm:admin
-	DealsSetPieceCidBlocklist(context.Context, []cid.Cid) error                  //perm:admin
-	DealsConsiderOfflineStorageDeals(context.Context) (bool, error)              //perm:admin
-	DealsSetConsiderOfflineStorageDeals(context.Context, bool) error             //perm:admin
-	DealsConsiderOfflineRetrievalDeals(context.Context) (bool, error)            //perm:admin
-	DealsSetConsiderOfflineRetrievalDeals(context.Context, bool) error           //perm:admin
-	DealsConsiderVerifiedStorageDeals(context.Context) (bool, error)             //perm:admin
-	DealsSetConsiderVerifiedStorageDeals(context.Context, bool) error            //perm:admin
-	DealsConsiderUnverifiedStorageDeals(context.Context) (bool, error)           //perm:admin
-	DealsSetConsiderUnverifiedStorageDeals(context.Context, bool) error          //perm:admin
-
-	StorageAddLocal(ctx context.Context, path string) error //perm:admin
-
-	PiecesListPieces(ctx context.Context) ([]cid.Cid, error)                                 //perm:read
-	PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error)                               //perm:read
-	PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) //perm:read
-	PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error)   //perm:read
-
-	// CreateBackup creates node backup onder the specified file name. The
-	// method requires that the lotus-miner is running with the
-	// LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that
-	// the path specified when calling CreateBackup is within the base path
-	CreateBackup(ctx context.Context, fpath string) error //perm:admin
-
-	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
-
-	ComputeProof(ctx context.Context, ssi []builtin.SectorInfo, rand abi.PoStRandomness) ([]builtin.PoStProof, error) //perm:read
+	StorageAddLocal(ctx context.Context, path string) error
 }
 
-var _ storiface.WorkerReturn = *new(StorageMiner)
-var _ stores.SectorIndex = *new(StorageMiner)
-
 type SealRes struct {
 	Err   string
 	GoErr error `json:"-"`
@@ -225,40 +75,24 @@ type SectorLog struct {
 }
 
 type SectorInfo struct {
-	SectorID     abi.SectorNumber
-	State        SectorState
-	CommD        *cid.Cid
-	CommR        *cid.Cid
-	Proof        []byte
-	Deals        []abi.DealID
-	Ticket       SealTicket
-	Seed         SealSeed
-	PreCommitMsg *cid.Cid
-	CommitMsg    *cid.Cid
-	Retries      uint64
-	ToUpgrade    bool
+	SectorID abi.SectorNumber
+	State    SectorState
+	CommD    *cid.Cid
+	CommR    *cid.Cid
+	Proof    []byte
+	Deals    []abi.DealID
+	Ticket   SealTicket
+	Seed     SealSeed
+	Retries  uint64
 
 	LastErr string
 
 	Log []SectorLog
-
-	// On Chain Info
-	SealProof          abi.RegisteredSealProof // The seal proof type implies the PoSt proof/s
-	Activation         abi.ChainEpoch          // Epoch during which the sector proof was accepted
-	Expiration         abi.ChainEpoch          // Epoch during which the sector expires
-	DealWeight         abi.DealWeight          // Integral of active deals over sector lifetime
-	VerifiedDealWeight abi.DealWeight          // Integral of active verified deals over sector lifetime
-	InitialPledge      abi.TokenAmount         // Pledge collected to commit this sector
-	// Expiration Info
-	OnTime abi.ChainEpoch
-	// non-zero if sector is faulty, epoch at which it will be permanently
-	// removed if it doesn't recover
-	Early abi.ChainEpoch
 }
 
 type SealedRef struct {
 	SectorID abi.SectorNumber
-	Offset   abi.PaddedPieceSize
+	Offset   uint64
 	Size     abi.UnpaddedPieceSize
 }
 
@@ -285,54 +119,3 @@ func (st *SealSeed) Equals(ost *SealSeed) bool {
 }
 
 type SectorState string
-
-type AddrUse int
-
-const (
-	PreCommitAddr AddrUse = iota
-	CommitAddr
-	DealPublishAddr
-	PoStAddr
-
-	TerminateSectorsAddr
-)
-
-type AddressConfig struct {
-	PreCommitControl   []address.Address
-	CommitControl      []address.Address
-	TerminateControl   []address.Address
-	DealPublishControl []address.Address
-
-	DisableOwnerFallback  bool
-	DisableWorkerFallback bool
-}
-
-// PendingDealInfo has info about pending deals and when they are due to be
-// published
-type PendingDealInfo struct {
-	Deals              []market.ClientDealProposal
-	PublishPeriodStart time.Time
-	PublishPeriod      time.Duration
-}
-
-type SectorOffset struct {
-	Sector abi.SectorNumber
-	Offset abi.PaddedPieceSize
-}
-
-// DealInfo is a tuple of deal identity and its schedule
-type PieceDealInfo struct {
-	PublishCid   *cid.Cid
-	DealID       abi.DealID
-	DealProposal *market.DealProposal
-	DealSchedule DealSchedule
-	KeepUnsealed bool
-}
-
-// DealSchedule communicates the time interval of a storage deal. The deal must
-// appear in a sealed (proven) sector no later than StartEpoch, otherwise it
-// is invalid.
-type DealSchedule struct {
-	StartEpoch abi.ChainEpoch
-	EndEpoch   abi.ChainEpoch
-}

api/api_test.go

@@ -1,16 +1,12 @@
 package api
 
 import (
-	"encoding/json"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"reflect"
 	"runtime"
 	"strings"
 	"testing"
-
-	"github.com/stretchr/testify/require"
 )
 
 func goCmd() string {
@@ -36,86 +32,3 @@ func TestDoesntDependOnFFI(t *testing.T) {
 		}
 	}
 }
-
-func TestDoesntDependOnBuild(t *testing.T) {
-	deps, err := exec.Command(goCmd(), "list", "-deps", "github.com/filecoin-project/lotus/api").Output()
-	if err != nil {
-		t.Fatal(err)
-	}
-	for _, pkg := range strings.Fields(string(deps)) {
-		if pkg == "github.com/filecoin-project/build" {
-			t.Fatal("api depends on filecoin-ffi")
-		}
-	}
-}
-
-func TestReturnTypes(t *testing.T) {
-	errType := reflect.TypeOf(new(error)).Elem()
-	bareIface := reflect.TypeOf(new(interface{})).Elem()
-	jmarsh := reflect.TypeOf(new(json.Marshaler)).Elem()
-
-	tst := func(api interface{}) func(t *testing.T) {
-		return func(t *testing.T) {
-			ra := reflect.TypeOf(api).Elem()
-			for i := 0; i < ra.NumMethod(); i++ {
-				m := ra.Method(i)
-				switch m.Type.NumOut() {
-				case 1: // if 1 return value, it must be an error
-					require.Equal(t, errType, m.Type.Out(0), m.Name)
-
-				case 2: // if 2 return values, first cant be an interface/function, second must be an error
-					seen := map[reflect.Type]struct{}{}
-					todo := []reflect.Type{m.Type.Out(0)}
-					for len(todo) > 0 {
-						typ := todo[len(todo)-1]
-						todo = todo[:len(todo)-1]
-
-						if _, ok := seen[typ]; ok {
-							continue
-						}
-						seen[typ] = struct{}{}
-
-						if typ.Kind() == reflect.Interface && typ != bareIface && !typ.Implements(jmarsh) {
-							t.Error("methods can't return interfaces", m.Name)
-						}
-
-						switch typ.Kind() {
-						case reflect.Ptr:
-							fallthrough
-						case reflect.Array:
-							fallthrough
-						case reflect.Slice:
-							fallthrough
-						case reflect.Chan:
-							todo = append(todo, typ.Elem())
-						case reflect.Map:
-							todo = append(todo, typ.Elem())
-							todo = append(todo, typ.Key())
-						case reflect.Struct:
-							for i := 0; i < typ.NumField(); i++ {
-								todo = append(todo, typ.Field(i).Type)
-							}
-						}
-					}
-
-					require.NotEqual(t, reflect.Func.String(), m.Type.Out(0).Kind().String(), m.Name)
-					require.Equal(t, errType, m.Type.Out(1), m.Name)
-
-				default:
-					t.Error("methods can only have 1 or 2 return values", m.Name)
-				}
-			}
-		}
-	}
-
-	t.Run("common", tst(new(Common)))
-	t.Run("full", tst(new(FullNode)))
-	t.Run("miner", tst(new(StorageMiner)))
-	t.Run("worker", tst(new(Worker)))
-}
-
-func TestPermTags(t *testing.T) {
-	_ = PermissionedFullAPI(&FullNodeStruct{})
-	_ = PermissionedStorMinerAPI(&StorageMinerStruct{})
-	_ = PermissionedWorkerAPI(&WorkerStruct{})
-}

api/api_wallet.go

@@ -1,47 +0,0 @@
-package api
-
-import (
-	"context"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/crypto"
-
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-type MsgType string
-
-const (
-	MTUnknown = "unknown"
-
-	// Signing message CID. MsgMeta.Extra contains raw cbor message bytes
-	MTChainMsg = "message"
-
-	// Signing a blockheader. signing raw cbor block bytes (MsgMeta.Extra is empty)
-	MTBlock = "block"
-
-	// Signing a deal proposal. signing raw cbor proposal bytes (MsgMeta.Extra is empty)
-	MTDealProposal = "dealproposal"
-
-	// TODO: Deals, Vouchers, VRF
-)
-
-type MsgMeta struct {
-	Type MsgType
-
-	// Additional data related to what is signed. Should be verifiable with the
-	// signed bytes (e.g. CID(Extra).Bytes() == toSign)
-	Extra []byte
-}
-
-type Wallet interface {
-	WalletNew(context.Context, types.KeyType) (address.Address, error) //perm:admin
-	WalletHas(context.Context, address.Address) (bool, error)          //perm:admin
-	WalletList(context.Context) ([]address.Address, error)             //perm:admin
-
-	WalletSign(ctx context.Context, signer address.Address, toSign []byte, meta MsgMeta) (*crypto.Signature, error) //perm:admin
-
-	WalletExport(context.Context, address.Address) (*types.KeyInfo, error) //perm:admin
-	WalletImport(context.Context, *types.KeyInfo) (address.Address, error) //perm:admin
-	WalletDelete(context.Context, address.Address) error                   //perm:admin
-}

api/api_worker.go

@@ -2,71 +2,35 @@ package api
 
 import (
 	"context"
+	"io"
 
-	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
 
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	"github.com/filecoin-project/sector-storage/sealtasks"
+	"github.com/filecoin-project/sector-storage/stores"
+	"github.com/filecoin-project/sector-storage/storiface"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-storage/storage"
+
+	"github.com/filecoin-project/lotus/build"
 )
 
-//                       MODIFYING THE API INTERFACE
-//
-// When adding / changing methods in this file:
-// * Do the change here
-// * Adjust implementation in `node/impl/`
-// * Run `make gen` - this will:
-//  * Generate proxy structs
-//  * Generate mocks
-//  * Generate markdown docs
-//  * Generate openrpc blobs
+type WorkerAPI interface {
+	Version(context.Context) (build.Version, error)
+	// TODO: Info() (name, ...) ?
 
-type Worker interface {
-	Version(context.Context) (Version, error) //perm:admin
+	TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) // TaskType -> Weight
+	Paths(context.Context) ([]stores.StoragePath, error)
+	Info(context.Context) (storiface.WorkerInfo, error)
 
-	// TaskType -> Weight
-	TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) //perm:admin
-	Paths(context.Context) ([]stores.StoragePath, error)                //perm:admin
-	Info(context.Context) (storiface.WorkerInfo, error)                 //perm:admin
+	storage.Sealer
 
-	// storiface.WorkerCalls
-	AddPiece(ctx context.Context, sector storage.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (storiface.CallID, error)                    //perm:admin
-	SealPreCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error)                                                           //perm:admin
-	SealPreCommit2(ctx context.Context, sector storage.SectorRef, pc1o storage.PreCommit1Out) (storiface.CallID, error)                                                                                  //perm:admin
-	SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error) //perm:admin
-	SealCommit2(ctx context.Context, sector storage.SectorRef, c1o storage.Commit1Out) (storiface.CallID, error)                                                                                         //perm:admin
-	FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                                //perm:admin
-	ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) (storiface.CallID, error)                                                                                 //perm:admin
-	MoveStorage(ctx context.Context, sector storage.SectorRef, types storiface.SectorFileType) (storiface.CallID, error)                                                                                 //perm:admin
-	UnsealPiece(context.Context, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error)                                           //perm:admin
-	Fetch(context.Context, storage.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error)                                                             //perm:admin
+	MoveStorage(ctx context.Context, sector abi.SectorID) error
 
-	TaskDisable(ctx context.Context, tt sealtasks.TaskType) error //perm:admin
-	TaskEnable(ctx context.Context, tt sealtasks.TaskType) error  //perm:admin
+	UnsealPiece(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error
+	ReadPiece(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) error
 
-	// Storage / Other
-	Remove(ctx context.Context, sector abi.SectorID) error //perm:admin
+	Fetch(context.Context, abi.SectorID, stores.SectorFileType, stores.PathType, stores.AcquireMode) error
 
-	StorageAddLocal(ctx context.Context, path string) error //perm:admin
-
-	// SetEnabled marks the worker as enabled/disabled. Not that this setting
-	// may take a few seconds to propagate to task scheduler
-	SetEnabled(ctx context.Context, enabled bool) error //perm:admin
-
-	Enabled(ctx context.Context) (bool, error) //perm:admin
-
-	// WaitQuiet blocks until there are no tasks running
-	WaitQuiet(ctx context.Context) error //perm:admin
-
-	// returns a random UUID of worker session, generated randomly when worker
-	// process starts
-	ProcessSession(context.Context) (uuid.UUID, error) //perm:admin
-
-	// Like ProcessSession, but returns an error when worker is disabled
-	Session(context.Context) (uuid.UUID, error) //perm:admin
+	Closing(context.Context) (<-chan struct{}, error)
 }
-
-var _ storiface.WorkerCalls = *new(Worker)

api/apibstore/apibstore.go

@@ -0,0 +1,67 @@
+package apibstore
+
+import (
+	"context"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	"golang.org/x/xerrors"
+)
+
+type ChainIO interface {
+	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
+	ChainHasObj(context.Context, cid.Cid) (bool, error)
+}
+
+type apiBStore struct {
+	api ChainIO
+}
+
+func NewAPIBlockstore(cio ChainIO) blockstore.Blockstore {
+	return &apiBStore{
+		api: cio,
+	}
+}
+
+func (a *apiBStore) DeleteBlock(cid.Cid) error {
+	return xerrors.New("not supported")
+}
+
+func (a *apiBStore) Has(c cid.Cid) (bool, error) {
+	return a.api.ChainHasObj(context.TODO(), c)
+}
+
+func (a *apiBStore) Get(c cid.Cid) (blocks.Block, error) {
+	bb, err := a.api.ChainReadObj(context.TODO(), c)
+	if err != nil {
+		return nil, err
+	}
+	return blocks.NewBlockWithCid(bb, c)
+}
+
+func (a *apiBStore) GetSize(c cid.Cid) (int, error) {
+	bb, err := a.api.ChainReadObj(context.TODO(), c)
+	if err != nil {
+		return 0, err
+	}
+	return len(bb), nil
+}
+
+func (a *apiBStore) Put(blocks.Block) error {
+	return xerrors.New("not supported")
+}
+
+func (a *apiBStore) PutMany([]blocks.Block) error {
+	return xerrors.New("not supported")
+}
+
+func (a *apiBStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+	return nil, xerrors.New("not supported")
+}
+
+func (a *apiBStore) HashOnRead(enabled bool) {
+	return
+}
+
+var _ blockstore.Blockstore = &apiBStore{}

api/apistruct/permissioned.go

@@ -0,0 +1,38 @@
+package apistruct
+
+import (
+	"github.com/filecoin-project/go-jsonrpc/auth"
+	"github.com/filecoin-project/lotus/api"
+)
+
+const (
+	// When changing these, update docs/API.md too
+
+	PermRead  auth.Permission = "read" // default
+	PermWrite auth.Permission = "write"
+	PermSign  auth.Permission = "sign"  // Use wallet keys for signing
+	PermAdmin auth.Permission = "admin" // Manage permissions
+)
+
+var AllPermissions = []auth.Permission{PermRead, PermWrite, PermSign, PermAdmin}
+var DefaultPerms = []auth.Permission{PermRead}
+
+func PermissionedStorMinerAPI(a api.StorageMiner) api.StorageMiner {
+	var out StorageMinerStruct
+	auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
+	auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.CommonStruct.Internal)
+	return &out
+}
+
+func PermissionedFullAPI(a api.FullNode) api.FullNode {
+	var out FullNodeStruct
+	auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
+	auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.CommonStruct.Internal)
+	return &out
+}
+
+func PermissionedWorkerAPI(a api.WorkerAPI) api.WorkerAPI {
+	var out WorkerStruct
+	auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
+	return &out
+}

api/apistruct/struct.go

@@ -0,0 +1,915 @@
+package apistruct
+
+import (
+	"context"
+	"io"
+
+	"github.com/ipfs/go-cid"
+	"github.com/libp2p/go-libp2p-core/network"
+	"github.com/libp2p/go-libp2p-core/peer"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/filecoin-project/go-jsonrpc/auth"
+
+	"github.com/filecoin-project/sector-storage/sealtasks"
+	"github.com/filecoin-project/sector-storage/stores"
+	"github.com/filecoin-project/sector-storage/storiface"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
+	"github.com/filecoin-project/specs-actors/actors/builtin/paych"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
+	"github.com/filecoin-project/specs-storage/storage"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+)
+
+// All permissions are listed in permissioned.go
+var _ = AllPermissions
+
+type CommonStruct struct {
+	Internal struct {
+		AuthVerify func(ctx context.Context, token string) ([]auth.Permission, error) `perm:"read"`
+		AuthNew    func(ctx context.Context, perms []auth.Permission) ([]byte, error) `perm:"admin"`
+
+		NetConnectedness func(context.Context, peer.ID) (network.Connectedness, error) `perm:"read"`
+		NetPeers         func(context.Context) ([]peer.AddrInfo, error)                `perm:"read"`
+		NetConnect       func(context.Context, peer.AddrInfo) error                    `perm:"write"`
+		NetAddrsListen   func(context.Context) (peer.AddrInfo, error)                  `perm:"read"`
+		NetDisconnect    func(context.Context, peer.ID) error                          `perm:"write"`
+		NetFindPeer      func(context.Context, peer.ID) (peer.AddrInfo, error)         `perm:"read"`
+		NetPubsubScores  func(context.Context) ([]api.PubsubScore, error)              `perm:"read"`
+
+		ID      func(context.Context) (peer.ID, error)     `perm:"read"`
+		Version func(context.Context) (api.Version, error) `perm:"read"`
+
+		LogList     func(context.Context) ([]string, error)     `perm:"write"`
+		LogSetLevel func(context.Context, string, string) error `perm:"write"`
+
+		Shutdown func(context.Context) error `perm:"admin"`
+	}
+}
+
+// FullNodeStruct implements API passing calls to user-provided function values.
+type FullNodeStruct struct {
+	CommonStruct
+
+	Internal struct {
+		ChainNotify            func(context.Context) (<-chan []*api.HeadChange, error)                                                            `perm:"read"`
+		ChainHead              func(context.Context) (*types.TipSet, error)                                                                       `perm:"read"`
+		ChainGetRandomness     func(context.Context, types.TipSetKey, crypto.DomainSeparationTag, abi.ChainEpoch, []byte) (abi.Randomness, error) `perm:"read"`
+		ChainGetBlock          func(context.Context, cid.Cid) (*types.BlockHeader, error)                                                         `perm:"read"`
+		ChainGetTipSet         func(context.Context, types.TipSetKey) (*types.TipSet, error)                                                      `perm:"read"`
+		ChainGetBlockMessages  func(context.Context, cid.Cid) (*api.BlockMessages, error)                                                         `perm:"read"`
+		ChainGetParentReceipts func(context.Context, cid.Cid) ([]*types.MessageReceipt, error)                                                    `perm:"read"`
+		ChainGetParentMessages func(context.Context, cid.Cid) ([]api.Message, error)                                                              `perm:"read"`
+		ChainGetTipSetByHeight func(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)                                      `perm:"read"`
+		ChainReadObj           func(context.Context, cid.Cid) ([]byte, error)                                                                     `perm:"read"`
+		ChainHasObj            func(context.Context, cid.Cid) (bool, error)                                                                       `perm:"read"`
+		ChainStatObj           func(context.Context, cid.Cid, cid.Cid) (api.ObjStat, error)                                                       `perm:"read"`
+		ChainSetHead           func(context.Context, types.TipSetKey) error                                                                       `perm:"admin"`
+		ChainGetGenesis        func(context.Context) (*types.TipSet, error)                                                                       `perm:"read"`
+		ChainTipSetWeight      func(context.Context, types.TipSetKey) (types.BigInt, error)                                                       `perm:"read"`
+		ChainGetNode           func(ctx context.Context, p string) (*api.IpldObject, error)                                                       `perm:"read"`
+		ChainGetMessage        func(context.Context, cid.Cid) (*types.Message, error)                                                             `perm:"read"`
+		ChainGetPath           func(context.Context, types.TipSetKey, types.TipSetKey) ([]*api.HeadChange, error)                                 `perm:"read"`
+		ChainExport            func(context.Context, types.TipSetKey) (<-chan []byte, error)                                                      `perm:"read"`
+
+		SyncState          func(context.Context) (*api.SyncState, error)                `perm:"read"`
+		SyncSubmitBlock    func(ctx context.Context, blk *types.BlockMsg) error         `perm:"write"`
+		SyncIncomingBlocks func(ctx context.Context) (<-chan *types.BlockHeader, error) `perm:"read"`
+		SyncMarkBad        func(ctx context.Context, bcid cid.Cid) error                `perm:"admin"`
+		SyncCheckBad       func(ctx context.Context, bcid cid.Cid) (string, error)      `perm:"read"`
+
+		MpoolPending          func(context.Context, types.TipSetKey) ([]*types.SignedMessage, error)                       `perm:"read"`
+		MpoolPush             func(context.Context, *types.SignedMessage) (cid.Cid, error)                                 `perm:"write"`
+		MpoolPushMessage      func(context.Context, *types.Message) (*types.SignedMessage, error)                          `perm:"sign"`
+		MpoolGetNonce         func(context.Context, address.Address) (uint64, error)                                       `perm:"read"`
+		MpoolSub              func(context.Context) (<-chan api.MpoolUpdate, error)                                        `perm:"read"`
+		MpoolEstimateGasPrice func(context.Context, uint64, address.Address, int64, types.TipSetKey) (types.BigInt, error) `perm:"read"`
+
+		MinerGetBaseInfo func(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*api.MiningBaseInfo, error) `perm:"read"`
+		MinerCreateBlock func(context.Context, *api.BlockTemplate) (*types.BlockMsg, error)                                   `perm:"write"`
+
+		WalletNew            func(context.Context, crypto.SigType) (address.Address, error)                       `perm:"write"`
+		WalletHas            func(context.Context, address.Address) (bool, error)                                 `perm:"write"`
+		WalletList           func(context.Context) ([]address.Address, error)                                     `perm:"write"`
+		WalletBalance        func(context.Context, address.Address) (types.BigInt, error)                         `perm:"read"`
+		WalletSign           func(context.Context, address.Address, []byte) (*crypto.Signature, error)            `perm:"sign"`
+		WalletSignMessage    func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"`
+		WalletVerify         func(context.Context, address.Address, []byte, *crypto.Signature) bool               `perm:"read"`
+		WalletDefaultAddress func(context.Context) (address.Address, error)                                       `perm:"write"`
+		WalletSetDefault     func(context.Context, address.Address) error                                         `perm:"admin"`
+		WalletExport         func(context.Context, address.Address) (*types.KeyInfo, error)                       `perm:"admin"`
+		WalletImport         func(context.Context, *types.KeyInfo) (address.Address, error)                       `perm:"admin"`
+		WalletDelete         func(context.Context, address.Address) error                                         `perm:"write"`
+
+		ClientImport      func(ctx context.Context, ref api.FileRef) (cid.Cid, error)                                          `perm:"admin"`
+		ClientListImports func(ctx context.Context) ([]api.Import, error)                                                      `perm:"write"`
+		ClientHasLocal    func(ctx context.Context, root cid.Cid) (bool, error)                                                `perm:"write"`
+		ClientFindData    func(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error)                                    `perm:"read"`
+		ClientStartDeal   func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error)                             `perm:"admin"`
+		ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error)                                                `perm:"read"`
+		ClientListDeals   func(ctx context.Context) ([]api.DealInfo, error)                                                    `perm:"write"`
+		ClientRetrieve    func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error                          `perm:"admin"`
+		ClientQueryAsk    func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) `perm:"read"`
+		ClientCalcCommP   func(ctx context.Context, inpath string, miner address.Address) (*api.CommPRet, error)               `perm:"read"`
+		ClientGenCar      func(ctx context.Context, ref api.FileRef, outpath string) error                                     `perm:"write"`
+
+		StateNetworkName                  func(context.Context) (dtypes.NetworkName, error)                                                                   `perm:"read"`
+		StateMinerSectors                 func(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error)        `perm:"read"`
+		StateMinerProvingSet              func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error)                             `perm:"read"`
+		StateMinerProvingDeadline         func(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error)                                `perm:"read"`
+		StateMinerPower                   func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)                                    `perm:"read"`
+		StateMinerInfo                    func(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error)                                    `perm:"read"`
+		StateMinerDeadlines               func(context.Context, address.Address, types.TipSetKey) (*miner.Deadlines, error)                                   `perm:"read"`
+		StateMinerFaults                  func(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)                                      `perm:"read"`
+		StateAllMinerFaults               func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error)                                        `perm:"read"`
+		StateMinerRecoveries              func(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)                                      `perm:"read"`
+		StateMinerInitialPledgeCollateral func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (types.BigInt, error)                     `perm:"read"`
+		StateMinerAvailableBalance        func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)                                       `perm:"read"`
+		StateSectorPreCommitInfo          func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) `perm:"read"`
+		StateSectorGetInfo                func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error)         `perm:"read"`
+		StateCall                         func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error)                                    `perm:"read"`
+		StateReplay                       func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error)                                           `perm:"read"`
+		StateGetActor                     func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error)                                       `perm:"read"`
+		StateReadState                    func(context.Context, *types.Actor, types.TipSetKey) (*api.ActorState, error)                                       `perm:"read"`
+		StatePledgeCollateral             func(context.Context, types.TipSetKey) (types.BigInt, error)                                                        `perm:"read"`
+		StateWaitMsg                      func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error)                                   `perm:"read"`
+		StateSearchMsg                    func(context.Context, cid.Cid) (*api.MsgLookup, error)                                                              `perm:"read"`
+		StateListMiners                   func(context.Context, types.TipSetKey) ([]address.Address, error)                                                   `perm:"read"`
+		StateListActors                   func(context.Context, types.TipSetKey) ([]address.Address, error)                                                   `perm:"read"`
+		StateMarketBalance                func(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error)                                  `perm:"read"`
+		StateMarketParticipants           func(context.Context, types.TipSetKey) (map[string]api.MarketBalance, error)                                        `perm:"read"`
+		StateMarketDeals                  func(context.Context, types.TipSetKey) (map[string]api.MarketDeal, error)                                           `perm:"read"`
+		StateMarketStorageDeal            func(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error)                                         `perm:"read"`
+		StateLookupID                     func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)                       `perm:"read"`
+		StateAccountKey                   func(context.Context, address.Address, types.TipSetKey) (address.Address, error)                                    `perm:"read"`
+		StateChangedActors                func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error)                                             `perm:"read"`
+		StateGetReceipt                   func(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)                                      `perm:"read"`
+		StateMinerSectorCount             func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error)                                   `perm:"read"`
+		StateListMessages                 func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error)        `perm:"read"`
+		StateCompute                      func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error)           `perm:"read"`
+
+		MsigGetAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)                                                                    `perm:"read"`
+		MsigCreate              func(context.Context, int64, []address.Address, types.BigInt, address.Address, types.BigInt) (cid.Cid, error)                                    `perm:"sign"`
+		MsigPropose             func(context.Context, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error)                          `perm:"sign"`
+		MsigApprove             func(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"`
+		MsigCancel              func(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"`
+
+		MarketEnsureAvailable func(context.Context, address.Address, address.Address, types.BigInt) (cid.Cid, error) `perm:"sign"`
+
+		PaychGet                   func(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*api.ChannelInfo, error)   `perm:"sign"`
+		PaychList                  func(context.Context) ([]address.Address, error)                                                          `perm:"read"`
+		PaychStatus                func(context.Context, address.Address) (*api.PaychStatus, error)                                          `perm:"read"`
+		PaychClose                 func(context.Context, address.Address) (cid.Cid, error)                                                   `perm:"sign"`
+		PaychAllocateLane          func(context.Context, address.Address) (uint64, error)                                                    `perm:"sign"`
+		PaychNewPayment            func(ctx context.Context, from, to address.Address, vouchers []api.VoucherSpec) (*api.PaymentInfo, error) `perm:"sign"`
+		PaychVoucherCheck          func(context.Context, *paych.SignedVoucher) error                                                         `perm:"read"`
+		PaychVoucherCheckValid     func(context.Context, address.Address, *paych.SignedVoucher) error                                        `perm:"read"`
+		PaychVoucherCheckSpendable func(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (bool, error)                `perm:"read"`
+		PaychVoucherAdd            func(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error)  `perm:"write"`
+		PaychVoucherCreate         func(context.Context, address.Address, big.Int, uint64) (*paych.SignedVoucher, error)                     `perm:"sign"`
+		PaychVoucherList           func(context.Context, address.Address) ([]*paych.SignedVoucher, error)                                    `perm:"write"`
+		PaychVoucherSubmit         func(context.Context, address.Address, *paych.SignedVoucher) (cid.Cid, error)                             `perm:"sign"`
+	}
+}
+
+func (c *FullNodeStruct) StateMinerSectorCount(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MinerSectors, error) {
+	return c.Internal.StateMinerSectorCount(ctx, addr, tsk)
+}
+
+type StorageMinerStruct struct {
+	CommonStruct
+
+	Internal struct {
+		ActorAddress    func(context.Context) (address.Address, error)                 `perm:"read"`
+		ActorSectorSize func(context.Context, address.Address) (abi.SectorSize, error) `perm:"read"`
+
+		MiningBase func(context.Context) (*types.TipSet, error) `perm:"read"`
+
+		MarketImportDealData      func(context.Context, cid.Cid, string) error                   `perm:"write"`
+		MarketListDeals           func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"`
+		MarketListIncompleteDeals func(ctx context.Context) ([]storagemarket.MinerDeal, error)   `perm:"read"`
+		MarketSetPrice            func(context.Context, types.BigInt) error                      `perm:"admin"`
+
+		PledgeSector func(context.Context) error `perm:"write"`
+
+		SectorsStatus func(context.Context, abi.SectorNumber) (api.SectorInfo, error) `perm:"read"`
+		SectorsList   func(context.Context) ([]abi.SectorNumber, error)               `perm:"read"`
+		SectorsRefs   func(context.Context) (map[string][]api.SealedRef, error)       `perm:"read"`
+		SectorsUpdate func(context.Context, abi.SectorNumber, api.SectorState) error  `perm:"write"`
+
+		WorkerConnect func(context.Context, string) error                             `perm:"admin"` // TODO: worker perm
+		WorkerStats   func(context.Context) (map[uint64]storiface.WorkerStats, error) `perm:"admin"`
+
+		StorageList          func(context.Context) (map[stores.ID][]stores.Decl, error)                                                                                `perm:"admin"`
+		StorageLocal         func(context.Context) (map[stores.ID]string, error)                                                                                       `perm:"admin"`
+		StorageStat          func(context.Context, stores.ID) (stores.FsStat, error)                                                                                   `perm:"admin"`
+		StorageAttach        func(context.Context, stores.StorageInfo, stores.FsStat) error                                                                            `perm:"admin"`
+		StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType, bool) error                                                         `perm:"admin"`
+		StorageDropSector    func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType) error                                                               `perm:"admin"`
+		StorageFindSector    func(context.Context, abi.SectorID, stores.SectorFileType, bool) ([]stores.SectorStorageInfo, error)                                      `perm:"admin"`
+		StorageInfo          func(context.Context, stores.ID) (stores.StorageInfo, error)                                                                              `perm:"admin"`
+		StorageBestAlloc     func(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredProof, sealing stores.PathType) ([]stores.StorageInfo, error) `perm:"admin"`
+		StorageReportHealth  func(ctx context.Context, id stores.ID, report stores.HealthReport) error                                                                 `perm:"admin"`
+		StorageLock          func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error                             `perm:"admin"`
+
+		DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"`
+		DealsList       func(ctx context.Context) ([]storagemarket.StorageDeal, error)    `perm:"read"`
+
+		StorageAddLocal func(ctx context.Context, path string) error `perm:"admin"`
+	}
+}
+
+type WorkerStruct struct {
+	Internal struct {
+		// TODO: lower perms
+
+		Version func(context.Context) (build.Version, error) `perm:"admin"`
+
+		TaskTypes func(context.Context) (map[sealtasks.TaskType]struct{}, error) `perm:"admin"`
+		Paths     func(context.Context) ([]stores.StoragePath, error)            `perm:"admin"`
+		Info      func(context.Context) (storiface.WorkerInfo, error)            `perm:"admin"`
+
+		SealPreCommit1 func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error)                                                           `perm:"admin"`
+		SealPreCommit2 func(context.Context, abi.SectorID, storage.PreCommit1Out) (cids storage.SectorCids, err error)                                                                                            `perm:"admin"`
+		SealCommit1    func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) `perm:"admin"`
+		SealCommit2    func(context.Context, abi.SectorID, storage.Commit1Out) (storage.Proof, error)                                                                                                             `perm:"admin"`
+		FinalizeSector func(context.Context, abi.SectorID) error                                                                                                                                                  `perm:"admin"`
+		MoveStorage    func(ctx context.Context, sector abi.SectorID) error                                                                                                                                       `perm:"admin"`
+
+		UnsealPiece func(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error `perm:"admin"`
+		ReadPiece   func(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) error                   `perm:"admin"`
+
+		Fetch func(context.Context, abi.SectorID, stores.SectorFileType, stores.PathType, stores.AcquireMode) error `perm:"admin"`
+
+		Closing func(context.Context) (<-chan struct{}, error) `perm:"admin"`
+	}
+}
+
+// CommonStruct
+
+func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]auth.Permission, error) {
+	return c.Internal.AuthVerify(ctx, token)
+}
+
+func (c *CommonStruct) AuthNew(ctx context.Context, perms []auth.Permission) ([]byte, error) {
+	return c.Internal.AuthNew(ctx, perms)
+}
+
+func (c *CommonStruct) NetPubsubScores(ctx context.Context) ([]api.PubsubScore, error) {
+	return c.Internal.NetPubsubScores(ctx)
+}
+func (c *CommonStruct) NetConnectedness(ctx context.Context, pid peer.ID) (network.Connectedness, error) {
+	return c.Internal.NetConnectedness(ctx, pid)
+}
+
+func (c *CommonStruct) NetPeers(ctx context.Context) ([]peer.AddrInfo, error) {
+	return c.Internal.NetPeers(ctx)
+}
+
+func (c *CommonStruct) NetConnect(ctx context.Context, p peer.AddrInfo) error {
+	return c.Internal.NetConnect(ctx, p)
+}
+
+func (c *CommonStruct) NetAddrsListen(ctx context.Context) (peer.AddrInfo, error) {
+	return c.Internal.NetAddrsListen(ctx)
+}
+
+func (c *CommonStruct) NetDisconnect(ctx context.Context, p peer.ID) error {
+	return c.Internal.NetDisconnect(ctx, p)
+}
+
+func (c *CommonStruct) NetFindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo, error) {
+	return c.Internal.NetFindPeer(ctx, p)
+}
+
+// ID implements API.ID
+func (c *CommonStruct) ID(ctx context.Context) (peer.ID, error) {
+	return c.Internal.ID(ctx)
+}
+
+// Version implements API.Version
+func (c *CommonStruct) Version(ctx context.Context) (api.Version, error) {
+	return c.Internal.Version(ctx)
+}
+
+func (c *CommonStruct) LogList(ctx context.Context) ([]string, error) {
+	return c.Internal.LogList(ctx)
+}
+
+func (c *CommonStruct) LogSetLevel(ctx context.Context, group, level string) error {
+	return c.Internal.LogSetLevel(ctx, group, level)
+}
+
+func (c *CommonStruct) Shutdown(ctx context.Context) error {
+	return c.Internal.Shutdown(ctx)
+}
+
+// FullNodeStruct
+
+func (c *FullNodeStruct) ClientListImports(ctx context.Context) ([]api.Import, error) {
+	return c.Internal.ClientListImports(ctx)
+}
+
+func (c *FullNodeStruct) ClientImport(ctx context.Context, ref api.FileRef) (cid.Cid, error) {
+	return c.Internal.ClientImport(ctx, ref)
+}
+
+func (c *FullNodeStruct) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) {
+	return c.Internal.ClientHasLocal(ctx, root)
+}
+
+func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error) {
+	return c.Internal.ClientFindData(ctx, root)
+}
+
+func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) {
+	return c.Internal.ClientStartDeal(ctx, params)
+}
+func (c *FullNodeStruct) ClientGetDealInfo(ctx context.Context, deal cid.Cid) (*api.DealInfo, error) {
+	return c.Internal.ClientGetDealInfo(ctx, deal)
+}
+
+func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) {
+	return c.Internal.ClientListDeals(ctx)
+}
+
+func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error {
+	return c.Internal.ClientRetrieve(ctx, order, ref)
+}
+
+func (c *FullNodeStruct) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) {
+	return c.Internal.ClientQueryAsk(ctx, p, miner)
+}
+func (c *FullNodeStruct) ClientCalcCommP(ctx context.Context, inpath string, miner address.Address) (*api.CommPRet, error) {
+	return c.Internal.ClientCalcCommP(ctx, inpath, miner)
+}
+
+func (c *FullNodeStruct) ClientGenCar(ctx context.Context, ref api.FileRef, outpath string) error {
+	return c.Internal.ClientGenCar(ctx, ref, outpath)
+}
+
+func (c *FullNodeStruct) MpoolPending(ctx context.Context, tsk types.TipSetKey) ([]*types.SignedMessage, error) {
+	return c.Internal.MpoolPending(ctx, tsk)
+}
+
+func (c *FullNodeStruct) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) {
+	return c.Internal.MpoolPush(ctx, smsg)
+}
+
+func (c *FullNodeStruct) MpoolPushMessage(ctx context.Context, msg *types.Message) (*types.SignedMessage, error) {
+	return c.Internal.MpoolPushMessage(ctx, msg)
+}
+
+func (c *FullNodeStruct) MpoolSub(ctx context.Context) (<-chan api.MpoolUpdate, error) {
+	return c.Internal.MpoolSub(ctx)
+}
+
+func (c *FullNodeStruct) MpoolEstimateGasPrice(ctx context.Context, nblocksincl uint64, sender address.Address, limit int64, tsk types.TipSetKey) (types.BigInt, error) {
+	return c.Internal.MpoolEstimateGasPrice(ctx, nblocksincl, sender, limit, tsk)
+}
+
+func (c *FullNodeStruct) MinerGetBaseInfo(ctx context.Context, maddr address.Address, epoch abi.ChainEpoch, tsk types.TipSetKey) (*api.MiningBaseInfo, error) {
+	return c.Internal.MinerGetBaseInfo(ctx, maddr, epoch, tsk)
+}
+
+func (c *FullNodeStruct) MinerCreateBlock(ctx context.Context, bt *api.BlockTemplate) (*types.BlockMsg, error) {
+	return c.Internal.MinerCreateBlock(ctx, bt)
+}
+
+func (c *FullNodeStruct) ChainHead(ctx context.Context) (*types.TipSet, error) {
+	return c.Internal.ChainHead(ctx)
+}
+
+func (c *FullNodeStruct) ChainGetRandomness(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
+	return c.Internal.ChainGetRandomness(ctx, tsk, personalization, randEpoch, entropy)
+}
+
+func (c *FullNodeStruct) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) {
+	return c.Internal.ChainGetTipSetByHeight(ctx, h, tsk)
+}
+
+func (c *FullNodeStruct) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) {
+	return c.Internal.WalletNew(ctx, typ)
+}
+
+func (c *FullNodeStruct) WalletHas(ctx context.Context, addr address.Address) (bool, error) {
+	return c.Internal.WalletHas(ctx, addr)
+}
+
+func (c *FullNodeStruct) WalletList(ctx context.Context) ([]address.Address, error) {
+	return c.Internal.WalletList(ctx)
+}
+
+func (c *FullNodeStruct) WalletBalance(ctx context.Context, a address.Address) (types.BigInt, error) {
+	return c.Internal.WalletBalance(ctx, a)
+}
+
+func (c *FullNodeStruct) WalletSign(ctx context.Context, k address.Address, msg []byte) (*crypto.Signature, error) {
+	return c.Internal.WalletSign(ctx, k, msg)
+}
+
+func (c *FullNodeStruct) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) {
+	return c.Internal.WalletSignMessage(ctx, k, msg)
+}
+
+func (c *FullNodeStruct) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) bool {
+	return c.Internal.WalletVerify(ctx, k, msg, sig)
+}
+
+func (c *FullNodeStruct) WalletDefaultAddress(ctx context.Context) (address.Address, error) {
+	return c.Internal.WalletDefaultAddress(ctx)
+}
+
+func (c *FullNodeStruct) WalletSetDefault(ctx context.Context, a address.Address) error {
+	return c.Internal.WalletSetDefault(ctx, a)
+}
+
+func (c *FullNodeStruct) WalletExport(ctx context.Context, a address.Address) (*types.KeyInfo, error) {
+	return c.Internal.WalletExport(ctx, a)
+}
+
+func (c *FullNodeStruct) WalletImport(ctx context.Context, ki *types.KeyInfo) (address.Address, error) {
+	return c.Internal.WalletImport(ctx, ki)
+}
+
+func (c *FullNodeStruct) WalletDelete(ctx context.Context, addr address.Address) error {
+	return c.Internal.WalletDelete(ctx, addr)
+}
+
+func (c *FullNodeStruct) MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) {
+	return c.Internal.MpoolGetNonce(ctx, addr)
+}
+
+func (c *FullNodeStruct) ChainGetBlock(ctx context.Context, b cid.Cid) (*types.BlockHeader, error) {
+	return c.Internal.ChainGetBlock(ctx, b)
+}
+
+func (c *FullNodeStruct) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) {
+	return c.Internal.ChainGetTipSet(ctx, key)
+}
+
+func (c *FullNodeStruct) ChainGetBlockMessages(ctx context.Context, b cid.Cid) (*api.BlockMessages, error) {
+	return c.Internal.ChainGetBlockMessages(ctx, b)
+}
+
+func (c *FullNodeStruct) ChainGetParentReceipts(ctx context.Context, b cid.Cid) ([]*types.MessageReceipt, error) {
+	return c.Internal.ChainGetParentReceipts(ctx, b)
+}
+
+func (c *FullNodeStruct) ChainGetParentMessages(ctx context.Context, b cid.Cid) ([]api.Message, error) {
+	return c.Internal.ChainGetParentMessages(ctx, b)
+}
+
+func (c *FullNodeStruct) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, error) {
+	return c.Internal.ChainNotify(ctx)
+}
+
+func (c *FullNodeStruct) ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error) {
+	return c.Internal.ChainReadObj(ctx, obj)
+}
+
+func (c *FullNodeStruct) ChainHasObj(ctx context.Context, o cid.Cid) (bool, error) {
+	return c.Internal.ChainHasObj(ctx, o)
+}
+
+func (c *FullNodeStruct) ChainStatObj(ctx context.Context, obj, base cid.Cid) (api.ObjStat, error) {
+	return c.Internal.ChainStatObj(ctx, obj, base)
+}
+
+func (c *FullNodeStruct) ChainSetHead(ctx context.Context, tsk types.TipSetKey) error {
+	return c.Internal.ChainSetHead(ctx, tsk)
+}
+
+func (c *FullNodeStruct) ChainGetGenesis(ctx context.Context) (*types.TipSet, error) {
+	return c.Internal.ChainGetGenesis(ctx)
+}
+
+func (c *FullNodeStruct) ChainTipSetWeight(ctx context.Context, tsk types.TipSetKey) (types.BigInt, error) {
+	return c.Internal.ChainTipSetWeight(ctx, tsk)
+}
+
+func (c *FullNodeStruct) ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error) {
+	return c.Internal.ChainGetNode(ctx, p)
+}
+
+func (c *FullNodeStruct) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) {
+	return c.Internal.ChainGetMessage(ctx, mc)
+}
+
+func (c *FullNodeStruct) ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*api.HeadChange, error) {
+	return c.Internal.ChainGetPath(ctx, from, to)
+}
+
+func (c *FullNodeStruct) ChainExport(ctx context.Context, tsk types.TipSetKey) (<-chan []byte, error) {
+	return c.Internal.ChainExport(ctx, tsk)
+}
+
+func (c *FullNodeStruct) SyncState(ctx context.Context) (*api.SyncState, error) {
+	return c.Internal.SyncState(ctx)
+}
+
+func (c *FullNodeStruct) SyncSubmitBlock(ctx context.Context, blk *types.BlockMsg) error {
+	return c.Internal.SyncSubmitBlock(ctx, blk)
+}
+
+func (c *FullNodeStruct) SyncIncomingBlocks(ctx context.Context) (<-chan *types.BlockHeader, error) {
+	return c.Internal.SyncIncomingBlocks(ctx)
+}
+
+func (c *FullNodeStruct) SyncMarkBad(ctx context.Context, bcid cid.Cid) error {
+	return c.Internal.SyncMarkBad(ctx, bcid)
+}
+
+func (c *FullNodeStruct) SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error) {
+	return c.Internal.SyncCheckBad(ctx, bcid)
+}
+
+func (c *FullNodeStruct) StateNetworkName(ctx context.Context) (dtypes.NetworkName, error) {
+	return c.Internal.StateNetworkName(ctx)
+}
+
+func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, filter *abi.BitField, filterOut bool, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) {
+	return c.Internal.StateMinerSectors(ctx, addr, filter, filterOut, tsk)
+}
+
+func (c *FullNodeStruct) StateMinerProvingSet(ctx context.Context, addr address.Address, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) {
+	return c.Internal.StateMinerProvingSet(ctx, addr, tsk)
+}
+
+func (c *FullNodeStruct) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*miner.DeadlineInfo, error) {
+	return c.Internal.StateMinerProvingDeadline(ctx, addr, tsk)
+}
+
+func (c *FullNodeStruct) StateMinerPower(ctx context.Context, a address.Address, tsk types.TipSetKey) (*api.MinerPower, error) {
+	return c.Internal.StateMinerPower(ctx, a, tsk)
+}
+
+func (c *FullNodeStruct) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) {
+	return c.Internal.StateMinerInfo(ctx, actor, tsk)
+}
+
+func (c *FullNodeStruct) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) (*miner.Deadlines, error) {
+	return c.Internal.StateMinerDeadlines(ctx, m, tsk)
+}
+
+func (c *FullNodeStruct) StateMinerFaults(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*abi.BitField, error) {
+	return c.Internal.StateMinerFaults(ctx, actor, tsk)
+}
+
+func (c *FullNodeStruct) StateAllMinerFaults(ctx context.Context, cutoff abi.ChainEpoch, endTsk types.TipSetKey) ([]*api.Fault, error) {
+	return c.Internal.StateAllMinerFaults(ctx, cutoff, endTsk)
+}
+
+func (c *FullNodeStruct) StateMinerRecoveries(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*abi.BitField, error) {
+	return c.Internal.StateMinerRecoveries(ctx, actor, tsk)
+}
+
+func (c *FullNodeStruct) StateMinerInitialPledgeCollateral(ctx context.Context, maddr address.Address, snum abi.SectorNumber, tsk types.TipSetKey) (types.BigInt, error) {
+	return c.Internal.StateMinerInitialPledgeCollateral(ctx, maddr, snum, tsk)
+}
+
+func (c *FullNodeStruct) StateMinerAvailableBalance(ctx context.Context, maddr address.Address, tsk types.TipSetKey) (types.BigInt, error) {
+	return c.Internal.StateMinerAvailableBalance(ctx, maddr, tsk)
+}
+
+func (c *FullNodeStruct) StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) {
+	return c.Internal.StateSectorPreCommitInfo(ctx, maddr, n, tsk)
+}
+
+func (c *FullNodeStruct) StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error) {
+	return c.Internal.StateSectorGetInfo(ctx, maddr, n, tsk)
+}
+
+func (c *FullNodeStruct) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) {
+	return c.Internal.StateCall(ctx, msg, tsk)
+}
+
+func (c *FullNodeStruct) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) {
+	return c.Internal.StateReplay(ctx, tsk, mc)
+}
+
+func (c *FullNodeStruct) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) {
+	return c.Internal.StateGetActor(ctx, actor, tsk)
+}
+
+func (c *FullNodeStruct) StateReadState(ctx context.Context, act *types.Actor, tsk types.TipSetKey) (*api.ActorState, error) {
+	return c.Internal.StateReadState(ctx, act, tsk)
+}
+
+func (c *FullNodeStruct) StatePledgeCollateral(ctx context.Context, tsk types.TipSetKey) (types.BigInt, error) {
+	return c.Internal.StatePledgeCollateral(ctx, tsk)
+}
+
+func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confidence uint64) (*api.MsgLookup, error) {
+	return c.Internal.StateWaitMsg(ctx, msgc, confidence)
+}
+
+func (c *FullNodeStruct) StateSearchMsg(ctx context.Context, msgc cid.Cid) (*api.MsgLookup, error) {
+	return c.Internal.StateSearchMsg(ctx, msgc)
+}
+
+func (c *FullNodeStruct) StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) {
+	return c.Internal.StateListMiners(ctx, tsk)
+}
+
+func (c *FullNodeStruct) StateListActors(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) {
+	return c.Internal.StateListActors(ctx, tsk)
+}
+
+func (c *FullNodeStruct) StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error) {
+	return c.Internal.StateMarketBalance(ctx, addr, tsk)
+}
+
+func (c *FullNodeStruct) StateMarketParticipants(ctx context.Context, tsk types.TipSetKey) (map[string]api.MarketBalance, error) {
+	return c.Internal.StateMarketParticipants(ctx, tsk)
+}
+
+func (c *FullNodeStruct) StateMarketDeals(ctx context.Context, tsk types.TipSetKey) (map[string]api.MarketDeal, error) {
+	return c.Internal.StateMarketDeals(ctx, tsk)
+}
+
+func (c *FullNodeStruct) StateMarketStorageDeal(ctx context.Context, dealid abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) {
+	return c.Internal.StateMarketStorageDeal(ctx, dealid, tsk)
+}
+
+func (c *FullNodeStruct) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) {
+	return c.Internal.StateLookupID(ctx, addr, tsk)
+}
+
+func (c *FullNodeStruct) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) {
+	return c.Internal.StateAccountKey(ctx, addr, tsk)
+}
+
+func (c *FullNodeStruct) StateChangedActors(ctx context.Context, olnstate cid.Cid, newstate cid.Cid) (map[string]types.Actor, error) {
+	return c.Internal.StateChangedActors(ctx, olnstate, newstate)
+}
+
+func (c *FullNodeStruct) StateGetReceipt(ctx context.Context, msg cid.Cid, tsk types.TipSetKey) (*types.MessageReceipt, error) {
+	return c.Internal.StateGetReceipt(ctx, msg, tsk)
+}
+
+func (c *FullNodeStruct) StateListMessages(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) {
+	return c.Internal.StateListMessages(ctx, match, tsk, toht)
+}
+
+func (c *FullNodeStruct) StateCompute(ctx context.Context, height abi.ChainEpoch, msgs []*types.Message, tsk types.TipSetKey) (*api.ComputeStateOutput, error) {
+	return c.Internal.StateCompute(ctx, height, msgs, tsk)
+}
+
+func (c *FullNodeStruct) MsigGetAvailableBalance(ctx context.Context, a address.Address, tsk types.TipSetKey) (types.BigInt, error) {
+	return c.Internal.MsigGetAvailableBalance(ctx, a, tsk)
+}
+
+func (c *FullNodeStruct) MsigCreate(ctx context.Context, req int64, addrs []address.Address, val types.BigInt, src address.Address, gp types.BigInt) (cid.Cid, error) {
+	return c.Internal.MsigCreate(ctx, req, addrs, val, src, gp)
+}
+
+func (c *FullNodeStruct) MsigPropose(ctx context.Context, msig address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) {
+	return c.Internal.MsigPropose(ctx, msig, to, amt, src, method, params)
+}
+
+func (c *FullNodeStruct) MsigApprove(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) {
+	return c.Internal.MsigApprove(ctx, msig, txID, proposer, to, amt, src, method, params)
+}
+
+func (c *FullNodeStruct) MsigCancel(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) {
+	return c.Internal.MsigCancel(ctx, msig, txID, proposer, to, amt, src, method, params)
+}
+
+func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr, wallet address.Address, amt types.BigInt) (cid.Cid, error) {
+	return c.Internal.MarketEnsureAvailable(ctx, addr, wallet, amt)
+}
+
+func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*api.ChannelInfo, error) {
+	return c.Internal.PaychGet(ctx, from, to, ensureFunds)
+}
+
+func (c *FullNodeStruct) PaychList(ctx context.Context) ([]address.Address, error) {
+	return c.Internal.PaychList(ctx)
+}
+
+func (c *FullNodeStruct) PaychStatus(ctx context.Context, pch address.Address) (*api.PaychStatus, error) {
+	return c.Internal.PaychStatus(ctx, pch)
+}
+
+func (c *FullNodeStruct) PaychVoucherCheckValid(ctx context.Context, addr address.Address, sv *paych.SignedVoucher) error {
+	return c.Internal.PaychVoucherCheckValid(ctx, addr, sv)
+}
+
+func (c *FullNodeStruct) PaychVoucherCheckSpendable(ctx context.Context, addr address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (bool, error) {
+	return c.Internal.PaychVoucherCheckSpendable(ctx, addr, sv, secret, proof)
+}
+
+func (c *FullNodeStruct) PaychVoucherAdd(ctx context.Context, addr address.Address, sv *paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) {
+	return c.Internal.PaychVoucherAdd(ctx, addr, sv, proof, minDelta)
+}
+
+func (c *FullNodeStruct) PaychVoucherCreate(ctx context.Context, pch address.Address, amt types.BigInt, lane uint64) (*paych.SignedVoucher, error) {
+	return c.Internal.PaychVoucherCreate(ctx, pch, amt, lane)
+}
+
+func (c *FullNodeStruct) PaychVoucherList(ctx context.Context, pch address.Address) ([]*paych.SignedVoucher, error) {
+	return c.Internal.PaychVoucherList(ctx, pch)
+}
+
+func (c *FullNodeStruct) PaychClose(ctx context.Context, a address.Address) (cid.Cid, error) {
+	return c.Internal.PaychClose(ctx, a)
+}
+
+func (c *FullNodeStruct) PaychAllocateLane(ctx context.Context, ch address.Address) (uint64, error) {
+	return c.Internal.PaychAllocateLane(ctx, ch)
+}
+
+func (c *FullNodeStruct) PaychNewPayment(ctx context.Context, from, to address.Address, vouchers []api.VoucherSpec) (*api.PaymentInfo, error) {
+	return c.Internal.PaychNewPayment(ctx, from, to, vouchers)
+}
+
+func (c *FullNodeStruct) PaychVoucherSubmit(ctx context.Context, ch address.Address, sv *paych.SignedVoucher) (cid.Cid, error) {
+	return c.Internal.PaychVoucherSubmit(ctx, ch, sv)
+}
+
+// StorageMinerStruct
+
+func (c *StorageMinerStruct) ActorAddress(ctx context.Context) (address.Address, error) {
+	return c.Internal.ActorAddress(ctx)
+}
+
+func (c *StorageMinerStruct) MiningBase(ctx context.Context) (*types.TipSet, error) {
+	return c.Internal.MiningBase(ctx)
+}
+
+func (c *StorageMinerStruct) ActorSectorSize(ctx context.Context, addr address.Address) (abi.SectorSize, error) {
+	return c.Internal.ActorSectorSize(ctx, addr)
+}
+
+func (c *StorageMinerStruct) PledgeSector(ctx context.Context) error {
+	return c.Internal.PledgeSector(ctx)
+}
+
+// Get the status of a given sector by ID
+func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid abi.SectorNumber) (api.SectorInfo, error) {
+	return c.Internal.SectorsStatus(ctx, sid)
+}
+
+// List all staged sectors
+func (c *StorageMinerStruct) SectorsList(ctx context.Context) ([]abi.SectorNumber, error) {
+	return c.Internal.SectorsList(ctx)
+}
+
+func (c *StorageMinerStruct) SectorsRefs(ctx context.Context) (map[string][]api.SealedRef, error) {
+	return c.Internal.SectorsRefs(ctx)
+}
+
+func (c *StorageMinerStruct) SectorsUpdate(ctx context.Context, id abi.SectorNumber, state api.SectorState) error {
+	return c.Internal.SectorsUpdate(ctx, id, state)
+}
+
+func (c *StorageMinerStruct) WorkerConnect(ctx context.Context, url string) error {
+	return c.Internal.WorkerConnect(ctx, url)
+}
+
+func (c *StorageMinerStruct) WorkerStats(ctx context.Context) (map[uint64]storiface.WorkerStats, error) {
+	return c.Internal.WorkerStats(ctx)
+}
+
+func (c *StorageMinerStruct) StorageAttach(ctx context.Context, si stores.StorageInfo, st stores.FsStat) error {
+	return c.Internal.StorageAttach(ctx, si, st)
+}
+
+func (c *StorageMinerStruct) StorageDeclareSector(ctx context.Context, storageId stores.ID, s abi.SectorID, ft stores.SectorFileType, primary bool) error {
+	return c.Internal.StorageDeclareSector(ctx, storageId, s, ft, primary)
+}
+
+func (c *StorageMinerStruct) StorageDropSector(ctx context.Context, storageId stores.ID, s abi.SectorID, ft stores.SectorFileType) error {
+	return c.Internal.StorageDropSector(ctx, storageId, s, ft)
+}
+
+func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types stores.SectorFileType, allowFetch bool) ([]stores.SectorStorageInfo, error) {
+	return c.Internal.StorageFindSector(ctx, si, types, allowFetch)
+}
+
+func (c *StorageMinerStruct) StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) {
+	return c.Internal.StorageList(ctx)
+}
+
+func (c *StorageMinerStruct) StorageLocal(ctx context.Context) (map[stores.ID]string, error) {
+	return c.Internal.StorageLocal(ctx)
+}
+
+func (c *StorageMinerStruct) StorageStat(ctx context.Context, id stores.ID) (stores.FsStat, error) {
+	return c.Internal.StorageStat(ctx, id)
+}
+
+func (c *StorageMinerStruct) StorageInfo(ctx context.Context, id stores.ID) (stores.StorageInfo, error) {
+	return c.Internal.StorageInfo(ctx, id)
+}
+
+func (c *StorageMinerStruct) StorageBestAlloc(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredProof, pt stores.PathType) ([]stores.StorageInfo, error) {
+	return c.Internal.StorageBestAlloc(ctx, allocate, spt, pt)
+}
+
+func (c *StorageMinerStruct) StorageReportHealth(ctx context.Context, id stores.ID, report stores.HealthReport) error {
+	return c.Internal.StorageReportHealth(ctx, id, report)
+}
+
+func (c *StorageMinerStruct) StorageLock(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error {
+	return c.Internal.StorageLock(ctx, sector, read, write)
+}
+
+func (c *StorageMinerStruct) MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error {
+	return c.Internal.MarketImportDealData(ctx, propcid, path)
+}
+
+func (c *StorageMinerStruct) MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error) {
+	return c.Internal.MarketListDeals(ctx)
+}
+
+func (c *StorageMinerStruct) MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error) {
+	return c.Internal.MarketListIncompleteDeals(ctx)
+}
+
+func (c *StorageMinerStruct) MarketSetPrice(ctx context.Context, p types.BigInt) error {
+	return c.Internal.MarketSetPrice(ctx, p)
+}
+
+func (c *StorageMinerStruct) DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error {
+	return c.Internal.DealsImportData(ctx, dealPropCid, file)
+}
+
+func (c *StorageMinerStruct) DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) {
+	return c.Internal.DealsList(ctx)
+}
+
+func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) error {
+	return c.Internal.StorageAddLocal(ctx, path)
+}
+
+// WorkerStruct
+
+func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) {
+	return w.Internal.Version(ctx)
+}
+
+func (w *WorkerStruct) TaskTypes(ctx context.Context) (map[sealtasks.TaskType]struct{}, error) {
+	return w.Internal.TaskTypes(ctx)
+}
+
+func (w *WorkerStruct) Paths(ctx context.Context) ([]stores.StoragePath, error) {
+	return w.Internal.Paths(ctx)
+}
+
+func (w *WorkerStruct) Info(ctx context.Context) (storiface.WorkerInfo, error) {
+	return w.Internal.Info(ctx)
+}
+
+func (w *WorkerStruct) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) {
+	return w.Internal.SealPreCommit1(ctx, sector, ticket, pieces)
+}
+
+func (w *WorkerStruct) SealPreCommit2(ctx context.Context, sector abi.SectorID, p1o storage.PreCommit1Out) (storage.SectorCids, error) {
+	return w.Internal.SealPreCommit2(ctx, sector, p1o)
+}
+
+func (w *WorkerStruct) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) {
+	return w.Internal.SealCommit1(ctx, sector, ticket, seed, pieces, cids)
+}
+
+func (w *WorkerStruct) SealCommit2(ctx context.Context, sector abi.SectorID, c1o storage.Commit1Out) (storage.Proof, error) {
+	return w.Internal.SealCommit2(ctx, sector, c1o)
+}
+
+func (w *WorkerStruct) FinalizeSector(ctx context.Context, sector abi.SectorID) error {
+	return w.Internal.FinalizeSector(ctx, sector)
+}
+
+func (w *WorkerStruct) MoveStorage(ctx context.Context, sector abi.SectorID) error {
+	return w.Internal.MoveStorage(ctx, sector)
+}
+
+func (w *WorkerStruct) UnsealPiece(ctx context.Context, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, c cid.Cid) error {
+	return w.Internal.UnsealPiece(ctx, id, index, size, randomness, c)
+}
+
+func (w *WorkerStruct) ReadPiece(ctx context.Context, writer io.Writer, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) error {
+	return w.Internal.ReadPiece(ctx, writer, id, index, size)
+}
+
+func (w *WorkerStruct) Fetch(ctx context.Context, id abi.SectorID, fileType stores.SectorFileType, ptype stores.PathType, am stores.AcquireMode) error {
+	return w.Internal.Fetch(ctx, id, fileType, ptype, am)
+}
+
+func (w *WorkerStruct) Closing(ctx context.Context) (<-chan struct{}, error) {
+	return w.Internal.Closing(ctx)
+}
+
+var _ api.Common = &CommonStruct{}
+var _ api.FullNode = &FullNodeStruct{}
+var _ api.StorageMiner = &StorageMinerStruct{}
+var _ api.WorkerAPI = &WorkerStruct{}

api/apistruct/struct_test.go

@@ -0,0 +1,9 @@
+package apistruct
+
+import "testing"
+
+func TestPermTags(t *testing.T) {
+	_ = PermissionedFullAPI(&FullNodeStruct{})
+	_ = PermissionedStorMinerAPI(&StorageMinerStruct{})
+	_ = PermissionedWorkerAPI(&WorkerStruct{})
+}

api/cbor_gen.go

@@ -5,21 +5,14 @@ package api
 import (
 	"fmt"
 	"io"
-	"math"
-	"sort"
 
-	abi "github.com/filecoin-project/go-state-types/abi"
-	market "github.com/filecoin-project/specs-actors/actors/builtin/market"
-	paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
-	cid "github.com/ipfs/go-cid"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/builtin/paych"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	xerrors "golang.org/x/xerrors"
 )
 
 var _ = xerrors.Errorf
-var _ = cid.Undef
-var _ = math.E
-var _ = sort.Sort
 
 func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	if t == nil {
@@ -40,7 +33,7 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Channel"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Channel")); err != nil {
+	if _, err := io.WriteString(w, "Channel"); err != nil {
 		return err
 	}
 
@@ -48,20 +41,26 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	// t.WaitSentinel (cid.Cid) (struct)
-	if len("WaitSentinel") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"WaitSentinel\" was too long")
+	// t.ChannelMessage (cid.Cid) (struct)
+	if len("ChannelMessage") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"ChannelMessage\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("WaitSentinel"))); err != nil {
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ChannelMessage"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("WaitSentinel")); err != nil {
+	if _, err := io.WriteString(w, "ChannelMessage"); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteCidBuf(scratch, w, t.WaitSentinel); err != nil {
-		return xerrors.Errorf("failed to write cid field t.WaitSentinel: %w", err)
+	if t.ChannelMessage == nil {
+		if _, err := w.Write(cbg.CborNull); err != nil {
+			return err
+		}
+	} else {
+		if err := cbg.WriteCidBuf(scratch, w, *t.ChannelMessage); err != nil {
+			return xerrors.Errorf("failed to write cid field t.ChannelMessage: %w", err)
+		}
 	}
 
 	// t.Vouchers ([]*paych.SignedVoucher) (slice)
@@ -72,7 +71,7 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Vouchers"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Vouchers")); err != nil {
+	if _, err := io.WriteString(w, "Vouchers"); err != nil {
 		return err
 	}
 
@@ -92,8 +91,6 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 }
 
 func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
-	*t = PaymentInfo{}
-
 	br := cbg.GetPeeker(r)
 	scratch := make([]byte, 8)
 
@@ -134,17 +131,29 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 				}
 
 			}
-			// t.WaitSentinel (cid.Cid) (struct)
-		case "WaitSentinel":
+			// t.ChannelMessage (cid.Cid) (struct)
+		case "ChannelMessage":
 
 			{
 
-				c, err := cbg.ReadCid(br)
+				pb, err := br.PeekByte()
 				if err != nil {
-					return xerrors.Errorf("failed to read cid field t.WaitSentinel: %w", err)
+					return err
 				}
+				if pb == cbg.CborNull[0] {
+					var nbuf [1]byte
+					if _, err := br.Read(nbuf[:]); err != nil {
+						return err
+					}
+				} else {
 
-				t.WaitSentinel = c
+					c, err := cbg.ReadCid(br)
+					if err != nil {
+						return xerrors.Errorf("failed to read cid field t.ChannelMessage: %w", err)
+					}
+
+					t.ChannelMessage = &c
+				}
 
 			}
 			// t.Vouchers ([]*paych.SignedVoucher) (slice)
@@ -178,8 +187,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			// Field doesn't exist on this type, so ignore it
-			cbg.ScanForLinks(r, func(cid.Cid) {})
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
 		}
 	}
 
@@ -204,7 +212,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SectorID"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("SectorID")); err != nil {
+	if _, err := io.WriteString(w, "SectorID"); err != nil {
 		return err
 	}
 
@@ -212,7 +220,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	// t.Offset (abi.PaddedPieceSize) (uint64)
+	// t.Offset (uint64) (uint64)
 	if len("Offset") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"Offset\" was too long")
 	}
@@ -220,7 +228,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Offset"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Offset")); err != nil {
+	if _, err := io.WriteString(w, "Offset"); err != nil {
 		return err
 	}
 
@@ -236,7 +244,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Size"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Size")); err != nil {
+	if _, err := io.WriteString(w, "Size"); err != nil {
 		return err
 	}
 
@@ -248,8 +256,6 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 }
 
 func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
-	*t = SealedRef{}
-
 	br := cbg.GetPeeker(r)
 	scratch := make([]byte, 8)
 
@@ -295,7 +301,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 				t.SectorID = abi.SectorNumber(extra)
 
 			}
-			// t.Offset (abi.PaddedPieceSize) (uint64)
+			// t.Offset (uint64) (uint64)
 		case "Offset":
 
 			{
@@ -307,7 +313,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 				if maj != cbg.MajUnsignedInt {
 					return fmt.Errorf("wrong type for uint64 field")
 				}
-				t.Offset = abi.PaddedPieceSize(extra)
+				t.Offset = uint64(extra)
 
 			}
 			// t.Size (abi.UnpaddedPieceSize) (uint64)
@@ -327,8 +333,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			// Field doesn't exist on this type, so ignore it
-			cbg.ScanForLinks(r, func(cid.Cid) {})
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
 		}
 	}
 
@@ -353,7 +358,7 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Refs"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Refs")); err != nil {
+	if _, err := io.WriteString(w, "Refs"); err != nil {
 		return err
 	}
 
@@ -373,8 +378,6 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
 }
 
 func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
-	*t = SealedRefs{}
-
 	br := cbg.GetPeeker(r)
 	scratch := make([]byte, 8)
 
@@ -436,8 +439,7 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			// Field doesn't exist on this type, so ignore it
-			cbg.ScanForLinks(r, func(cid.Cid) {})
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
 		}
 	}
 
@@ -462,7 +464,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Value")); err != nil {
+	if _, err := io.WriteString(w, "Value"); err != nil {
 		return err
 	}
 
@@ -474,7 +476,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	if _, err := w.Write(t.Value[:]); err != nil {
+	if _, err := w.Write(t.Value); err != nil {
 		return err
 	}
 
@@ -486,7 +488,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Epoch")); err != nil {
+	if _, err := io.WriteString(w, "Epoch"); err != nil {
 		return err
 	}
 
@@ -503,8 +505,6 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 }
 
 func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
-	*t = SealTicket{}
-
 	br := cbg.GetPeeker(r)
 	scratch := make([]byte, 8)
 
@@ -549,12 +549,8 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
 			if maj != cbg.MajByteString {
 				return fmt.Errorf("expected byte array")
 			}
-
-			if extra > 0 {
-				t.Value = make([]uint8, extra)
-			}
-
-			if _, err := io.ReadFull(br, t.Value[:]); err != nil {
+			t.Value = make([]byte, extra)
+			if _, err := io.ReadFull(br, t.Value); err != nil {
 				return err
 			}
 			// t.Epoch (abi.ChainEpoch) (int64)
@@ -585,8 +581,7 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			// Field doesn't exist on this type, so ignore it
-			cbg.ScanForLinks(r, func(cid.Cid) {})
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
 		}
 	}
 
@@ -611,7 +606,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Value")); err != nil {
+	if _, err := io.WriteString(w, "Value"); err != nil {
 		return err
 	}
 
@@ -623,7 +618,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	if _, err := w.Write(t.Value[:]); err != nil {
+	if _, err := w.Write(t.Value); err != nil {
 		return err
 	}
 
@@ -635,7 +630,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Epoch")); err != nil {
+	if _, err := io.WriteString(w, "Epoch"); err != nil {
 		return err
 	}
 
@@ -652,8 +647,6 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 }
 
 func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
-	*t = SealSeed{}
-
 	br := cbg.GetPeeker(r)
 	scratch := make([]byte, 8)
 
@@ -698,12 +691,8 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
 			if maj != cbg.MajByteString {
 				return fmt.Errorf("expected byte array")
 			}
-
-			if extra > 0 {
-				t.Value = make([]uint8, extra)
-			}
-
-			if _, err := io.ReadFull(br, t.Value[:]); err != nil {
+			t.Value = make([]byte, extra)
+			if _, err := io.ReadFull(br, t.Value); err != nil {
 				return err
 			}
 			// t.Epoch (abi.ChainEpoch) (int64)
@@ -734,386 +723,7 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			// Field doesn't exist on this type, so ignore it
-			cbg.ScanForLinks(r, func(cid.Cid) {})
-		}
-	}
-
-	return nil
-}
-func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{165}); err != nil {
-		return err
-	}
-
-	scratch := make([]byte, 9)
-
-	// t.PublishCid (cid.Cid) (struct)
-	if len("PublishCid") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"PublishCid\" was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PublishCid"))); err != nil {
-		return err
-	}
-	if _, err := io.WriteString(w, string("PublishCid")); err != nil {
-		return err
-	}
-
-	if t.PublishCid == nil {
-		if _, err := w.Write(cbg.CborNull); err != nil {
-			return err
-		}
-	} else {
-		if err := cbg.WriteCidBuf(scratch, w, *t.PublishCid); err != nil {
-			return xerrors.Errorf("failed to write cid field t.PublishCid: %w", err)
-		}
-	}
-
-	// t.DealID (abi.DealID) (uint64)
-	if len("DealID") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"DealID\" was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealID"))); err != nil {
-		return err
-	}
-	if _, err := io.WriteString(w, string("DealID")); err != nil {
-		return err
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.DealID)); err != nil {
-		return err
-	}
-
-	// t.DealProposal (market.DealProposal) (struct)
-	if len("DealProposal") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"DealProposal\" was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealProposal"))); err != nil {
-		return err
-	}
-	if _, err := io.WriteString(w, string("DealProposal")); err != nil {
-		return err
-	}
-
-	if err := t.DealProposal.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.DealSchedule (api.DealSchedule) (struct)
-	if len("DealSchedule") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"DealSchedule\" was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealSchedule"))); err != nil {
-		return err
-	}
-	if _, err := io.WriteString(w, string("DealSchedule")); err != nil {
-		return err
-	}
-
-	if err := t.DealSchedule.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.KeepUnsealed (bool) (bool)
-	if len("KeepUnsealed") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"KeepUnsealed\" was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("KeepUnsealed"))); err != nil {
-		return err
-	}
-	if _, err := io.WriteString(w, string("KeepUnsealed")); err != nil {
-		return err
-	}
-
-	if err := cbg.WriteBool(w, t.KeepUnsealed); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
-	*t = PieceDealInfo{}
-
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
-
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajMap {
-		return fmt.Errorf("cbor input should be of type map")
-	}
-
-	if extra > cbg.MaxLength {
-		return fmt.Errorf("PieceDealInfo: map struct too large (%d)", extra)
-	}
-
-	var name string
-	n := extra
-
-	for i := uint64(0); i < n; i++ {
-
-		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
-			if err != nil {
-				return err
-			}
-
-			name = string(sval)
-		}
-
-		switch name {
-		// t.PublishCid (cid.Cid) (struct)
-		case "PublishCid":
-
-			{
-
-				b, err := br.ReadByte()
-				if err != nil {
-					return err
-				}
-				if b != cbg.CborNull[0] {
-					if err := br.UnreadByte(); err != nil {
-						return err
-					}
-
-					c, err := cbg.ReadCid(br)
-					if err != nil {
-						return xerrors.Errorf("failed to read cid field t.PublishCid: %w", err)
-					}
-
-					t.PublishCid = &c
-				}
-
-			}
-			// t.DealID (abi.DealID) (uint64)
-		case "DealID":
-
-			{
-
-				maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
-				if err != nil {
-					return err
-				}
-				if maj != cbg.MajUnsignedInt {
-					return fmt.Errorf("wrong type for uint64 field")
-				}
-				t.DealID = abi.DealID(extra)
-
-			}
-			// t.DealProposal (market.DealProposal) (struct)
-		case "DealProposal":
-
-			{
-
-				b, err := br.ReadByte()
-				if err != nil {
-					return err
-				}
-				if b != cbg.CborNull[0] {
-					if err := br.UnreadByte(); err != nil {
-						return err
-					}
-					t.DealProposal = new(market.DealProposal)
-					if err := t.DealProposal.UnmarshalCBOR(br); err != nil {
-						return xerrors.Errorf("unmarshaling t.DealProposal pointer: %w", err)
-					}
-				}
-
-			}
-			// t.DealSchedule (api.DealSchedule) (struct)
-		case "DealSchedule":
-
-			{
-
-				if err := t.DealSchedule.UnmarshalCBOR(br); err != nil {
-					return xerrors.Errorf("unmarshaling t.DealSchedule: %w", err)
-				}
-
-			}
-			// t.KeepUnsealed (bool) (bool)
-		case "KeepUnsealed":
-
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
-			if err != nil {
-				return err
-			}
-			if maj != cbg.MajOther {
-				return fmt.Errorf("booleans must be major type 7")
-			}
-			switch extra {
-			case 20:
-				t.KeepUnsealed = false
-			case 21:
-				t.KeepUnsealed = true
-			default:
-				return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra)
-			}
-
-		default:
-			// Field doesn't exist on this type, so ignore it
-			cbg.ScanForLinks(r, func(cid.Cid) {})
-		}
-	}
-
-	return nil
-}
-func (t *DealSchedule) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{162}); err != nil {
-		return err
-	}
-
-	scratch := make([]byte, 9)
-
-	// t.StartEpoch (abi.ChainEpoch) (int64)
-	if len("StartEpoch") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"StartEpoch\" was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("StartEpoch"))); err != nil {
-		return err
-	}
-	if _, err := io.WriteString(w, string("StartEpoch")); err != nil {
-		return err
-	}
-
-	if t.StartEpoch >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.StartEpoch)); err != nil {
-			return err
-		}
-	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.StartEpoch-1)); err != nil {
-			return err
-		}
-	}
-
-	// t.EndEpoch (abi.ChainEpoch) (int64)
-	if len("EndEpoch") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"EndEpoch\" was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("EndEpoch"))); err != nil {
-		return err
-	}
-	if _, err := io.WriteString(w, string("EndEpoch")); err != nil {
-		return err
-	}
-
-	if t.EndEpoch >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.EndEpoch)); err != nil {
-			return err
-		}
-	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.EndEpoch-1)); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error {
-	*t = DealSchedule{}
-
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
-
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajMap {
-		return fmt.Errorf("cbor input should be of type map")
-	}
-
-	if extra > cbg.MaxLength {
-		return fmt.Errorf("DealSchedule: map struct too large (%d)", extra)
-	}
-
-	var name string
-	n := extra
-
-	for i := uint64(0); i < n; i++ {
-
-		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
-			if err != nil {
-				return err
-			}
-
-			name = string(sval)
-		}
-
-		switch name {
-		// t.StartEpoch (abi.ChainEpoch) (int64)
-		case "StartEpoch":
-			{
-				maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
-				var extraI int64
-				if err != nil {
-					return err
-				}
-				switch maj {
-				case cbg.MajUnsignedInt:
-					extraI = int64(extra)
-					if extraI < 0 {
-						return fmt.Errorf("int64 positive overflow")
-					}
-				case cbg.MajNegativeInt:
-					extraI = int64(extra)
-					if extraI < 0 {
-						return fmt.Errorf("int64 negative oveflow")
-					}
-					extraI = -1 - extraI
-				default:
-					return fmt.Errorf("wrong type for int64 field: %d", maj)
-				}
-
-				t.StartEpoch = abi.ChainEpoch(extraI)
-			}
-			// t.EndEpoch (abi.ChainEpoch) (int64)
-		case "EndEpoch":
-			{
-				maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
-				var extraI int64
-				if err != nil {
-					return err
-				}
-				switch maj {
-				case cbg.MajUnsignedInt:
-					extraI = int64(extra)
-					if extraI < 0 {
-						return fmt.Errorf("int64 positive overflow")
-					}
-				case cbg.MajNegativeInt:
-					extraI = int64(extra)
-					if extraI < 0 {
-						return fmt.Errorf("int64 negative oveflow")
-					}
-					extraI = -1 - extraI
-				default:
-					return fmt.Errorf("wrong type for int64 field: %d", maj)
-				}
-
-				t.EndEpoch = abi.ChainEpoch(extraI)
-			}
-
-		default:
-			// Field doesn't exist on this type, so ignore it
-			cbg.ScanForLinks(r, func(cid.Cid) {})
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
 		}
 	}
 

api/checkstatuscode_string.go

@@ -1,35 +0,0 @@
-// Code generated by "stringer -type=CheckStatusCode -trimprefix=CheckStatus"; DO NOT EDIT.
-
-package api
-
-import "strconv"
-
-func _() {
-	// An "invalid array index" compiler error signifies that the constant values have changed.
-	// Re-run the stringer command to generate them again.
-	var x [1]struct{}
-	_ = x[CheckStatusMessageSerialize-1]
-	_ = x[CheckStatusMessageSize-2]
-	_ = x[CheckStatusMessageValidity-3]
-	_ = x[CheckStatusMessageMinGas-4]
-	_ = x[CheckStatusMessageMinBaseFee-5]
-	_ = x[CheckStatusMessageBaseFee-6]
-	_ = x[CheckStatusMessageBaseFeeLowerBound-7]
-	_ = x[CheckStatusMessageBaseFeeUpperBound-8]
-	_ = x[CheckStatusMessageGetStateNonce-9]
-	_ = x[CheckStatusMessageNonce-10]
-	_ = x[CheckStatusMessageGetStateBalance-11]
-	_ = x[CheckStatusMessageBalance-12]
-}
-
-const _CheckStatusCode_name = "MessageSerializeMessageSizeMessageValidityMessageMinGasMessageMinBaseFeeMessageBaseFeeMessageBaseFeeLowerBoundMessageBaseFeeUpperBoundMessageGetStateNonceMessageNonceMessageGetStateBalanceMessageBalance"
-
-var _CheckStatusCode_index = [...]uint8{0, 16, 27, 42, 55, 72, 86, 110, 134, 154, 166, 188, 202}
-
-func (i CheckStatusCode) String() string {
-	i -= 1
-	if i < 0 || i >= CheckStatusCode(len(_CheckStatusCode_index)-1) {
-		return "CheckStatusCode(" + strconv.FormatInt(int64(i+1), 10) + ")"
-	}
-	return _CheckStatusCode_name[_CheckStatusCode_index[i]:_CheckStatusCode_index[i+1]]
-}

api/client/client.go

@@ -1,128 +1,59 @@
 package client
 
 import (
-	"context"
 	"net/http"
-	"net/url"
-	"path"
-	"time"
 
 	"github.com/filecoin-project/go-jsonrpc"
 
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/api/v0api"
-	"github.com/filecoin-project/lotus/api/v1api"
-	"github.com/filecoin-project/lotus/lib/rpcenc"
+	"github.com/filecoin-project/lotus/api/apistruct"
 )
 
-// NewCommonRPCV0 creates a new http jsonrpc client.
-func NewCommonRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.CommonNet, jsonrpc.ClientCloser, error) {
-	var res v0api.CommonNetStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
-		api.GetInternalStructs(&res), requestHeader)
-
-	return &res, closer, err
-}
-
-// NewFullNodeRPCV0 creates a new http jsonrpc client.
-func NewFullNodeRPCV0(ctx context.Context, addr string, requestHeader http.Header) (v0api.FullNode, jsonrpc.ClientCloser, error) {
-	var res v0api.FullNodeStruct
-
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
-		api.GetInternalStructs(&res), requestHeader)
-
-	return &res, closer, err
-}
-
-// NewFullNodeRPCV1 creates a new http jsonrpc client.
-func NewFullNodeRPCV1(ctx context.Context, addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
-	var res v1api.FullNodeStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
-		api.GetInternalStructs(&res), requestHeader)
-
-	return &res, closer, err
-}
-
-func getPushUrl(addr string) (string, error) {
-	pushUrl, err := url.Parse(addr)
-	if err != nil {
-		return "", err
-	}
-	switch pushUrl.Scheme {
-	case "ws":
-		pushUrl.Scheme = "http"
-	case "wss":
-		pushUrl.Scheme = "https"
-	}
-	///rpc/v0 -> /rpc/streams/v0/push
-
-	pushUrl.Path = path.Join(pushUrl.Path, "../streams/v0/push")
-	return pushUrl.String(), nil
-}
-
-// NewStorageMinerRPCV0 creates a new http jsonrpc client for miner
-func NewStorageMinerRPCV0(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (v0api.StorageMiner, jsonrpc.ClientCloser, error) {
-	pushUrl, err := getPushUrl(addr)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	var res v0api.StorageMinerStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
-		api.GetInternalStructs(&res), requestHeader,
-		append([]jsonrpc.Option{
-			rpcenc.ReaderParamEncoder(pushUrl),
-		}, opts...)...)
-
-	return &res, closer, err
-}
-
-func NewWorkerRPCV0(ctx context.Context, addr string, requestHeader http.Header) (v0api.Worker, jsonrpc.ClientCloser, error) {
-	pushUrl, err := getPushUrl(addr)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	var res api.WorkerStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
-		api.GetInternalStructs(&res),
+// NewCommonRPC creates a new http jsonrpc client.
+func NewCommonRPC(addr string, requestHeader http.Header) (api.Common, jsonrpc.ClientCloser, error) {
+	var res apistruct.CommonStruct
+	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
+		[]interface{}{
+			&res.Internal,
+		},
 		requestHeader,
-		rpcenc.ReaderParamEncoder(pushUrl),
-		jsonrpc.WithNoReconnect(),
-		jsonrpc.WithTimeout(30*time.Second),
 	)
 
 	return &res, closer, err
 }
 
-// NewGatewayRPCV1 creates a new http jsonrpc client for a gateway node.
-func NewGatewayRPCV1(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.Gateway, jsonrpc.ClientCloser, error) {
-	var res api.GatewayStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
-		api.GetInternalStructs(&res),
-		requestHeader,
-		opts...,
-	)
+// NewFullNodeRPC creates a new http jsonrpc client.
+func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
+	var res apistruct.FullNodeStruct
+	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
+		[]interface{}{
+			&res.CommonStruct.Internal,
+			&res.Internal,
+		}, requestHeader)
 
 	return &res, closer, err
 }
 
-// NewGatewayRPCV0 creates a new http jsonrpc client for a gateway node.
-func NewGatewayRPCV0(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (v0api.Gateway, jsonrpc.ClientCloser, error) {
-	var res v0api.GatewayStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
-		api.GetInternalStructs(&res),
+// NewStorageMinerRPC creates a new http jsonrpc client for storage miner
+func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMiner, jsonrpc.ClientCloser, error) {
+	var res apistruct.StorageMinerStruct
+	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
+		[]interface{}{
+			&res.CommonStruct.Internal,
+			&res.Internal,
+		},
 		requestHeader,
-		opts...,
 	)
 
 	return &res, closer, err
 }
 
-func NewWalletRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.Wallet, jsonrpc.ClientCloser, error) {
-	var res api.WalletStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
-		api.GetInternalStructs(&res),
+func NewWorkerRPC(addr string, requestHeader http.Header) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
+	var res apistruct.WorkerStruct
+	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
+		[]interface{}{
+			&res.Internal,
+		},
 		requestHeader,
 	)
 

api/docgen-openrpc/cmd/docgen_openrpc.go

@@ -1,74 +0,0 @@
-package main
-
-import (
-	"compress/gzip"
-	"encoding/json"
-	"io"
-	"log"
-	"os"
-
-	"github.com/filecoin-project/lotus/api/docgen"
-
-	docgen_openrpc "github.com/filecoin-project/lotus/api/docgen-openrpc"
-)
-
-/*
-main defines a small program that writes an OpenRPC document describing
-a Lotus API to stdout.
-
-If the first argument is "miner", the document will describe the StorageMiner API.
-If not (no, or any other args), the document will describe the Full API.
-
-Use:
-
-		go run ./api/openrpc/cmd ["api/api_full.go"|"api/api_storage.go"|"api/api_worker.go"] ["FullNode"|"StorageMiner"|"Worker"]
-
-	With gzip compression: a '-gzip' flag is made available as an optional third argument. Note that position matters.
-
-		go run ./api/openrpc/cmd ["api/api_full.go"|"api/api_storage.go"|"api/api_worker.go"] ["FullNode"|"StorageMiner"|"Worker"] -gzip
-
-*/
-
-func main() {
-	Comments, GroupDocs := docgen.ParseApiASTInfo(os.Args[1], os.Args[2], os.Args[3], os.Args[4])
-
-	doc := docgen_openrpc.NewLotusOpenRPCDocument(Comments, GroupDocs)
-
-	i, _, _ := docgen.GetAPIType(os.Args[2], os.Args[3])
-	doc.RegisterReceiverName("Filecoin", i)
-
-	out, err := doc.Discover()
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	var jsonOut []byte
-	var writer io.WriteCloser
-
-	// Use os.Args to handle a somewhat hacky flag for the gzip option.
-	// Could use flags package to handle this more cleanly, but that requires changes elsewhere
-	// the scope of which just isn't warranted by this one use case which will usually be run
-	// programmatically anyways.
-	if len(os.Args) > 5 && os.Args[5] == "-gzip" {
-		jsonOut, err = json.Marshal(out)
-		if err != nil {
-			log.Fatalln(err)
-		}
-		writer = gzip.NewWriter(os.Stdout)
-	} else {
-		jsonOut, err = json.MarshalIndent(out, "", "    ")
-		if err != nil {
-			log.Fatalln(err)
-		}
-		writer = os.Stdout
-	}
-
-	_, err = writer.Write(jsonOut)
-	if err != nil {
-		log.Fatalln(err)
-	}
-	err = writer.Close()
-	if err != nil {
-		log.Fatalln(err)
-	}
-}

api/docgen-openrpc/openrpc.go

@@ -1,161 +0,0 @@
-package docgenopenrpc
-
-import (
-	"encoding/json"
-	"go/ast"
-	"net"
-	"reflect"
-
-	"github.com/alecthomas/jsonschema"
-	go_openrpc_reflect "github.com/etclabscore/go-openrpc-reflect"
-	"github.com/filecoin-project/lotus/api/docgen"
-	"github.com/filecoin-project/lotus/build"
-	"github.com/ipfs/go-cid"
-	meta_schema "github.com/open-rpc/meta-schema"
-)
-
-// schemaDictEntry represents a type association passed to the jsonschema reflector.
-type schemaDictEntry struct {
-	example interface{}
-	rawJson string
-}
-
-const integerD = `{
-          "title": "number",
-          "type": "number",
-          "description": "Number is a number"
-        }`
-
-const cidCidD = `{"title": "Content Identifier", "type": "string", "description": "Cid represents a self-describing content addressed identifier. It is formed by a Version, a Codec (which indicates a multicodec-packed content type) and a Multihash."}`
-
-func OpenRPCSchemaTypeMapper(ty reflect.Type) *jsonschema.Type {
-	unmarshalJSONToJSONSchemaType := func(input string) *jsonschema.Type {
-		var js jsonschema.Type
-		err := json.Unmarshal([]byte(input), &js)
-		if err != nil {
-			panic(err)
-		}
-		return &js
-	}
-
-	if ty.Kind() == reflect.Ptr {
-		ty = ty.Elem()
-	}
-
-	if ty == reflect.TypeOf((*interface{})(nil)).Elem() {
-		return &jsonschema.Type{Type: "object", AdditionalProperties: []byte("true")}
-	}
-
-	// Second, handle other types.
-	// Use a slice instead of a map because it preserves order, as a logic safeguard/fallback.
-	dict := []schemaDictEntry{
-		{cid.Cid{}, cidCidD},
-	}
-
-	for _, d := range dict {
-		if reflect.TypeOf(d.example) == ty {
-			tt := unmarshalJSONToJSONSchemaType(d.rawJson)
-
-			return tt
-		}
-	}
-
-	// Handle primitive types in case there are generic cases
-	// specific to our services.
-	switch ty.Kind() {
-	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
-		// Return all integer types as the hex representation integer schemea.
-		ret := unmarshalJSONToJSONSchemaType(integerD)
-		return ret
-	case reflect.Uintptr:
-		return &jsonschema.Type{Type: "number", Title: "uintptr-title"}
-	case reflect.Struct:
-	case reflect.Map:
-	case reflect.Slice, reflect.Array:
-	case reflect.Float32, reflect.Float64:
-	case reflect.Bool:
-	case reflect.String:
-	case reflect.Ptr, reflect.Interface:
-	default:
-	}
-
-	return nil
-}
-
-// NewLotusOpenRPCDocument defines application-specific documentation and configuration for its OpenRPC document.
-func NewLotusOpenRPCDocument(Comments, GroupDocs map[string]string) *go_openrpc_reflect.Document {
-	d := &go_openrpc_reflect.Document{}
-
-	// Register "Meta" document fields.
-	// These include getters for
-	// - Servers object
-	// - Info object
-	// - ExternalDocs object
-	//
-	// These objects represent server-specific data that cannot be
-	// reflected.
-	d.WithMeta(&go_openrpc_reflect.MetaT{
-		GetServersFn: func() func(listeners []net.Listener) (*meta_schema.Servers, error) {
-			return func(listeners []net.Listener) (*meta_schema.Servers, error) {
-				return nil, nil
-			}
-		},
-		GetInfoFn: func() (info *meta_schema.InfoObject) {
-			info = &meta_schema.InfoObject{}
-			title := "Lotus RPC API"
-			info.Title = (*meta_schema.InfoObjectProperties)(&title)
-
-			version := build.BuildVersion
-			info.Version = (*meta_schema.InfoObjectVersion)(&version)
-			return info
-		},
-		GetExternalDocsFn: func() (exdocs *meta_schema.ExternalDocumentationObject) {
-			return nil // FIXME
-		},
-	})
-
-	// Use a provided Ethereum default configuration as a base.
-	appReflector := &go_openrpc_reflect.EthereumReflectorT{}
-
-	// Install overrides for the json schema->type map fn used by the jsonschema reflect package.
-	appReflector.FnSchemaTypeMap = func() func(ty reflect.Type) *jsonschema.Type {
-		return OpenRPCSchemaTypeMapper
-	}
-
-	appReflector.FnIsMethodEligible = func(m reflect.Method) bool {
-		for i := 0; i < m.Func.Type().NumOut(); i++ {
-			if m.Func.Type().Out(i).Kind() == reflect.Chan {
-				return false
-			}
-		}
-		return go_openrpc_reflect.EthereumReflector.IsMethodEligible(m)
-	}
-	appReflector.FnGetMethodName = func(moduleName string, r reflect.Value, m reflect.Method, funcDecl *ast.FuncDecl) (string, error) {
-		if m.Name == "ID" {
-			return moduleName + "_ID", nil
-		}
-		if moduleName == "rpc" && m.Name == "Discover" {
-			return "rpc.discover", nil
-		}
-
-		return moduleName + "." + m.Name, nil
-	}
-
-	appReflector.FnGetMethodSummary = func(r reflect.Value, m reflect.Method, funcDecl *ast.FuncDecl) (string, error) {
-		if v, ok := Comments[m.Name]; ok {
-			return v, nil
-		}
-		return "", nil // noComment
-	}
-
-	appReflector.FnSchemaExamples = func(ty reflect.Type) (examples *meta_schema.Examples, err error) {
-		v := docgen.ExampleValue("unknown", ty, ty) // This isn't ideal, but seems to work well enough.
-		return &meta_schema.Examples{
-			meta_schema.AlwaysTrue(v),
-		}, nil
-	}
-
-	// Finally, register the configured reflector to the document.
-	d.WithReflector(appReflector)
-	return d
-}

api/docgen/cmd/docgen.go

@@ -1,121 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-	"reflect"
-	"sort"
-	"strings"
-
-	"github.com/filecoin-project/lotus/api/docgen"
-)
-
-func main() {
-	comments, groupComments := docgen.ParseApiASTInfo(os.Args[1], os.Args[2], os.Args[3], os.Args[4])
-
-	groups := make(map[string]*docgen.MethodGroup)
-
-	_, t, permStruct := docgen.GetAPIType(os.Args[2], os.Args[3])
-
-	for i := 0; i < t.NumMethod(); i++ {
-		m := t.Method(i)
-
-		groupName := docgen.MethodGroupFromName(m.Name)
-
-		g, ok := groups[groupName]
-		if !ok {
-			g = new(docgen.MethodGroup)
-			g.Header = groupComments[groupName]
-			g.GroupName = groupName
-			groups[groupName] = g
-		}
-
-		var args []interface{}
-		ft := m.Func.Type()
-		for j := 2; j < ft.NumIn(); j++ {
-			inp := ft.In(j)
-			args = append(args, docgen.ExampleValue(m.Name, inp, nil))
-		}
-
-		v, err := json.MarshalIndent(args, "", "  ")
-		if err != nil {
-			panic(err)
-		}
-
-		outv := docgen.ExampleValue(m.Name, ft.Out(0), nil)
-
-		ov, err := json.MarshalIndent(outv, "", "  ")
-		if err != nil {
-			panic(err)
-		}
-
-		g.Methods = append(g.Methods, &docgen.Method{
-			Name:            m.Name,
-			Comment:         comments[m.Name],
-			InputExample:    string(v),
-			ResponseExample: string(ov),
-		})
-	}
-
-	var groupslice []*docgen.MethodGroup
-	for _, g := range groups {
-		groupslice = append(groupslice, g)
-	}
-
-	sort.Slice(groupslice, func(i, j int) bool {
-		return groupslice[i].GroupName < groupslice[j].GroupName
-	})
-
-	fmt.Printf("# Groups\n")
-
-	for _, g := range groupslice {
-		fmt.Printf("* [%s](#%s)\n", g.GroupName, g.GroupName)
-		for _, method := range g.Methods {
-			fmt.Printf("  * [%s](#%s)\n", method.Name, method.Name)
-		}
-	}
-
-	for _, g := range groupslice {
-		g := g
-		fmt.Printf("## %s\n", g.GroupName)
-		fmt.Printf("%s\n\n", g.Header)
-
-		sort.Slice(g.Methods, func(i, j int) bool {
-			return g.Methods[i].Name < g.Methods[j].Name
-		})
-
-		for _, m := range g.Methods {
-			fmt.Printf("### %s\n", m.Name)
-			fmt.Printf("%s\n\n", m.Comment)
-
-			var meth reflect.StructField
-			var ok bool
-			for _, ps := range permStruct {
-				meth, ok = ps.FieldByName(m.Name)
-				if ok {
-					break
-				}
-			}
-			if !ok {
-				panic("no perms for method: " + m.Name)
-			}
-
-			perms := meth.Tag.Get("perm")
-
-			fmt.Printf("Perms: %s\n\n", perms)
-
-			if strings.Count(m.InputExample, "\n") > 0 {
-				fmt.Printf("Inputs:\n```json\n%s\n```\n\n", m.InputExample)
-			} else {
-				fmt.Printf("Inputs: `%s`\n\n", m.InputExample)
-			}
-
-			if strings.Count(m.ResponseExample, "\n") > 0 {
-				fmt.Printf("Response:\n```json\n%s\n```\n\n", m.ResponseExample)
-			} else {
-				fmt.Printf("Response: `%s`\n\n", m.ResponseExample)
-			}
-		}
-	}
-}

api/docgen/docgen.go

@@ -1,58 +1,40 @@
-package docgen
+package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"go/ast"
 	"go/parser"
 	"go/token"
-	"path/filepath"
 	"reflect"
+	"sort"
 	"strings"
 	"time"
 	"unicode"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
-	"github.com/filecoin-project/go-multistore"
-	"github.com/filecoin-project/lotus/node/repo/importmgr"
-	"github.com/google/uuid"
-	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-filestore"
-	"github.com/libp2p/go-libp2p-core/metrics"
-	"github.com/libp2p/go-libp2p-core/network"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"github.com/libp2p/go-libp2p-core/protocol"
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
-	"github.com/multiformats/go-multiaddr"
-
-	datatransfer "github.com/filecoin-project/go-data-transfer"
-	filestore2 "github.com/filecoin-project/go-fil-markets/filestore"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-jsonrpc/auth"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/crypto"
-	"github.com/filecoin-project/go-state-types/exitcode"
-
 	"github.com/filecoin-project/lotus/api"
-	apitypes "github.com/filecoin-project/lotus/api/types"
-	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
+	"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-filestore"
+	"github.com/libp2p/go-libp2p-core/network"
+	peer "github.com/libp2p/go-libp2p-peer"
+	"github.com/multiformats/go-multiaddr"
 )
 
 var ExampleValues = map[reflect.Type]interface{}{
-	reflect.TypeOf(api.MinerSubsystem(0)): api.MinerSubsystem(1),
-	reflect.TypeOf(auth.Permission("")):   auth.Permission("write"),
-	reflect.TypeOf(""):                    "string value",
-	reflect.TypeOf(uint64(42)):            uint64(42),
-	reflect.TypeOf(byte(7)):               byte(7),
-	reflect.TypeOf([]byte{}):              []byte("byte array"),
+	reflect.TypeOf(auth.Permission("")): auth.Permission("write"),
+	reflect.TypeOf(""):                  "string value",
+	reflect.TypeOf(uint64(42)):          uint64(42),
+	reflect.TypeOf(byte(7)):             byte(7),
+	reflect.TypeOf([]byte{}):            []byte("byte array"),
 }
 
 func addExample(v interface{}) {
@@ -83,26 +65,17 @@ func init() {
 
 	ExampleValues[reflect.TypeOf(addr)] = addr
 
-	pid, err := peer.Decode("12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf")
+	pid, err := peer.IDB58Decode("12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf")
 	if err != nil {
 		panic(err)
 	}
 	addExample(pid)
-	addExample(&pid)
-
-	multistoreIDExample := multistore.StoreID(50)
-	storeIDExample := importmgr.ImportID(50)
 
 	addExample(bitfield.NewFromSet([]uint64{5}))
-	addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1)
-	addExample(abi.RegisteredPoStProof_StackedDrgWindow32GiBV1)
+	addExample(abi.RegisteredProof_StackedDRG32GiBPoSt)
 	addExample(abi.ChainEpoch(10101))
 	addExample(crypto.SigTypeBLS)
-	addExample(types.KTBLS)
 	addExample(int64(9))
-	addExample(12.3)
-	addExample(123)
-	addExample(uintptr(0))
 	addExample(abi.MethodNum(1))
 	addExample(exitcode.ExitCode(0))
 	addExample(crypto.DomainSeparationTag_ElectionProofProduction)
@@ -117,57 +90,21 @@ func init() {
 	addExample(network.Connected)
 	addExample(dtypes.NetworkName("lotus"))
 	addExample(api.SyncStateStage(1))
-	addExample(api.FullAPIVersion1)
+	addExample(build.APIVersion)
 	addExample(api.PCHInbound)
 	addExample(time.Minute)
-	addExample(datatransfer.TransferID(3))
-	addExample(datatransfer.Ongoing)
-	addExample(storeIDExample)
-	addExample(&storeIDExample)
-	addExample(multistoreIDExample)
-	addExample(&multistoreIDExample)
-	addExample(retrievalmarket.ClientEventDealAccepted)
-	addExample(retrievalmarket.DealStatusNew)
-	addExample(network.ReachabilityPublic)
-	addExample(build.NewestNetworkVersion)
-	addExample(map[string]int{"name": 42})
-	addExample(map[string]time.Time{"name": time.Unix(1615243938, 0).UTC()})
-	addExample(&types.ExecutionTrace{
-		Msg:    ExampleValue("init", reflect.TypeOf(&types.Message{}), nil).(*types.Message),
-		MsgRct: ExampleValue("init", reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt),
+	addExample(&types.ExecutionResult{
+		Msg:    exampleValue(reflect.TypeOf(&types.Message{})).(*types.Message),
+		MsgRct: exampleValue(reflect.TypeOf(&types.MessageReceipt{})).(*types.MessageReceipt),
 	})
 	addExample(map[string]types.Actor{
-		"t01236": ExampleValue("init", reflect.TypeOf(types.Actor{}), nil).(types.Actor),
+		"t01236": exampleValue(reflect.TypeOf(types.Actor{})).(types.Actor),
 	})
 	addExample(map[string]api.MarketDeal{
-		"t026363": ExampleValue("init", reflect.TypeOf(api.MarketDeal{}), nil).(api.MarketDeal),
+		"t026363": exampleValue(reflect.TypeOf(api.MarketDeal{})).(api.MarketDeal),
 	})
 	addExample(map[string]api.MarketBalance{
-		"t026363": ExampleValue("init", reflect.TypeOf(api.MarketBalance{}), nil).(api.MarketBalance),
-	})
-	addExample(map[string]*pubsub.TopicScoreSnapshot{
-		"/blocks": {
-			TimeInMesh:               time.Minute,
-			FirstMessageDeliveries:   122,
-			MeshMessageDeliveries:    1234,
-			InvalidMessageDeliveries: 3,
-		},
-	})
-	addExample(map[string]metrics.Stats{
-		"12D3KooWSXmXLJmBR1M7i9RW9GQPNUhZSzXKzxDHWtAgNuJAbyEJ": {
-			RateIn:   100,
-			RateOut:  50,
-			TotalIn:  174000,
-			TotalOut: 12500,
-		},
-	})
-	addExample(map[protocol.ID]metrics.Stats{
-		"/fil/hello/1.0.0": {
-			RateIn:   100,
-			RateOut:  50,
-			TotalIn:  174000,
-			TotalOut: 12500,
-		},
+		"t026363": exampleValue(reflect.TypeOf(api.MarketBalance{})).(api.MarketBalance),
 	})
 
 	maddr, err := multiaddr.NewMultiaddr("/ip4/52.36.61.156/tcp/1347/p2p/12D3KooWFETiESTf1v4PGUvtnxMAcEFMzLZbJGg4tjWfGEimYior")
@@ -178,145 +115,9 @@ func init() {
 	// because reflect.TypeOf(maddr) returns the concrete type...
 	ExampleValues[reflect.TypeOf(struct{ A multiaddr.Multiaddr }{}).Field(0).Type] = maddr
 
-	// miner specific
-	addExample(filestore2.Path(".lotusminer/fstmp123"))
-	si := uint64(12)
-	addExample(&si)
-	addExample(retrievalmarket.DealID(5))
-	addExample(abi.ActorID(1000))
-	addExample(map[string][]api.SealedRef{
-		"98000": {
-			api.SealedRef{
-				SectorID: 100,
-				Offset:   10 << 20,
-				Size:     1 << 20,
-			},
-		},
-	})
-	addExample(api.SectorState(sealing.Proving))
-	addExample(stores.ID("76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8"))
-	addExample(storiface.FTUnsealed)
-	addExample(storiface.PathSealing)
-	addExample(map[stores.ID][]stores.Decl{
-		"76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8": {
-			{
-				SectorID:       abi.SectorID{Miner: 1000, Number: 100},
-				SectorFileType: storiface.FTSealed,
-			},
-		},
-	})
-	addExample(map[stores.ID]string{
-		"76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8": "/data/path",
-	})
-	addExample(map[uuid.UUID][]storiface.WorkerJob{
-		uuid.MustParse("ef8d99a2-6865-4189-8ffa-9fef0f806eee"): {
-			{
-				ID: storiface.CallID{
-					Sector: abi.SectorID{Miner: 1000, Number: 100},
-					ID:     uuid.MustParse("76081ba0-61bd-45a5-bc08-af05f1c26e5d"),
-				},
-				Sector:   abi.SectorID{Miner: 1000, Number: 100},
-				Task:     sealtasks.TTPreCommit2,
-				RunWait:  0,
-				Start:    time.Unix(1605172927, 0).UTC(),
-				Hostname: "host",
-			},
-		},
-	})
-	addExample(map[uuid.UUID]storiface.WorkerStats{
-		uuid.MustParse("ef8d99a2-6865-4189-8ffa-9fef0f806eee"): {
-			Info: storiface.WorkerInfo{
-				Hostname: "host",
-				Resources: storiface.WorkerResources{
-					MemPhysical: 256 << 30,
-					MemSwap:     120 << 30,
-					MemReserved: 2 << 30,
-					CPUs:        64,
-					GPUs:        []string{"aGPU 1337"},
-				},
-			},
-			Enabled:    true,
-			MemUsedMin: 0,
-			MemUsedMax: 0,
-			GpuUsed:    false,
-			CpuUse:     0,
-		},
-	})
-	addExample(storiface.ErrorCode(0))
-	addExample(map[abi.SectorNumber]string{
-		123: "can't acquire read lock",
-	})
-	addExample(map[api.SectorState]int{
-		api.SectorState(sealing.Proving): 120,
-	})
-	addExample([]abi.SectorNumber{123, 124})
-
-	// worker specific
-	addExample(storiface.AcquireMove)
-	addExample(storiface.UnpaddedByteIndex(abi.PaddedPieceSize(1 << 20).Unpadded()))
-	addExample(map[sealtasks.TaskType]struct{}{
-		sealtasks.TTPreCommit2: {},
-	})
-	addExample(sealtasks.TTCommit2)
-	addExample(apitypes.OpenRPCDocument{
-		"openrpc": "1.2.6",
-		"info": map[string]interface{}{
-			"title":   "Lotus RPC API",
-			"version": "1.2.1/generated=2020-11-22T08:22:42-06:00",
-		},
-		"methods": []interface{}{}},
-	)
-
-	addExample(api.CheckStatusCode(0))
-	addExample(map[string]interface{}{"abc": 123})
-	addExample(api.MinerSubsystems{
-		api.SubsystemMining,
-		api.SubsystemSealing,
-		api.SubsystemSectorStorage,
-		api.SubsystemMarkets,
-	})
 }
 
-func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) {
-
-	switch pkg {
-	case "api": // latest
-		switch name {
-		case "FullNode":
-			i = &api.FullNodeStruct{}
-			t = reflect.TypeOf(new(struct{ api.FullNode })).Elem()
-			permStruct = append(permStruct, reflect.TypeOf(api.FullNodeStruct{}.Internal))
-			permStruct = append(permStruct, reflect.TypeOf(api.CommonStruct{}.Internal))
-			permStruct = append(permStruct, reflect.TypeOf(api.NetStruct{}.Internal))
-		case "StorageMiner":
-			i = &api.StorageMinerStruct{}
-			t = reflect.TypeOf(new(struct{ api.StorageMiner })).Elem()
-			permStruct = append(permStruct, reflect.TypeOf(api.StorageMinerStruct{}.Internal))
-			permStruct = append(permStruct, reflect.TypeOf(api.CommonStruct{}.Internal))
-			permStruct = append(permStruct, reflect.TypeOf(api.NetStruct{}.Internal))
-		case "Worker":
-			i = &api.WorkerStruct{}
-			t = reflect.TypeOf(new(struct{ api.Worker })).Elem()
-			permStruct = append(permStruct, reflect.TypeOf(api.WorkerStruct{}.Internal))
-		default:
-			panic("unknown type")
-		}
-	case "v0api":
-		switch name {
-		case "FullNode":
-			i = v0api.FullNodeStruct{}
-			t = reflect.TypeOf(new(struct{ v0api.FullNode })).Elem()
-			permStruct = append(permStruct, reflect.TypeOf(v0api.FullNodeStruct{}.Internal))
-			permStruct = append(permStruct, reflect.TypeOf(v0api.CommonStruct{}.Internal))
-			permStruct = append(permStruct, reflect.TypeOf(v0api.NetStruct{}.Internal))
-		default:
-			panic("unknown type")
-		}
-	}
-	return
-}
-
-func ExampleValue(method string, t, parent reflect.Type) interface{} {
+func exampleValue(t reflect.Type) interface{} {
 	v, ok := ExampleValues[t]
 	if ok {
 		return v
@@ -325,25 +126,25 @@ func ExampleValue(method string, t, parent reflect.Type) interface{} {
 	switch t.Kind() {
 	case reflect.Slice:
 		out := reflect.New(t).Elem()
-		reflect.Append(out, reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
+		reflect.Append(out, reflect.ValueOf(exampleValue(t.Elem())))
 		return out.Interface()
 	case reflect.Chan:
-		return ExampleValue(method, t.Elem(), nil)
+		return exampleValue(t.Elem())
 	case reflect.Struct:
-		es := exampleStruct(method, t, parent)
+		es := exampleStruct(t)
 		v := reflect.ValueOf(es).Elem().Interface()
 		ExampleValues[t] = v
 		return v
 	case reflect.Array:
 		out := reflect.New(t).Elem()
 		for i := 0; i < t.Len(); i++ {
-			out.Index(i).Set(reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
+			out.Index(i).Set(reflect.ValueOf(exampleValue(t.Elem())))
 		}
 		return out.Interface()
 
 	case reflect.Ptr:
 		if t.Elem().Kind() == reflect.Struct {
-			es := exampleStruct(method, t.Elem(), t)
+			es := exampleStruct(t.Elem())
 			//ExampleValues[t] = es
 			return es
 		}
@@ -351,18 +152,15 @@ func ExampleValue(method string, t, parent reflect.Type) interface{} {
 		return struct{}{}
 	}
 
-	panic(fmt.Sprintf("No example value for type: %s (method '%s')", t, method))
+	panic(fmt.Sprintf("No example value for type: %s", t))
 }
 
-func exampleStruct(method string, t, parent reflect.Type) interface{} {
+func exampleStruct(t reflect.Type) interface{} {
 	ns := reflect.New(t)
 	for i := 0; i < t.NumField(); i++ {
 		f := t.Field(i)
-		if f.Type == parent {
-			continue
-		}
 		if strings.Title(f.Name) == f.Name {
-			ns.Elem().Field(i).Set(reflect.ValueOf(ExampleValue(method, f.Type, t)))
+			ns.Elem().Field(i).Set(reflect.ValueOf(exampleValue(f.Type)))
 		}
 	}
 
@@ -370,7 +168,6 @@ func exampleStruct(method string, t, parent reflect.Type) interface{} {
 }
 
 type Visitor struct {
-	Root    string
 	Methods map[string]ast.Node
 }
 
@@ -380,7 +177,7 @@ func (v *Visitor) Visit(node ast.Node) ast.Visitor {
 		return v
 	}
 
-	if st.Name.Name != v.Root {
+	if st.Name.Name != "FullNode" {
 		return nil
 	}
 
@@ -394,43 +191,32 @@ func (v *Visitor) Visit(node ast.Node) ast.Visitor {
 	return v
 }
 
-const NoComment = "There are not yet any comments for this method."
+const noComment = "There are not yet any comments for this method."
 
-func ParseApiASTInfo(apiFile, iface, pkg, dir string) (comments map[string]string, groupDocs map[string]string) { //nolint:golint
+func parseApiASTInfo() (map[string]string, map[string]string) { //nolint:golint
 	fset := token.NewFileSet()
-	apiDir, err := filepath.Abs(dir)
-	if err != nil {
-		fmt.Println("./api filepath absolute error: ", err)
-		return
-	}
-	apiFile, err = filepath.Abs(apiFile)
-	if err != nil {
-		fmt.Println("filepath absolute error: ", err, "file:", apiFile)
-		return
-	}
-	pkgs, err := parser.ParseDir(fset, apiDir, nil, parser.AllErrors|parser.ParseComments)
+	pkgs, err := parser.ParseDir(fset, "./api", nil, parser.AllErrors|parser.ParseComments)
 	if err != nil {
 		fmt.Println("parse error: ", err)
-		return
 	}
 
-	ap := pkgs[pkg]
+	ap := pkgs["api"]
 
-	f := ap.Files[apiFile]
+	f := ap.Files["api/api_full.go"]
 
 	cmap := ast.NewCommentMap(fset, f, f.Comments)
 
-	v := &Visitor{iface, make(map[string]ast.Node)}
-	ast.Walk(v, ap)
+	v := &Visitor{make(map[string]ast.Node)}
+	ast.Walk(v, pkgs["api"])
 
-	comments = make(map[string]string)
-	groupDocs = make(map[string]string)
+	groupDocs := make(map[string]string)
+	out := make(map[string]string)
 	for mn, node := range v.Methods {
-		filteredComments := cmap.Filter(node).Comments()
-		if len(filteredComments) == 0 {
-			comments[mn] = NoComment
+		cs := cmap.Filter(node).Comments()
+		if len(cs) == 0 {
+			out[mn] = noComment
 		} else {
-			for _, c := range filteredComments {
+			for _, c := range cs {
 				if strings.HasPrefix(c.Text(), "MethodGroup:") {
 					parts := strings.Split(c.Text(), "\n")
 					groupName := strings.TrimSpace(parts[0][12:])
@@ -441,19 +227,15 @@ func ParseApiASTInfo(apiFile, iface, pkg, dir string) (comments map[string]strin
 				}
 			}
 
-			l := len(filteredComments) - 1
-			if len(filteredComments) > 1 {
-				l = len(filteredComments) - 2
-			}
-			last := filteredComments[l].Text()
+			last := cs[len(cs)-1].Text()
 			if !strings.HasPrefix(last, "MethodGroup:") {
-				comments[mn] = last
+				out[mn] = last
 			} else {
-				comments[mn] = NoComment
+				out[mn] = noComment
 			}
 		}
 	}
-	return comments, groupDocs
+	return out, groupDocs
 }
 
 type MethodGroup struct {
@@ -469,7 +251,7 @@ type Method struct {
 	ResponseExample string
 }
 
-func MethodGroupFromName(mn string) string {
+func methodGroupFromName(mn string) string {
 	i := strings.IndexFunc(mn[1:], func(r rune) bool {
 		return unicode.IsUpper(r)
 	})
@@ -478,3 +260,79 @@ func MethodGroupFromName(mn string) string {
 	}
 	return mn[:i+1]
 }
+
+func main() {
+
+	comments, groupComments := parseApiASTInfo()
+
+	groups := make(map[string]*MethodGroup)
+
+	var api struct{ api.FullNode }
+	t := reflect.TypeOf(api)
+	for i := 0; i < t.NumMethod(); i++ {
+		m := t.Method(i)
+
+		groupName := methodGroupFromName(m.Name)
+
+		g, ok := groups[groupName]
+		if !ok {
+			g = new(MethodGroup)
+			g.Header = groupComments[groupName]
+			g.GroupName = groupName
+			groups[groupName] = g
+		}
+
+		var args []interface{}
+		ft := m.Func.Type()
+		for j := 2; j < ft.NumIn(); j++ {
+			inp := ft.In(j)
+			args = append(args, exampleValue(inp))
+		}
+
+		v, err := json.Marshal(args)
+		if err != nil {
+			panic(err)
+		}
+
+		outv := exampleValue(ft.Out(0))
+
+		ov, err := json.Marshal(outv)
+		if err != nil {
+			panic(err)
+		}
+
+		g.Methods = append(g.Methods, &Method{
+			Name:            m.Name,
+			Comment:         comments[m.Name],
+			InputExample:    string(v),
+			ResponseExample: string(ov),
+		})
+	}
+
+	var groupslice []*MethodGroup
+	for _, g := range groups {
+		groupslice = append(groupslice, g)
+	}
+
+	sort.Slice(groupslice, func(i, j int) bool {
+		return groupslice[i].GroupName < groupslice[j].GroupName
+	})
+
+	for _, g := range groupslice {
+		g := g
+		fmt.Printf("## %s\n", g.GroupName)
+		fmt.Printf("%s\n\n", g.Header)
+
+		sort.Slice(g.Methods, func(i, j int) bool {
+			return g.Methods[i].Name < g.Methods[j].Name
+		})
+
+		for _, m := range g.Methods {
+			fmt.Printf("### %s\n", m.Name)
+			fmt.Printf("%s\n\n", m.Comment)
+
+			fmt.Printf("Inputs: `%s`\n\n", m.InputExample)
+			fmt.Printf("Response: `%s`\n\n", m.ResponseExample)
+		}
+	}
+}

api/miner_subsystems.go

@@ -1,79 +0,0 @@
-package api
-
-import (
-	"encoding/json"
-)
-
-// MinerSubsystem represents a miner subsystem. Int and string values are not
-// guaranteed to be stable over time  is not
-// guaranteed to be stable over time.
-type MinerSubsystem int
-
-const (
-	// SubsystemUnknown is a placeholder for the zero value. It should never
-	// be used.
-	SubsystemUnknown MinerSubsystem = iota
-	// SubsystemMarkets signifies the storage and retrieval
-	// deal-making subsystem.
-	SubsystemMarkets
-	// SubsystemMining signifies the mining subsystem.
-	SubsystemMining
-	// SubsystemSealing signifies the sealing subsystem.
-	SubsystemSealing
-	// SubsystemSectorStorage signifies the sector storage subsystem.
-	SubsystemSectorStorage
-)
-
-var MinerSubsystemToString = map[MinerSubsystem]string{
-	SubsystemUnknown:       "Unknown",
-	SubsystemMarkets:       "Markets",
-	SubsystemMining:        "Mining",
-	SubsystemSealing:       "Sealing",
-	SubsystemSectorStorage: "SectorStorage",
-}
-
-var MinerSubsystemToID = map[string]MinerSubsystem{
-	"Unknown":       SubsystemUnknown,
-	"Markets":       SubsystemMarkets,
-	"Mining":        SubsystemMining,
-	"Sealing":       SubsystemSealing,
-	"SectorStorage": SubsystemSectorStorage,
-}
-
-func (ms MinerSubsystem) MarshalJSON() ([]byte, error) {
-	return json.Marshal(MinerSubsystemToString[ms])
-}
-
-func (ms *MinerSubsystem) UnmarshalJSON(b []byte) error {
-	var j string
-	err := json.Unmarshal(b, &j)
-	if err != nil {
-		return err
-	}
-	s, ok := MinerSubsystemToID[j]
-	if !ok {
-		*ms = SubsystemUnknown
-	} else {
-		*ms = s
-	}
-	return nil
-}
-
-type MinerSubsystems []MinerSubsystem
-
-func (ms MinerSubsystems) Has(entry MinerSubsystem) bool {
-	for _, v := range ms {
-		if v == entry {
-			return true
-		}
-	}
-	return false
-}
-
-func (ms MinerSubsystem) String() string {
-	s, ok := MinerSubsystemToString[ms]
-	if !ok {
-		return MinerSubsystemToString[SubsystemUnknown]
-	}
-	return s
-}

api/permissioned.go

@@ -1,48 +0,0 @@
-package api
-
-import (
-	"github.com/filecoin-project/go-jsonrpc/auth"
-)
-
-const (
-	// When changing these, update docs/API.md too
-
-	PermRead  auth.Permission = "read" // default
-	PermWrite auth.Permission = "write"
-	PermSign  auth.Permission = "sign"  // Use wallet keys for signing
-	PermAdmin auth.Permission = "admin" // Manage permissions
-)
-
-var AllPermissions = []auth.Permission{PermRead, PermWrite, PermSign, PermAdmin}
-var DefaultPerms = []auth.Permission{PermRead}
-
-func permissionedProxies(in, out interface{}) {
-	outs := GetInternalStructs(out)
-	for _, o := range outs {
-		auth.PermissionedProxy(AllPermissions, DefaultPerms, in, o)
-	}
-}
-
-func PermissionedStorMinerAPI(a StorageMiner) StorageMiner {
-	var out StorageMinerStruct
-	permissionedProxies(a, &out)
-	return &out
-}
-
-func PermissionedFullAPI(a FullNode) FullNode {
-	var out FullNodeStruct
-	permissionedProxies(a, &out)
-	return &out
-}
-
-func PermissionedWorkerAPI(a Worker) Worker {
-	var out WorkerStruct
-	permissionedProxies(a, &out)
-	return &out
-}
-
-func PermissionedWalletAPI(a Wallet) Wallet {
-	var out WalletStruct
-	permissionedProxies(a, &out)
-	return &out
-}

api/proxy_util.go

@@ -1,30 +0,0 @@
-package api
-
-import "reflect"
-
-var _internalField = "Internal"
-
-// GetInternalStructs extracts all pointers to 'Internal' sub-structs from the provided pointer to a proxy struct
-func GetInternalStructs(in interface{}) []interface{} {
-	return getInternalStructs(reflect.ValueOf(in).Elem())
-}
-
-func getInternalStructs(rv reflect.Value) []interface{} {
-	var out []interface{}
-
-	internal := rv.FieldByName(_internalField)
-	ii := internal.Addr().Interface()
-	out = append(out, ii)
-
-	for i := 0; i < rv.NumField(); i++ {
-		if rv.Type().Field(i).Name == _internalField {
-			continue
-		}
-
-		sub := getInternalStructs(rv.Field(i))
-
-		out = append(out, sub...)
-	}
-
-	return out
-}

api/proxy_util_test.go

@@ -1,62 +0,0 @@
-package api
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-type StrA struct {
-	StrB
-
-	Internal struct {
-		A int
-	}
-}
-
-type StrB struct {
-	Internal struct {
-		B int
-	}
-}
-
-type StrC struct {
-	Internal struct {
-		Internal struct {
-			C int
-		}
-	}
-}
-
-func TestGetInternalStructs(t *testing.T) {
-	var proxy StrA
-
-	sts := GetInternalStructs(&proxy)
-	require.Len(t, sts, 2)
-
-	sa := sts[0].(*struct{ A int })
-	sa.A = 3
-	sb := sts[1].(*struct{ B int })
-	sb.B = 4
-
-	require.Equal(t, 3, proxy.Internal.A)
-	require.Equal(t, 4, proxy.StrB.Internal.B)
-}
-
-func TestNestedInternalStructs(t *testing.T) {
-	var proxy StrC
-
-	// check that only the top-level internal struct gets picked up
-
-	sts := GetInternalStructs(&proxy)
-	require.Len(t, sts, 1)
-
-	sa := sts[0].(*struct {
-		Internal struct {
-			C int
-		}
-	})
-	sa.Internal.C = 5
-
-	require.Equal(t, 5, proxy.Internal.Internal.C)
-}

api/test/deals.go

@@ -0,0 +1,254 @@
+package test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/ipfs/go-cid"
+
+	files "github.com/ipfs/go-ipfs-files"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/ipld/go-car"
+
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
+	dag "github.com/ipfs/go-merkledag"
+	dstest "github.com/ipfs/go-merkledag/test"
+	unixfile "github.com/ipfs/go-unixfs/file"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/node/impl"
+	ipld "github.com/ipfs/go-ipld-format"
+)
+
+func init() {
+	logging.SetAllLoggers(logging.LevelInfo)
+	build.InsecurePoStValidation = true
+}
+
+func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport bool) {
+	_ = os.Setenv("BELLMAN_NO_GPU", "1")
+
+	ctx := context.Background()
+	n, sn := b(t, 1, oneMiner)
+	client := n[0].FullNode.(*impl.FullNodeAPI)
+	miner := sn[0]
+
+	addrinfo, err := client.NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := miner.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Second)
+
+	mine := true
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for mine {
+			time.Sleep(blocktime)
+			if err := sn[0].MineOne(ctx, func(bool) {}); err != nil {
+				t.Error(err)
+			}
+		}
+	}()
+
+	makeDeal(t, ctx, 6, client, miner, carExport)
+
+	mine = false
+	fmt.Println("shutting down mining")
+	<-done
+}
+
+func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
+	_ = os.Setenv("BELLMAN_NO_GPU", "1")
+
+	ctx := context.Background()
+	n, sn := b(t, 1, oneMiner)
+	client := n[0].FullNode.(*impl.FullNodeAPI)
+	miner := sn[0]
+
+	addrinfo, err := client.NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := miner.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Second)
+
+	mine := true
+	done := make(chan struct{})
+
+	go func() {
+		defer close(done)
+		for mine {
+			time.Sleep(blocktime)
+			if err := sn[0].MineOne(ctx, func(bool) {}); err != nil {
+				t.Error(err)
+			}
+		}
+	}()
+
+	makeDeal(t, ctx, 6, client, miner, false)
+	makeDeal(t, ctx, 7, client, miner, false)
+
+	mine = false
+	fmt.Println("shutting down mining")
+	<-done
+}
+
+func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNodeAPI, miner TestStorageNode, carExport bool) {
+	data := make([]byte, 1600)
+	rand.New(rand.NewSource(int64(rseed))).Read(data)
+
+	r := bytes.NewReader(data)
+	fcid, err := client.ClientImportLocal(ctx, r)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Println("FILE CID: ", fcid)
+
+	deal := startDeal(t, ctx, miner, client, fcid)
+
+	// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
+	time.Sleep(time.Second)
+	waitDealSealed(t, ctx, client, deal)
+
+	// Retrieval
+
+	testRetrieval(t, ctx, err, client, fcid, carExport, data)
+}
+
+func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, fcid cid.Cid) *cid.Cid {
+	maddr, err := miner.ActorAddress(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	addr, err := client.WalletDefaultAddress(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	deal, err := client.ClientStartDeal(ctx, &api.StartDealParams{
+		Data:              &storagemarket.DataRef{Root: fcid},
+		Wallet:            addr,
+		Miner:             maddr,
+		EpochPrice:        types.NewInt(1000000),
+		MinBlocksDuration: 100,
+	})
+	if err != nil {
+		t.Fatalf("%+v", err)
+	}
+	return deal
+}
+
+func waitDealSealed(t *testing.T, ctx context.Context, client *impl.FullNodeAPI, deal *cid.Cid) {
+loop:
+	for {
+		di, err := client.ClientGetDealInfo(ctx, *deal)
+		if err != nil {
+			t.Fatal(err)
+		}
+		switch di.State {
+		case storagemarket.StorageDealProposalRejected:
+			t.Fatal("deal rejected")
+		case storagemarket.StorageDealFailing:
+			t.Fatal("deal failed")
+		case storagemarket.StorageDealError:
+			t.Fatal("deal errored", di.Message)
+		case storagemarket.StorageDealActive:
+			fmt.Println("COMPLETE", di)
+			break loop
+		}
+		fmt.Println("Deal state: ", storagemarket.DealStates[di.State])
+		time.Sleep(time.Second / 2)
+	}
+}
+
+func testRetrieval(t *testing.T, ctx context.Context, err error, client *impl.FullNodeAPI, fcid cid.Cid, carExport bool, data []byte) {
+	offers, err := client.ClientFindData(ctx, fcid)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(offers) < 1 {
+		t.Fatal("no offers")
+	}
+
+	rpath, err := ioutil.TempDir("", "lotus-retrieve-test-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(rpath) //nolint:errcheck
+
+	caddr, err := client.WalletDefaultAddress(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ref := &api.FileRef{
+		Path:  filepath.Join(rpath, "ret"),
+		IsCAR: carExport,
+	}
+	err = client.ClientRetrieve(ctx, offers[0].Order(caddr), ref)
+	if err != nil {
+		t.Fatalf("%+v", err)
+	}
+
+	rdata, err := ioutil.ReadFile(filepath.Join(rpath, "ret"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if carExport {
+		rdata = extractCarData(t, ctx, rdata, rpath)
+	}
+
+	if !bytes.Equal(rdata, data) {
+		t.Fatal("wrong data retrieved")
+	}
+}
+
+func extractCarData(t *testing.T, ctx context.Context, rdata []byte, rpath string) []byte {
+	bserv := dstest.Bserv()
+	ch, err := car.LoadCar(bserv.Blockstore(), bytes.NewReader(rdata))
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, err := bserv.GetBlock(ctx, ch.Roots[0])
+	if err != nil {
+		t.Fatal(err)
+	}
+	nd, err := ipld.Decode(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dserv := dag.NewDAGService(bserv)
+	fil, err := unixfile.NewUnixfsFile(ctx, dserv, nd)
+	if err != nil {
+		t.Fatal(err)
+	}
+	outPath := filepath.Join(rpath, "retLoadedCAR")
+	if err := files.WriteTo(fil, outPath); err != nil {
+		t.Fatal(err)
+	}
+	rdata, err = ioutil.ReadFile(outPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return rdata
+}

api/test/mining.go

@@ -0,0 +1,199 @@
+package test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"math/rand"
+	"os"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	logging "github.com/ipfs/go-log/v2"
+
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/node/impl"
+)
+
+var log = logging.Logger("apitest")
+
+func (ts *testSuite) testMining(t *testing.T) {
+	ctx := context.Background()
+	apis, sn := ts.makeNodes(t, 1, oneMiner)
+	api := apis[0]
+
+	h1, err := api.ChainHead(ctx)
+	require.NoError(t, err)
+	require.Equal(t, abi.ChainEpoch(0), h1.Height())
+
+	newHeads, err := api.ChainNotify(ctx)
+	require.NoError(t, err)
+	<-newHeads
+
+	err = sn[0].MineOne(ctx, func(bool) {})
+	require.NoError(t, err)
+
+	<-newHeads
+
+	h2, err := api.ChainHead(ctx)
+	require.NoError(t, err)
+	require.Equal(t, abi.ChainEpoch(1), h2.Height())
+}
+
+func (ts *testSuite) testMiningReal(t *testing.T) {
+	build.InsecurePoStValidation = false
+	defer func() {
+		build.InsecurePoStValidation = true
+	}()
+
+	ctx := context.Background()
+	apis, sn := ts.makeNodes(t, 1, oneMiner)
+	api := apis[0]
+
+	h1, err := api.ChainHead(ctx)
+	require.NoError(t, err)
+	require.Equal(t, abi.ChainEpoch(0), h1.Height())
+
+	newHeads, err := api.ChainNotify(ctx)
+	require.NoError(t, err)
+	<-newHeads
+
+	err = sn[0].MineOne(ctx, func(bool) {})
+	require.NoError(t, err)
+
+	<-newHeads
+
+	h2, err := api.ChainHead(ctx)
+	require.NoError(t, err)
+	require.Equal(t, abi.ChainEpoch(1), h2.Height())
+
+	err = sn[0].MineOne(ctx, func(bool) {})
+	require.NoError(t, err)
+
+	<-newHeads
+
+	h2, err = api.ChainHead(ctx)
+	require.NoError(t, err)
+	require.Equal(t, abi.ChainEpoch(2), h2.Height())
+}
+
+func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExport bool) {
+	_ = os.Setenv("BELLMAN_NO_GPU", "1")
+
+	// test making a deal with a fresh miner, and see if it starts to mine
+
+	ctx := context.Background()
+	n, sn := b(t, 1, []StorageMiner{
+		{Full: 0, Preseal: PresealGenesis},
+		{Full: 0, Preseal: 0}, // TODO: Add support for storage miners on non-first full node
+	})
+	client := n[0].FullNode.(*impl.FullNodeAPI)
+	provider := sn[1]
+	genesisMiner := sn[0]
+
+	addrinfo, err := client.NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := provider.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := genesisMiner.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
+
+	time.Sleep(time.Second)
+
+	data := make([]byte, 600)
+	rand.New(rand.NewSource(5)).Read(data)
+
+	r := bytes.NewReader(data)
+	fcid, err := client.ClientImportLocal(ctx, r)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Println("FILE CID: ", fcid)
+
+	var mine int32 = 1
+	done := make(chan struct{})
+	minedTwo := make(chan struct{})
+
+	go func() {
+		defer close(done)
+
+		prevExpect := 0
+		for atomic.LoadInt32(&mine) != 0 {
+			wait := make(chan int, 2)
+			mdone := func(mined bool) {
+				go func() {
+					n := 0
+					if mined {
+						n = 1
+					}
+					wait <- n
+				}()
+			}
+
+			if err := sn[0].MineOne(ctx, mdone); err != nil {
+				t.Error(err)
+			}
+
+			if err := sn[1].MineOne(ctx, mdone); err != nil {
+				t.Error(err)
+			}
+
+			expect := <-wait
+			expect += <-wait
+
+			time.Sleep(blocktime)
+
+			for {
+				n := 0
+				for i, node := range sn {
+					mb, err := node.MiningBase(ctx)
+					if err != nil {
+						t.Error(err)
+						return
+					}
+
+					if len(mb.Cids()) != expect {
+						log.Warnf("node %d mining base not complete (%d, want %d)", i, len(mb.Cids()), expect)
+						continue
+					}
+					n++
+				}
+				if n == len(sn) {
+					break
+				}
+				time.Sleep(blocktime)
+			}
+
+			if prevExpect == 2 && expect == 2 && minedTwo != nil {
+				close(minedTwo)
+				minedTwo = nil
+			}
+
+			prevExpect = expect
+		}
+	}()
+
+	deal := startDeal(t, ctx, provider, client, fcid)
+
+	// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
+	time.Sleep(time.Second)
+
+	waitDealSealed(t, ctx, client, deal)
+
+	<-minedTwo
+
+	atomic.StoreInt32(&mine, 0)
+	fmt.Println("shutting down mining")
+	<-done
+}

api/test/test.go

@@ -0,0 +1,126 @@
+package test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
+)
+
+type TestNode struct {
+	api.FullNode
+}
+
+type TestStorageNode struct {
+	api.StorageMiner
+
+	MineOne func(context.Context, func(bool)) error
+}
+
+var PresealGenesis = -1
+
+const GenesisPreseals = 2
+
+type StorageMiner struct {
+	Full    int
+	Preseal int
+}
+
+// APIBuilder is a function which is invoked in test suite to provide
+// test nodes and networks
+//
+// storage array defines storage nodes, numbers in the array specify full node
+// index the storage node 'belongs' to
+type APIBuilder func(t *testing.T, nFull int, storage []StorageMiner) ([]TestNode, []TestStorageNode)
+type testSuite struct {
+	makeNodes APIBuilder
+}
+
+// TestApis is the entry point to API test suite
+func TestApis(t *testing.T, b APIBuilder) {
+	ts := testSuite{
+		makeNodes: b,
+	}
+
+	t.Run("version", ts.testVersion)
+	t.Run("id", ts.testID)
+	t.Run("testConnectTwo", ts.testConnectTwo)
+	t.Run("testMining", ts.testMining)
+	t.Run("testMiningReal", ts.testMiningReal)
+}
+
+var oneMiner = []StorageMiner{{Full: 0, Preseal: PresealGenesis}}
+
+func (ts *testSuite) testVersion(t *testing.T) {
+	ctx := context.Background()
+	apis, _ := ts.makeNodes(t, 1, oneMiner)
+	api := apis[0]
+
+	v, err := api.Version(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	require.Equal(t, v.Version, build.BuildVersion)
+}
+
+func (ts *testSuite) testID(t *testing.T) {
+	ctx := context.Background()
+	apis, _ := ts.makeNodes(t, 1, oneMiner)
+	api := apis[0]
+
+	id, err := api.ID(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Regexp(t, "^12", id.Pretty())
+}
+
+func (ts *testSuite) testConnectTwo(t *testing.T) {
+	ctx := context.Background()
+	apis, _ := ts.makeNodes(t, 2, oneMiner)
+
+	p, err := apis[0].NetPeers(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(p) != 0 {
+		t.Error("Node 0 has a peer")
+	}
+
+	p, err = apis[1].NetPeers(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(p) != 0 {
+		t.Error("Node 1 has a peer")
+	}
+
+	addrs, err := apis[1].NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := apis[0].NetConnect(ctx, addrs); err != nil {
+		t.Fatal(err)
+	}
+
+	p, err = apis[0].NetPeers(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(p) != 1 {
+		t.Error("Node 0 doesn't have 1 peer")
+	}
+
+	p, err = apis[1].NetPeers(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(p) != 1 {
+		t.Error("Node 0 doesn't have 1 peer")
+	}
+}

api/test/window_post.go

@@ -0,0 +1,169 @@
+package test
+
+import (
+	"context"
+	"fmt"
+	"github.com/filecoin-project/lotus/api"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	miner2 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
+	sealing "github.com/filecoin-project/storage-fsm"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/node/impl"
+)
+
+func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) {
+	os.Setenv("BELLMAN_NO_GPU", "1")
+
+	ctx := context.Background()
+	n, sn := b(t, 1, oneMiner)
+	client := n[0].FullNode.(*impl.FullNodeAPI)
+	miner := sn[0]
+
+	addrinfo, err := client.NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := miner.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Second)
+
+	mine := true
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for mine {
+			time.Sleep(blocktime)
+			if err := sn[0].MineOne(ctx, func(bool) {}); err != nil {
+				t.Error(err)
+			}
+		}
+	}()
+
+	pledgeSectors(t, ctx, miner, nSectors)
+
+	mine = false
+	<-done
+}
+
+func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n int) {
+	for i := 0; i < n; i++ {
+		err := miner.PledgeSector(ctx)
+		require.NoError(t, err)
+	}
+
+	for {
+		s, err := miner.SectorsList(ctx) // Note - the test builder doesn't import genesis sectors into FSM
+		require.NoError(t, err)
+		fmt.Printf("Sectors: %d\n", len(s))
+		if len(s) >= n {
+			break
+		}
+
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	fmt.Printf("All sectors is fsm\n")
+
+	s, err := miner.SectorsList(ctx)
+	require.NoError(t, err)
+
+	toCheck := map[abi.SectorNumber]struct{}{}
+	for _, number := range s {
+		toCheck[number] = struct{}{}
+	}
+
+	for len(toCheck) > 0 {
+		for n := range toCheck {
+			st, err := miner.SectorsStatus(ctx, n)
+			require.NoError(t, err)
+			if st.State == api.SectorState(sealing.Proving) {
+				delete(toCheck, n)
+			}
+			if strings.Contains(string(st.State), "Fail") {
+				t.Fatal("sector in a failed state", st.State)
+			}
+		}
+
+		time.Sleep(100 * time.Millisecond)
+		fmt.Printf("WaitSeal: %d\n", len(s))
+	}
+}
+
+func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) {
+	os.Setenv("BELLMAN_NO_GPU", "1")
+
+	ctx := context.Background()
+	n, sn := b(t, 1, oneMiner)
+	client := n[0].FullNode.(*impl.FullNodeAPI)
+	miner := sn[0]
+
+	addrinfo, err := client.NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := miner.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Second)
+
+	mine := true
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for mine {
+			time.Sleep(blocktime)
+			if err := sn[0].MineOne(ctx, func(bool) {}); err != nil {
+				t.Error(err)
+			}
+		}
+	}()
+
+	pledgeSectors(t, ctx, miner, nSectors)
+
+	maddr, err := miner.ActorAddress(ctx)
+	require.NoError(t, err)
+
+	di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+
+	fmt.Printf("Running one proving periods\n")
+
+	for {
+		head, err := client.ChainHead(ctx)
+		require.NoError(t, err)
+
+		if head.Height() > di.PeriodStart+(miner2.WPoStProvingPeriod)+2 {
+			break
+		}
+
+		if head.Height()%100 == 0 {
+			fmt.Printf("@%d\n", head.Height())
+		}
+		time.Sleep(blocktime)
+	}
+
+	p, err := client.StateMinerPower(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+
+	ssz, err := miner.ActorSectorSize(ctx, maddr)
+	require.NoError(t, err)
+
+	require.Equal(t, p.MinerPower, p.TotalPower)
+	require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*uint64(nSectors+GenesisPreseals)))
+
+	// TODO: Inject faults here
+
+	mine = false
+	<-done
+}

api/types.go

@@ -2,18 +2,8 @@ package api
 
 import (
 	"encoding/json"
-	"fmt"
-	"time"
-
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/lotus/chain/types"
-
-	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/ipfs/go-cid"
 
 	"github.com/libp2p/go-libp2p-core/peer"
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	ma "github.com/multiformats/go-multiaddr"
 )
 
@@ -47,151 +37,5 @@ type ObjStat struct {
 
 type PubsubScore struct {
 	ID    peer.ID
-	Score *pubsub.PeerScoreSnapshot
-}
-
-type MessageSendSpec struct {
-	MaxFee abi.TokenAmount
-}
-
-type DataTransferChannel struct {
-	TransferID  datatransfer.TransferID
-	Status      datatransfer.Status
-	BaseCID     cid.Cid
-	IsInitiator bool
-	IsSender    bool
-	Voucher     string
-	Message     string
-	OtherPeer   peer.ID
-	Transferred uint64
-	Stages      *datatransfer.ChannelStages
-}
-
-// NewDataTransferChannel constructs an API DataTransferChannel type from full channel state snapshot and a host id
-func NewDataTransferChannel(hostID peer.ID, channelState datatransfer.ChannelState) DataTransferChannel {
-	channel := DataTransferChannel{
-		TransferID: channelState.TransferID(),
-		Status:     channelState.Status(),
-		BaseCID:    channelState.BaseCID(),
-		IsSender:   channelState.Sender() == hostID,
-		Message:    channelState.Message(),
-	}
-	stringer, ok := channelState.Voucher().(fmt.Stringer)
-	if ok {
-		channel.Voucher = stringer.String()
-	} else {
-		voucherJSON, err := json.Marshal(channelState.Voucher())
-		if err != nil {
-			channel.Voucher = fmt.Errorf("Voucher Serialization: %w", err).Error()
-		} else {
-			channel.Voucher = string(voucherJSON)
-		}
-	}
-	if channel.IsSender {
-		channel.IsInitiator = !channelState.IsPull()
-		channel.Transferred = channelState.Sent()
-		channel.OtherPeer = channelState.Recipient()
-	} else {
-		channel.IsInitiator = channelState.IsPull()
-		channel.Transferred = channelState.Received()
-		channel.OtherPeer = channelState.Sender()
-	}
-	return channel
-}
-
-type NetBlockList struct {
-	Peers     []peer.ID
-	IPAddrs   []string
-	IPSubnets []string
-}
-
-type ExtendedPeerInfo struct {
-	ID          peer.ID
-	Agent       string
-	Addrs       []string
-	Protocols   []string
-	ConnMgrMeta *ConnMgrInfo
-}
-
-type ConnMgrInfo struct {
-	FirstSeen time.Time
-	Value     int
-	Tags      map[string]int
-	Conns     map[string]time.Time
-}
-
-type NodeStatus struct {
-	SyncStatus  NodeSyncStatus
-	PeerStatus  NodePeerStatus
-	ChainStatus NodeChainStatus
-}
-
-type NodeSyncStatus struct {
-	Epoch  uint64
-	Behind uint64
-}
-
-type NodePeerStatus struct {
-	PeersToPublishMsgs   int
-	PeersToPublishBlocks int
-}
-
-type NodeChainStatus struct {
-	BlocksPerTipsetLast100      float64
-	BlocksPerTipsetLastFinality float64
-}
-
-type CheckStatusCode int
-
-//go:generate go run golang.org/x/tools/cmd/stringer -type=CheckStatusCode -trimprefix=CheckStatus
-const (
-	_ CheckStatusCode = iota
-	// Message Checks
-	CheckStatusMessageSerialize
-	CheckStatusMessageSize
-	CheckStatusMessageValidity
-	CheckStatusMessageMinGas
-	CheckStatusMessageMinBaseFee
-	CheckStatusMessageBaseFee
-	CheckStatusMessageBaseFeeLowerBound
-	CheckStatusMessageBaseFeeUpperBound
-	CheckStatusMessageGetStateNonce
-	CheckStatusMessageNonce
-	CheckStatusMessageGetStateBalance
-	CheckStatusMessageBalance
-)
-
-type CheckStatus struct {
-	Code CheckStatusCode
-	OK   bool
-	Err  string
-	Hint map[string]interface{}
-}
-
-type MessageCheckStatus struct {
-	Cid cid.Cid
-	CheckStatus
-}
-
-type MessagePrototype struct {
-	Message    types.Message
-	ValidNonce bool
-}
-
-type RetrievalInfo struct {
-	PayloadCID   cid.Cid
-	ID           retrievalmarket.DealID
-	PieceCID     *cid.Cid
-	PricePerByte abi.TokenAmount
-	UnsealPrice  abi.TokenAmount
-
-	Status        retrievalmarket.DealStatus
-	Message       string // more information about deal state, particularly errors
-	Provider      peer.ID
-	BytesReceived uint64
-	BytesPaidFor  uint64
-	TotalPaid     abi.TokenAmount
-
-	TransferChannelID *datatransfer.ChannelID
-	DataTransfer      *DataTransferChannel
+	Score float64
 }

api/types/actors.go

@@ -1,5 +0,0 @@
-package apitypes
-
-import "github.com/filecoin-project/go-state-types/network"
-
-type NetworkVersion = network.Version

api/types/openrpc.go

@@ -1,3 +0,0 @@
-package apitypes
-
-type OpenRPCDocument map[string]interface{}

api/utils.go

@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
 )
 
 type SignFunc = func(context.Context, []byte) (*crypto.Signature, error)

api/v0api/full.go

@@ -1,711 +0,0 @@
-package v0api
-
-import (
-	"context"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-bitfield"
-	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/go-fil-markets/storagemarket"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/crypto"
-	"github.com/filecoin-project/go-state-types/dline"
-	"github.com/filecoin-project/lotus/node/repo/importmgr"
-	"github.com/ipfs/go-cid"
-	"github.com/libp2p/go-libp2p-core/peer"
-
-	"github.com/filecoin-project/lotus/api"
-	apitypes "github.com/filecoin-project/lotus/api/types"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
-	"github.com/filecoin-project/lotus/chain/types"
-	marketevents "github.com/filecoin-project/lotus/markets/loggers"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-)
-
-//go:generate go run github.com/golang/mock/mockgen -destination=v0mocks/mock_full.go -package=v0mocks . FullNode
-
-//                       MODIFYING THE API INTERFACE
-//
-// NOTE: This is the V0 (Stable) API - when adding methods to this interface,
-// you'll need to make sure they are also present on the V1 (Unstable) API
-//
-// This API is implemented in `v1_wrapper.go` as a compatibility layer backed
-// by the V1 api
-//
-// When adding / changing methods in this file:
-// * Do the change here
-// * Adjust implementation in `node/impl/`
-// * Run `make gen` - this will:
-//  * Generate proxy structs
-//  * Generate mocks
-//  * Generate markdown docs
-//  * Generate openrpc blobs
-
-// FullNode API is a low-level interface to the Filecoin network full node
-type FullNode interface {
-	Common
-	Net
-
-	// MethodGroup: Chain
-	// The Chain method group contains methods for interacting with the
-	// blockchain, but that do not require any form of state computation.
-
-	// ChainNotify returns channel with chain head updates.
-	// First message is guaranteed to be of len == 1, and type == 'current'.
-	ChainNotify(context.Context) (<-chan []*api.HeadChange, error) //perm:read
-
-	// ChainHead returns the current head of the chain.
-	ChainHead(context.Context) (*types.TipSet, error) //perm:read
-
-	// ChainGetRandomnessFromTickets is used to sample the chain for randomness.
-	ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) //perm:read
-
-	// ChainGetRandomnessFromBeacon is used to sample the beacon for randomness.
-	ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) //perm:read
-
-	// ChainGetBlock returns the block specified by the given CID.
-	ChainGetBlock(context.Context, cid.Cid) (*types.BlockHeader, error) //perm:read
-	// ChainGetTipSet returns the tipset specified by the given TipSetKey.
-	ChainGetTipSet(context.Context, types.TipSetKey) (*types.TipSet, error) //perm:read
-
-	// ChainGetBlockMessages returns messages stored in the specified block.
-	//
-	// Note: If there are multiple blocks in a tipset, it's likely that some
-	// messages will be duplicated. It's also possible for blocks in a tipset to have
-	// different messages from the same sender at the same nonce. When that happens,
-	// only the first message (in a block with lowest ticket) will be considered
-	// for execution
-	//
-	// NOTE: THIS METHOD SHOULD ONLY BE USED FOR GETTING MESSAGES IN A SPECIFIC BLOCK
-	//
-	// DO NOT USE THIS METHOD TO GET MESSAGES INCLUDED IN A TIPSET
-	// Use ChainGetParentMessages, which will perform correct message deduplication
-	ChainGetBlockMessages(ctx context.Context, blockCid cid.Cid) (*api.BlockMessages, error) //perm:read
-
-	// ChainGetParentReceipts returns receipts for messages in parent tipset of
-	// the specified block. The receipts in the list returned is one-to-one with the
-	// messages returned by a call to ChainGetParentMessages with the same blockCid.
-	ChainGetParentReceipts(ctx context.Context, blockCid cid.Cid) ([]*types.MessageReceipt, error) //perm:read
-
-	// ChainGetParentMessages returns messages stored in parent tipset of the
-	// specified block.
-	ChainGetParentMessages(ctx context.Context, blockCid cid.Cid) ([]api.Message, error) //perm:read
-
-	// ChainGetMessagesInTipset returns message stores in current tipset
-	ChainGetMessagesInTipset(ctx context.Context, tsk types.TipSetKey) ([]api.Message, error) //perm:read
-
-	// ChainGetTipSetByHeight looks back for a tipset at the specified epoch.
-	// If there are no blocks at the specified epoch, a tipset at an earlier epoch
-	// will be returned.
-	ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) //perm:read
-
-	// ChainReadObj reads ipld nodes referenced by the specified CID from chain
-	// blockstore and returns raw bytes.
-	ChainReadObj(context.Context, cid.Cid) ([]byte, error) //perm:read
-
-	// ChainDeleteObj deletes node referenced by the given CID
-	ChainDeleteObj(context.Context, cid.Cid) error //perm:admin
-
-	// ChainHasObj checks if a given CID exists in the chain blockstore.
-	ChainHasObj(context.Context, cid.Cid) (bool, error) //perm:read
-
-	// ChainStatObj returns statistics about the graph referenced by 'obj'.
-	// If 'base' is also specified, then the returned stat will be a diff
-	// between the two objects.
-	ChainStatObj(ctx context.Context, obj cid.Cid, base cid.Cid) (api.ObjStat, error) //perm:read
-
-	// ChainSetHead forcefully sets current chain head. Use with caution.
-	ChainSetHead(context.Context, types.TipSetKey) error //perm:admin
-
-	// ChainGetGenesis returns the genesis tipset.
-	ChainGetGenesis(context.Context) (*types.TipSet, error) //perm:read
-
-	// ChainTipSetWeight computes weight for the specified tipset.
-	ChainTipSetWeight(context.Context, types.TipSetKey) (types.BigInt, error) //perm:read
-	ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error)      //perm:read
-
-	// ChainGetMessage reads a message referenced by the specified CID from the
-	// chain blockstore.
-	ChainGetMessage(context.Context, cid.Cid) (*types.Message, error) //perm:read
-
-	// ChainGetPath returns a set of revert/apply operations needed to get from
-	// one tipset to another, for example:
-	//```
-	//        to
-	//         ^
-	// from   tAA
-	//   ^     ^
-	// tBA    tAB
-	//  ^---*--^
-	//      ^
-	//     tRR
-	//```
-	// Would return `[revert(tBA), apply(tAB), apply(tAA)]`
-	ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*api.HeadChange, error) //perm:read
-
-	// ChainExport returns a stream of bytes with CAR dump of chain data.
-	// The exported chain data includes the header chain from the given tipset
-	// back to genesis, the entire genesis state, and the most recent 'nroots'
-	// state trees.
-	// If oldmsgskip is set, messages from before the requested roots are also not included.
-	ChainExport(ctx context.Context, nroots abi.ChainEpoch, oldmsgskip bool, tsk types.TipSetKey) (<-chan []byte, error) //perm:read
-
-	// MethodGroup: Beacon
-	// The Beacon method group contains methods for interacting with the random beacon (DRAND)
-
-	// BeaconGetEntry returns the beacon entry for the given filecoin epoch. If
-	// the entry has not yet been produced, the call will block until the entry
-	// becomes available
-	BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) //perm:read
-
-	// GasEstimateFeeCap estimates gas fee cap
-	GasEstimateFeeCap(context.Context, *types.Message, int64, types.TipSetKey) (types.BigInt, error) //perm:read
-
-	// GasEstimateGasLimit estimates gas used by the message and returns it.
-	// It fails if message fails to execute.
-	GasEstimateGasLimit(context.Context, *types.Message, types.TipSetKey) (int64, error) //perm:read
-
-	// GasEstimateGasPremium estimates what gas price should be used for a
-	// message to have high likelihood of inclusion in `nblocksincl` epochs.
-
-	GasEstimateGasPremium(_ context.Context, nblocksincl uint64,
-		sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) //perm:read
-
-	// GasEstimateMessageGas estimates gas values for unset message gas fields
-	GasEstimateMessageGas(context.Context, *types.Message, *api.MessageSendSpec, types.TipSetKey) (*types.Message, error) //perm:read
-
-	// MethodGroup: Sync
-	// The Sync method group contains methods for interacting with and
-	// observing the lotus sync service.
-
-	// SyncState returns the current status of the lotus sync system.
-	SyncState(context.Context) (*api.SyncState, error) //perm:read
-
-	// SyncSubmitBlock can be used to submit a newly created block to the.
-	// network through this node
-	SyncSubmitBlock(ctx context.Context, blk *types.BlockMsg) error //perm:write
-
-	// SyncIncomingBlocks returns a channel streaming incoming, potentially not
-	// yet synced block headers.
-	SyncIncomingBlocks(ctx context.Context) (<-chan *types.BlockHeader, error) //perm:read
-
-	// SyncCheckpoint marks a blocks as checkpointed, meaning that it won't ever fork away from it.
-	SyncCheckpoint(ctx context.Context, tsk types.TipSetKey) error //perm:admin
-
-	// SyncMarkBad marks a blocks as bad, meaning that it won't ever by synced.
-	// Use with extreme caution.
-	SyncMarkBad(ctx context.Context, bcid cid.Cid) error //perm:admin
-
-	// SyncUnmarkBad unmarks a blocks as bad, making it possible to be validated and synced again.
-	SyncUnmarkBad(ctx context.Context, bcid cid.Cid) error //perm:admin
-
-	// SyncUnmarkAllBad purges bad block cache, making it possible to sync to chains previously marked as bad
-	SyncUnmarkAllBad(ctx context.Context) error //perm:admin
-
-	// SyncCheckBad checks if a block was marked as bad, and if it was, returns
-	// the reason.
-	SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error) //perm:read
-
-	// SyncValidateTipset indicates whether the provided tipset is valid or not
-	SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) (bool, error) //perm:read
-
-	// MethodGroup: Mpool
-	// The Mpool methods are for interacting with the message pool. The message pool
-	// manages all incoming and outgoing 'messages' going over the network.
-
-	// MpoolPending returns pending mempool messages.
-	MpoolPending(context.Context, types.TipSetKey) ([]*types.SignedMessage, error) //perm:read
-
-	// MpoolSelect returns a list of pending messages for inclusion in the next block
-	MpoolSelect(context.Context, types.TipSetKey, float64) ([]*types.SignedMessage, error) //perm:read
-
-	// MpoolPush pushes a signed message to mempool.
-	MpoolPush(context.Context, *types.SignedMessage) (cid.Cid, error) //perm:write
-
-	// MpoolPushUntrusted pushes a signed message to mempool from untrusted sources.
-	MpoolPushUntrusted(context.Context, *types.SignedMessage) (cid.Cid, error) //perm:write
-
-	// MpoolPushMessage atomically assigns a nonce, signs, and pushes a message
-	// to mempool.
-	// maxFee is only used when GasFeeCap/GasPremium fields aren't specified
-	//
-	// When maxFee is set to 0, MpoolPushMessage will guess appropriate fee
-	// based on current chain conditions
-	MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) //perm:sign
-
-	// MpoolBatchPush batch pushes a signed message to mempool.
-	MpoolBatchPush(context.Context, []*types.SignedMessage) ([]cid.Cid, error) //perm:write
-
-	// MpoolBatchPushUntrusted batch pushes a signed message to mempool from untrusted sources.
-	MpoolBatchPushUntrusted(context.Context, []*types.SignedMessage) ([]cid.Cid, error) //perm:write
-
-	// MpoolBatchPushMessage batch pushes a unsigned message to mempool.
-	MpoolBatchPushMessage(context.Context, []*types.Message, *api.MessageSendSpec) ([]*types.SignedMessage, error) //perm:sign
-
-	// MpoolGetNonce gets next nonce for the specified sender.
-	// Note that this method may not be atomic. Use MpoolPushMessage instead.
-	MpoolGetNonce(context.Context, address.Address) (uint64, error) //perm:read
-	MpoolSub(context.Context) (<-chan api.MpoolUpdate, error)       //perm:read
-
-	// MpoolClear clears pending messages from the mpool
-	MpoolClear(context.Context, bool) error //perm:write
-
-	// MpoolGetConfig returns (a copy of) the current mpool config
-	MpoolGetConfig(context.Context) (*types.MpoolConfig, error) //perm:read
-	// MpoolSetConfig sets the mpool config to (a copy of) the supplied config
-	MpoolSetConfig(context.Context, *types.MpoolConfig) error //perm:admin
-
-	// MethodGroup: Miner
-
-	MinerGetBaseInfo(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*api.MiningBaseInfo, error) //perm:read
-	MinerCreateBlock(context.Context, *api.BlockTemplate) (*types.BlockMsg, error)                                   //perm:write
-
-	// // UX ?
-
-	// MethodGroup: Wallet
-
-	// WalletNew creates a new address in the wallet with the given sigType.
-	// Available key types: bls, secp256k1, secp256k1-ledger
-	// Support for numerical types: 1 - secp256k1, 2 - BLS is deprecated
-	WalletNew(context.Context, types.KeyType) (address.Address, error) //perm:write
-	// WalletHas indicates whether the given address is in the wallet.
-	WalletHas(context.Context, address.Address) (bool, error) //perm:write
-	// WalletList lists all the addresses in the wallet.
-	WalletList(context.Context) ([]address.Address, error) //perm:write
-	// WalletBalance returns the balance of the given address at the current head of the chain.
-	WalletBalance(context.Context, address.Address) (types.BigInt, error) //perm:read
-	// WalletSign signs the given bytes using the given address.
-	WalletSign(context.Context, address.Address, []byte) (*crypto.Signature, error) //perm:sign
-	// WalletSignMessage signs the given message using the given address.
-	WalletSignMessage(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) //perm:sign
-	// WalletVerify takes an address, a signature, and some bytes, and indicates whether the signature is valid.
-	// The address does not have to be in the wallet.
-	WalletVerify(context.Context, address.Address, []byte, *crypto.Signature) (bool, error) //perm:read
-	// WalletDefaultAddress returns the address marked as default in the wallet.
-	WalletDefaultAddress(context.Context) (address.Address, error) //perm:write
-	// WalletSetDefault marks the given address as as the default one.
-	WalletSetDefault(context.Context, address.Address) error //perm:write
-	// WalletExport returns the private key of an address in the wallet.
-	WalletExport(context.Context, address.Address) (*types.KeyInfo, error) //perm:admin
-	// WalletImport receives a KeyInfo, which includes a private key, and imports it into the wallet.
-	WalletImport(context.Context, *types.KeyInfo) (address.Address, error) //perm:admin
-	// WalletDelete deletes an address from the wallet.
-	WalletDelete(context.Context, address.Address) error //perm:admin
-	// WalletValidateAddress validates whether a given string can be decoded as a well-formed address
-	WalletValidateAddress(context.Context, string) (address.Address, error) //perm:read
-
-	// Other
-
-	// MethodGroup: Client
-	// The Client methods all have to do with interacting with the storage and
-	// retrieval markets as a client
-
-	// ClientImport imports file under the specified path into filestore.
-	ClientImport(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) //perm:admin
-	// ClientRemoveImport removes file import
-	ClientRemoveImport(ctx context.Context, importID importmgr.ImportID) error //perm:admin
-	// ClientStartDeal proposes a deal with a miner.
-	ClientStartDeal(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) //perm:admin
-	// ClientStatelessDeal fire-and-forget-proposes an offline deal to a miner without subsequent tracking.
-	ClientStatelessDeal(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) //perm:write
-	// ClientGetDealInfo returns the latest information about a given deal.
-	ClientGetDealInfo(context.Context, cid.Cid) (*api.DealInfo, error) //perm:read
-	// ClientListDeals returns information about the deals made by the local client.
-	ClientListDeals(ctx context.Context) ([]api.DealInfo, error) //perm:write
-	// ClientGetDealUpdates returns the status of updated deals
-	ClientGetDealUpdates(ctx context.Context) (<-chan api.DealInfo, error) //perm:write
-	// ClientGetDealStatus returns status given a code
-	ClientGetDealStatus(ctx context.Context, statusCode uint64) (string, error) //perm:read
-	// ClientHasLocal indicates whether a certain CID is locally stored.
-	ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) //perm:write
-	// ClientFindData identifies peers that have a certain file, and returns QueryOffers (one per peer).
-	ClientFindData(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]api.QueryOffer, error) //perm:read
-	// ClientMinerQueryOffer returns a QueryOffer for the specific miner and file.
-	ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error) //perm:read
-	// ClientRetrieve initiates the retrieval of a file, as specified in the order.
-	ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error //perm:admin
-	// ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel
-	// of status updates.
-	ClientRetrieveWithEvents(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin
-	// ClientQueryAsk returns a signed StorageAsk from the specified miner.
-	// ClientListRetrievals returns information about retrievals made by the local client
-	ClientListRetrievals(ctx context.Context) ([]api.RetrievalInfo, error) //perm:write
-	// ClientGetRetrievalUpdates returns status of updated retrieval deals
-	ClientGetRetrievalUpdates(ctx context.Context) (<-chan api.RetrievalInfo, error)                         //perm:write
-	ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error) //perm:read
-	// ClientCalcCommP calculates the CommP and data size of the specified CID
-	ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCIDSize, error) //perm:read
-	// ClientCalcCommP calculates the CommP for a specified file
-	ClientCalcCommP(ctx context.Context, inpath string) (*api.CommPRet, error) //perm:write
-	// ClientGenCar generates a CAR file for the specified file.
-	ClientGenCar(ctx context.Context, ref api.FileRef, outpath string) error //perm:write
-	// ClientDealSize calculates real deal data size
-	ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, error) //perm:read
-	// ClientListTransfers returns the status of all ongoing transfers of data
-	ClientListDataTransfers(ctx context.Context) ([]api.DataTransferChannel, error)        //perm:write
-	ClientDataTransferUpdates(ctx context.Context) (<-chan api.DataTransferChannel, error) //perm:write
-	// ClientRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer
-	ClientRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
-	// ClientCancelDataTransfer cancels a data transfer with the given transfer ID and other peer
-	ClientCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
-	// ClientRetrieveTryRestartInsufficientFunds attempts to restart stalled retrievals on a given payment channel
-	// which are stuck due to insufficient funds
-	ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error //perm:write
-
-	// ClientCancelRetrievalDeal cancels an ongoing retrieval deal based on DealID
-	ClientCancelRetrievalDeal(ctx context.Context, dealid retrievalmarket.DealID) error //perm:write
-
-	// ClientUnimport removes references to the specified file from filestore
-	//ClientUnimport(path string)
-
-	// ClientListImports lists imported files and their root CIDs
-	ClientListImports(ctx context.Context) ([]api.Import, error) //perm:write
-
-	//ClientListAsks() []Ask
-
-	// MethodGroup: State
-	// The State methods are used to query, inspect, and interact with chain state.
-	// Most methods take a TipSetKey as a parameter. The state looked up is the parent state of the tipset.
-	// A nil TipSetKey can be provided as a param, this will cause the heaviest tipset in the chain to be used.
-
-	// StateCall runs the given message and returns its result without any persisted changes.
-	//
-	// StateCall applies the message to the tipset's parent state. The
-	// message is not applied on-top-of the messages in the passed-in
-	// tipset.
-	StateCall(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) //perm:read
-	// StateReplay replays a given message, assuming it was included in a block in the specified tipset.
-	//
-	// If a tipset key is provided, and a replacing message is found on chain,
-	// the method will return an error saying that the message wasn't found
-	//
-	// If no tipset key is provided, the appropriate tipset is looked up, and if
-	// the message was gas-repriced, the on-chain message will be replayed - in
-	// that case the returned InvocResult.MsgCid will not match the Cid param
-	//
-	// If the caller wants to ensure that exactly the requested message was executed,
-	// they MUST check that InvocResult.MsgCid is equal to the provided Cid.
-	// Without this check both the requested and original message may appear as
-	// successfully executed on-chain, which may look like a double-spend.
-	//
-	// A replacing message is a message with a different CID, any of Gas values, and
-	// different signature, but with all other parameters matching (source/destination,
-	// nonce, params, etc.)
-	StateReplay(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) //perm:read
-	// StateGetActor returns the indicated actor's nonce and balance.
-	StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) //perm:read
-	// StateReadState returns the indicated actor's state.
-	StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) //perm:read
-	// StateListMessages looks back and returns all messages with a matching to or from address, stopping at the given height.
-	StateListMessages(ctx context.Context, match *api.MessageMatch, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) //perm:read
-	// StateDecodeParams attempts to decode the provided params, based on the recipient actor address and method number.
-	StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error) //perm:read
-
-	// StateNetworkName returns the name of the network the node is synced to
-	StateNetworkName(context.Context) (dtypes.NetworkName, error) //perm:read
-	// StateMinerSectors returns info about the given miner's sectors. If the filter bitfield is nil, all sectors are included.
-	StateMinerSectors(context.Context, address.Address, *bitfield.BitField, types.TipSetKey) ([]*miner.SectorOnChainInfo, error) //perm:read
-	// StateMinerActiveSectors returns info about sectors that a given miner is actively proving.
-	StateMinerActiveSectors(context.Context, address.Address, types.TipSetKey) ([]*miner.SectorOnChainInfo, error) //perm:read
-	// StateMinerProvingDeadline calculates the deadline at some epoch for a proving period
-	// and returns the deadline-related calculations.
-	StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) //perm:read
-	// StateMinerPower returns the power of the indicated miner
-	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) //perm:read
-	// StateMinerInfo returns info about the indicated miner
-	StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) //perm:read
-	// StateMinerDeadlines returns all the proving deadlines for the given miner
-	StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]api.Deadline, error) //perm:read
-	// StateMinerPartitions returns all partitions in the specified deadline
-	StateMinerPartitions(ctx context.Context, m address.Address, dlIdx uint64, tsk types.TipSetKey) ([]api.Partition, error) //perm:read
-	// StateMinerFaults returns a bitfield indicating the faulty sectors of the given miner
-	StateMinerFaults(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) //perm:read
-	// StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset
-	StateAllMinerFaults(ctx context.Context, lookback abi.ChainEpoch, ts types.TipSetKey) ([]*api.Fault, error) //perm:read
-	// StateMinerRecoveries returns a bitfield indicating the recovering sectors of the given miner
-	StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) //perm:read
-	// StateMinerInitialPledgeCollateral returns the precommit deposit for the specified miner's sector
-	StateMinerPreCommitDepositForPower(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) //perm:read
-	// StateMinerInitialPledgeCollateral returns the initial pledge collateral for the specified miner's sector
-	StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) //perm:read
-	// StateMinerAvailableBalance returns the portion of a miner's balance that can be withdrawn or spent
-	StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) //perm:read
-	// StateMinerSectorAllocated checks if a sector is allocated
-	StateMinerSectorAllocated(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (bool, error) //perm:read
-	// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector
-	StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) //perm:read
-	// StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found
-	// NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate
-	// expiration epoch
-	StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) //perm:read
-	// StateSectorExpiration returns epoch at which given sector will expire
-	StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error) //perm:read
-	// StateSectorPartition finds deadline/partition with the specified sector
-	StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) //perm:read
-	// StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed
-	//
-	// NOTE: If a replacing message is found on chain, this method will return
-	// a MsgLookup for the replacing message - the MsgLookup.Message will be a different
-	// CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
-	// result of the execution of the replacing message.
-	//
-	// If the caller wants to ensure that exactly the requested message was executed,
-	// they MUST check that MsgLookup.Message is equal to the provided 'cid'.
-	// Without this check both the requested and original message may appear as
-	// successfully executed on-chain, which may look like a double-spend.
-	//
-	// A replacing message is a message with a different CID, any of Gas values, and
-	// different signature, but with all other parameters matching (source/destination,
-	// nonce, params, etc.)
-	StateSearchMsg(context.Context, cid.Cid) (*api.MsgLookup, error) //perm:read
-	// StateSearchMsgLimited looks back up to limit epochs in the chain for a message, and returns its receipt and the tipset where it was executed
-	//
-	// NOTE: If a replacing message is found on chain, this method will return
-	// a MsgLookup for the replacing message - the MsgLookup.Message will be a different
-	// CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
-	// result of the execution of the replacing message.
-	//
-	// If the caller wants to ensure that exactly the requested message was executed,
-	// they MUST check that MsgLookup.Message is equal to the provided 'cid'.
-	// Without this check both the requested and original message may appear as
-	// successfully executed on-chain, which may look like a double-spend.
-	//
-	// A replacing message is a message with a different CID, any of Gas values, and
-	// different signature, but with all other parameters matching (source/destination,
-	// nonce, params, etc.)
-	StateSearchMsgLimited(ctx context.Context, msg cid.Cid, limit abi.ChainEpoch) (*api.MsgLookup, error) //perm:read
-	// StateWaitMsg looks back in the chain for a message. If not found, it blocks until the
-	// message arrives on chain, and gets to the indicated confidence depth.
-	//
-	// NOTE: If a replacing message is found on chain, this method will return
-	// a MsgLookup for the replacing message - the MsgLookup.Message will be a different
-	// CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
-	// result of the execution of the replacing message.
-	//
-	// If the caller wants to ensure that exactly the requested message was executed,
-	// they MUST check that MsgLookup.Message is equal to the provided 'cid'.
-	// Without this check both the requested and original message may appear as
-	// successfully executed on-chain, which may look like a double-spend.
-	//
-	// A replacing message is a message with a different CID, any of Gas values, and
-	// different signature, but with all other parameters matching (source/destination,
-	// nonce, params, etc.)
-	StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) //perm:read
-	// StateWaitMsgLimited looks back up to limit epochs in the chain for a message.
-	// If not found, it blocks until the message arrives on chain, and gets to the
-	// indicated confidence depth.
-	//
-	// NOTE: If a replacing message is found on chain, this method will return
-	// a MsgLookup for the replacing message - the MsgLookup.Message will be a different
-	// CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
-	// result of the execution of the replacing message.
-	//
-	// If the caller wants to ensure that exactly the requested message was executed,
-	// they MUST check that MsgLookup.Message is equal to the provided 'cid'.
-	// Without this check both the requested and original message may appear as
-	// successfully executed on-chain, which may look like a double-spend.
-	//
-	// A replacing message is a message with a different CID, any of Gas values, and
-	// different signature, but with all other parameters matching (source/destination,
-	// nonce, params, etc.)
-	StateWaitMsgLimited(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch) (*api.MsgLookup, error) //perm:read
-	// StateListMiners returns the addresses of every miner that has claimed power in the Power Actor
-	StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error) //perm:read
-	// StateListActors returns the addresses of every actor in the state
-	StateListActors(context.Context, types.TipSetKey) ([]address.Address, error) //perm:read
-	// StateMarketBalance looks up the Escrow and Locked balances of the given address in the Storage Market
-	StateMarketBalance(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error) //perm:read
-	// StateMarketParticipants returns the Escrow and Locked balances of every participant in the Storage Market
-	StateMarketParticipants(context.Context, types.TipSetKey) (map[string]api.MarketBalance, error) //perm:read
-	// StateMarketDeals returns information about every deal in the Storage Market
-	StateMarketDeals(context.Context, types.TipSetKey) (map[string]api.MarketDeal, error) //perm:read
-	// StateMarketStorageDeal returns information about the indicated deal
-	StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error) //perm:read
-	// StateLookupID retrieves the ID address of the given address
-	StateLookupID(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
-	// StateAccountKey returns the public key address of the given ID address
-	StateAccountKey(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
-	// StateChangedActors returns all the actors whose states change between the two given state CIDs
-	// TODO: Should this take tipset keys instead?
-	StateChangedActors(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) //perm:read
-	// StateGetReceipt returns the message receipt for the given message or for a
-	// matching gas-repriced replacing message
-	//
-	// NOTE: If the requested message was replaced, this method will return the receipt
-	// for the replacing message - if the caller needs the receipt for exactly the
-	// requested message, use StateSearchMsg().Receipt, and check that MsgLookup.Message
-	// is matching the requested CID
-	//
-	// DEPRECATED: Use StateSearchMsg, this method won't be supported in v1 API
-	StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) //perm:read
-	// StateMinerSectorCount returns the number of sectors in a miner's sector set and proving set
-	StateMinerSectorCount(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error) //perm:read
-	// StateCompute is a flexible command that applies the given messages on the given tipset.
-	// The messages are run as though the VM were at the provided height.
-	//
-	// When called, StateCompute will:
-	// - Load the provided tipset, or use the current chain head if not provided
-	// - Compute the tipset state of the provided tipset on top of the parent state
-	//   - (note that this step runs before vmheight is applied to the execution)
-	//   - Execute state upgrade if any were scheduled at the epoch, or in null
-	//     blocks preceding the tipset
-	//   - Call the cron actor on null blocks preceding the tipset
-	//   - For each block in the tipset
-	//     - Apply messages in blocks in the specified
-	//     - Award block reward by calling the reward actor
-	//   - Call the cron actor for the current epoch
-	// - If the specified vmheight is higher than the current epoch, apply any
-	//   needed state upgrades to the state
-	// - Apply the specified messages to the state
-	//
-	// The vmheight parameter sets VM execution epoch, and can be used to simulate
-	// message execution in different network versions. If the specified vmheight
-	// epoch is higher than the epoch of the specified tipset, any state upgrades
-	// until the vmheight will be executed on the state before applying messages
-	// specified by the user.
-	//
-	// Note that the initial tipset state computation is not affected by the
-	// vmheight parameter - only the messages in the `apply` set are
-	//
-	// If the caller wants to simply compute the state, vmheight should be set to
-	// the epoch of the specified tipset.
-	//
-	// Messages in the `apply` parameter must have the correct nonces, and gas
-	// values set.
-	StateCompute(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error) //perm:read
-	// StateVerifierStatus returns the data cap for the given address.
-	// Returns nil if there is no entry in the data cap table for the
-	// address.
-	StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) //perm:read
-	// StateVerifiedClientStatus returns the data cap for the given address.
-	// Returns nil if there is no entry in the data cap table for the
-	// address.
-	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) //perm:read
-	// StateVerifiedClientStatus returns the address of the Verified Registry's root key
-	StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) //perm:read
-	// StateDealProviderCollateralBounds returns the min and max collateral a storage provider
-	// can issue. It takes the deal size and verified status as parameters.
-	StateDealProviderCollateralBounds(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error) //perm:read
-
-	// StateCirculatingSupply returns the exact circulating supply of Filecoin at the given tipset.
-	// This is not used anywhere in the protocol itself, and is only for external consumption.
-	StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error) //perm:read
-	// StateVMCirculatingSupplyInternal returns an approximation of the circulating supply of Filecoin at the given tipset.
-	// This is the value reported by the runtime interface to actors code.
-	StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error) //perm:read
-	// StateNetworkVersion returns the network version at the given tipset
-	StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) //perm:read
-
-	// MethodGroup: Msig
-	// The Msig methods are used to interact with multisig wallets on the
-	// filecoin network
-
-	// MsigGetAvailableBalance returns the portion of a multisig's balance that can be withdrawn or spent
-	MsigGetAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) //perm:read
-	// MsigGetVestingSchedule returns the vesting details of a given multisig.
-	MsigGetVestingSchedule(context.Context, address.Address, types.TipSetKey) (api.MsigVesting, error) //perm:read
-	// MsigGetVested returns the amount of FIL that vested in a multisig in a certain period.
-	// It takes the following params: <multisig address>, <start epoch>, <end epoch>
-	MsigGetVested(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) //perm:read
-
-	//MsigGetPending returns pending transactions for the given multisig
-	//wallet. Once pending transactions are fully approved, they will no longer
-	//appear here.
-	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*api.MsigTransaction, error) //perm:read
-
-	// MsigCreate creates a multisig wallet
-	// It takes the following params: <required number of senders>, <approving addresses>, <unlock duration>
-	//<initial balance>, <sender address of the create msg>, <gas price>
-	MsigCreate(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error) //perm:sign
-	// MsigPropose proposes a multisig message
-	// It takes the following params: <multisig address>, <recipient address>, <value to transfer>,
-	// <sender address of the propose msg>, <method to call in the proposed message>, <params to include in the proposed message>
-	MsigPropose(context.Context, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) //perm:sign
-
-	// MsigApprove approves a previously-proposed multisig message by transaction ID
-	// It takes the following params: <multisig address>, <proposed transaction ID> <signer address>
-	MsigApprove(context.Context, address.Address, uint64, address.Address) (cid.Cid, error) //perm:sign
-
-	// MsigApproveTxnHash approves a previously-proposed multisig message, specified
-	// using both transaction ID and a hash of the parameters used in the
-	// proposal. This method of approval can be used to ensure you only approve
-	// exactly the transaction you think you are.
-	// It takes the following params: <multisig address>, <proposed message ID>, <proposer address>, <recipient address>, <value to transfer>,
-	// <sender address of the approve msg>, <method to call in the approved message>, <params to include in the proposed message>
-	MsigApproveTxnHash(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) //perm:sign
-
-	// MsigCancel cancels a previously-proposed multisig message
-	// It takes the following params: <multisig address>, <proposed transaction ID>, <recipient address>, <value to transfer>,
-	// <sender address of the cancel msg>, <method to call in the proposed message>, <params to include in the proposed message>
-	MsigCancel(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) //perm:sign
-	// MsigAddPropose proposes adding a signer in the multisig
-	// It takes the following params: <multisig address>, <sender address of the propose msg>,
-	// <new signer>, <whether the number of required signers should be increased>
-	MsigAddPropose(context.Context, address.Address, address.Address, address.Address, bool) (cid.Cid, error) //perm:sign
-	// MsigAddApprove approves a previously proposed AddSigner message
-	// It takes the following params: <multisig address>, <sender address of the approve msg>, <proposed message ID>,
-	// <proposer address>, <new signer>, <whether the number of required signers should be increased>
-	MsigAddApprove(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, bool) (cid.Cid, error) //perm:sign
-	// MsigAddCancel cancels a previously proposed AddSigner message
-	// It takes the following params: <multisig address>, <sender address of the cancel msg>, <proposed message ID>,
-	// <new signer>, <whether the number of required signers should be increased>
-	MsigAddCancel(context.Context, address.Address, address.Address, uint64, address.Address, bool) (cid.Cid, error) //perm:sign
-	// MsigSwapPropose proposes swapping 2 signers in the multisig
-	// It takes the following params: <multisig address>, <sender address of the propose msg>,
-	// <old signer>, <new signer>
-	MsigSwapPropose(context.Context, address.Address, address.Address, address.Address, address.Address) (cid.Cid, error) //perm:sign
-	// MsigSwapApprove approves a previously proposed SwapSigner
-	// It takes the following params: <multisig address>, <sender address of the approve msg>, <proposed message ID>,
-	// <proposer address>, <old signer>, <new signer>
-	MsigSwapApprove(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, address.Address) (cid.Cid, error) //perm:sign
-	// MsigSwapCancel cancels a previously proposed SwapSigner message
-	// It takes the following params: <multisig address>, <sender address of the cancel msg>, <proposed message ID>,
-	// <old signer>, <new signer>
-	MsigSwapCancel(context.Context, address.Address, address.Address, uint64, address.Address, address.Address) (cid.Cid, error) //perm:sign
-
-	// MsigRemoveSigner proposes the removal of a signer from the multisig.
-	// It accepts the multisig to make the change on, the proposer address to
-	// send the message from, the address to be removed, and a boolean
-	// indicating whether or not the signing threshold should be lowered by one
-	// along with the address removal.
-	MsigRemoveSigner(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error) //perm:sign
-
-	// MarketAddBalance adds funds to the market actor
-	MarketAddBalance(ctx context.Context, wallet, addr address.Address, amt types.BigInt) (cid.Cid, error) //perm:sign
-	// MarketGetReserved gets the amount of funds that are currently reserved for the address
-	MarketGetReserved(ctx context.Context, addr address.Address) (types.BigInt, error) //perm:sign
-	// MarketReserveFunds reserves funds for a deal
-	MarketReserveFunds(ctx context.Context, wallet address.Address, addr address.Address, amt types.BigInt) (cid.Cid, error) //perm:sign
-	// MarketReleaseFunds releases funds reserved by MarketReserveFunds
-	MarketReleaseFunds(ctx context.Context, addr address.Address, amt types.BigInt) error //perm:sign
-	// MarketWithdraw withdraws unlocked funds from the market actor
-	MarketWithdraw(ctx context.Context, wallet, addr address.Address, amt types.BigInt) (cid.Cid, error) //perm:sign
-
-	// MethodGroup: Paych
-	// The Paych methods are for interacting with and managing payment channels
-
-	PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*api.ChannelInfo, error)                  //perm:sign
-	PaychGetWaitReady(context.Context, cid.Cid) (address.Address, error)                                                 //perm:sign
-	PaychAvailableFunds(ctx context.Context, ch address.Address) (*api.ChannelAvailableFunds, error)                     //perm:sign
-	PaychAvailableFundsByFromTo(ctx context.Context, from, to address.Address) (*api.ChannelAvailableFunds, error)       //perm:sign
-	PaychList(context.Context) ([]address.Address, error)                                                                //perm:read
-	PaychStatus(context.Context, address.Address) (*api.PaychStatus, error)                                              //perm:read
-	PaychSettle(context.Context, address.Address) (cid.Cid, error)                                                       //perm:sign
-	PaychCollect(context.Context, address.Address) (cid.Cid, error)                                                      //perm:sign
-	PaychAllocateLane(ctx context.Context, ch address.Address) (uint64, error)                                           //perm:sign
-	PaychNewPayment(ctx context.Context, from, to address.Address, vouchers []api.VoucherSpec) (*api.PaymentInfo, error) //perm:sign
-	PaychVoucherCheckValid(context.Context, address.Address, *paych.SignedVoucher) error                                 //perm:read
-	PaychVoucherCheckSpendable(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (bool, error)     //perm:read
-	PaychVoucherCreate(context.Context, address.Address, types.BigInt, uint64) (*api.VoucherCreateResult, error)         //perm:sign
-	PaychVoucherAdd(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error)  //perm:write
-	PaychVoucherList(context.Context, address.Address) ([]*paych.SignedVoucher, error)                                   //perm:write
-	PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error)          //perm:sign
-
-	// CreateBackup creates node backup onder the specified file name. The
-	// method requires that the lotus daemon is running with the
-	// LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that
-	// the path specified when calling CreateBackup is within the base path
-	CreateBackup(ctx context.Context, fpath string) error //perm:admin
-}

api/v0api/gateway.go

@@ -1,69 +0,0 @@
-package v0api
-
-import (
-	"context"
-
-	"github.com/ipfs/go-cid"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/dline"
-	"github.com/filecoin-project/go-state-types/network"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-//                       MODIFYING THE API INTERFACE
-//
-// NOTE: This is the V0 (Stable) API - when adding methods to this interface,
-// you'll need to make sure they are also present on the V1 (Unstable) API
-//
-// This API is implemented in `v1_wrapper.go` as a compatibility layer backed
-// by the V1 api
-//
-// When adding / changing methods in this file:
-// * Do the change here
-// * Adjust implementation in `node/impl/`
-// * Run `make gen` - this will:
-//  * Generate proxy structs
-//  * Generate mocks
-//  * Generate markdown docs
-//  * Generate openrpc blobs
-
-type Gateway interface {
-	ChainHasObj(context.Context, cid.Cid) (bool, error)
-	ChainHead(ctx context.Context) (*types.TipSet, error)
-	ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error)
-	ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
-	ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error)
-	ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error)
-	ChainNotify(context.Context) (<-chan []*api.HeadChange, error)
-	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
-	GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
-	MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
-	MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
-	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
-	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*api.MsigTransaction, error)
-	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
-	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error)
-	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
-	StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)
-	StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
-	StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
-	StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error)
-	StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error)
-	StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error)
-	StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
-	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)
-	StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error)
-	StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error)
-	StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
-	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
-	StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error)
-	WalletBalance(context.Context, address.Address) (types.BigInt, error)
-	Version(context.Context) (api.APIVersion, error)
-}
-
-var _ Gateway = *new(FullNode)

api/v0api/latest.go

@@ -1,32 +0,0 @@
-package v0api
-
-import (
-	"github.com/filecoin-project/lotus/api"
-)
-
-type Common = api.Common
-type Net = api.Net
-type CommonNet = api.CommonNet
-
-type CommonStruct = api.CommonStruct
-type CommonStub = api.CommonStub
-type NetStruct = api.NetStruct
-type NetStub = api.NetStub
-type CommonNetStruct = api.CommonNetStruct
-type CommonNetStub = api.CommonNetStub
-
-type StorageMiner = api.StorageMiner
-type StorageMinerStruct = api.StorageMinerStruct
-
-type Worker = api.Worker
-type WorkerStruct = api.WorkerStruct
-
-type Wallet = api.Wallet
-
-func PermissionedStorMinerAPI(a StorageMiner) StorageMiner {
-	return api.PermissionedStorMinerAPI(a)
-}
-
-func PermissionedWorkerAPI(a Worker) Worker {
-	return api.PermissionedWorkerAPI(a)
-}

api/v0api/permissioned.go

@@ -1,13 +0,0 @@
-package v0api
-
-import (
-	"github.com/filecoin-project/go-jsonrpc/auth"
-	"github.com/filecoin-project/lotus/api"
-)
-
-func PermissionedFullAPI(a FullNode) FullNode {
-	var out FullNodeStruct
-	auth.PermissionedProxy(api.AllPermissions, api.DefaultPerms, a, &out.Internal)
-	auth.PermissionedProxy(api.AllPermissions, api.DefaultPerms, a, &out.CommonStruct.Internal)
-	return &out
-}

api/v0api/v1_wrapper.go

@@ -1,187 +0,0 @@
-package v0api
-
-import (
-	"context"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/lotus/chain/types"
-	"golang.org/x/xerrors"
-
-	"github.com/ipfs/go-cid"
-
-	"github.com/filecoin-project/go-state-types/abi"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/api/v1api"
-)
-
-type WrapperV1Full struct {
-	v1api.FullNode
-}
-
-func (w *WrapperV1Full) StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error) {
-	return w.FullNode.StateSearchMsg(ctx, types.EmptyTSK, msg, api.LookbackNoLimit, true)
-}
-
-func (w *WrapperV1Full) StateSearchMsgLimited(ctx context.Context, msg cid.Cid, limit abi.ChainEpoch) (*api.MsgLookup, error) {
-	return w.FullNode.StateSearchMsg(ctx, types.EmptyTSK, msg, limit, true)
-}
-
-func (w *WrapperV1Full) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) {
-	return w.FullNode.StateWaitMsg(ctx, msg, confidence, api.LookbackNoLimit, true)
-}
-
-func (w *WrapperV1Full) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, limit abi.ChainEpoch) (*api.MsgLookup, error) {
-	return w.FullNode.StateWaitMsg(ctx, msg, confidence, limit, true)
-}
-
-func (w *WrapperV1Full) StateGetReceipt(ctx context.Context, msg cid.Cid, from types.TipSetKey) (*types.MessageReceipt, error) {
-	ml, err := w.FullNode.StateSearchMsg(ctx, from, msg, api.LookbackNoLimit, true)
-	if err != nil {
-		return nil, err
-	}
-
-	if ml == nil {
-		return nil, nil
-	}
-
-	return &ml.Receipt, nil
-}
-
-func (w *WrapperV1Full) Version(ctx context.Context) (api.APIVersion, error) {
-	ver, err := w.FullNode.Version(ctx)
-	if err != nil {
-		return api.APIVersion{}, err
-	}
-
-	ver.APIVersion = api.FullAPIVersion0
-
-	return ver, nil
-}
-
-func (w *WrapperV1Full) executePrototype(ctx context.Context, p *api.MessagePrototype) (cid.Cid, error) {
-	sm, err := w.FullNode.MpoolPushMessage(ctx, &p.Message, nil)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("pushing message: %w", err)
-	}
-
-	return sm.Cid(), nil
-}
-func (w *WrapperV1Full) MsigCreate(ctx context.Context, req uint64, addrs []address.Address, duration abi.ChainEpoch, val types.BigInt, src address.Address, gp types.BigInt) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigCreate(ctx, req, addrs, duration, val, src, gp)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigPropose(ctx context.Context, msig address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigPropose(ctx, msig, to, amt, src, method, params)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-func (w *WrapperV1Full) MsigApprove(ctx context.Context, msig address.Address, txID uint64, src address.Address) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigApprove(ctx, msig, txID, src)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigApproveTxnHash(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) {
-	p, err := w.FullNode.MsigApproveTxnHash(ctx, msig, txID, proposer, to, amt, src, method, params)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigCancel(ctx context.Context, msig address.Address, txID uint64, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) {
-	p, err := w.FullNode.MsigCancel(ctx, msig, txID, to, amt, src, method, params)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigAddPropose(ctx context.Context, msig address.Address, src address.Address, newAdd address.Address, inc bool) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigAddPropose(ctx, msig, src, newAdd, inc)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigAddApprove(ctx context.Context, msig address.Address, src address.Address, txID uint64, proposer address.Address, newAdd address.Address, inc bool) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigAddApprove(ctx, msig, src, txID, proposer, newAdd, inc)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigAddCancel(ctx context.Context, msig address.Address, src address.Address, txID uint64, newAdd address.Address, inc bool) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigAddCancel(ctx, msig, src, txID, newAdd, inc)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigSwapPropose(ctx context.Context, msig address.Address, src address.Address, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigSwapPropose(ctx, msig, src, oldAdd, newAdd)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigSwapApprove(ctx context.Context, msig address.Address, src address.Address, txID uint64, proposer address.Address, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigSwapApprove(ctx, msig, src, txID, proposer, oldAdd, newAdd)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigSwapCancel(ctx context.Context, msig address.Address, src address.Address, txID uint64, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigSwapCancel(ctx, msig, src, txID, oldAdd, newAdd)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigRemoveSigner(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigRemoveSigner(ctx, msig, proposer, toRemove, decrease)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-var _ FullNode = &WrapperV1Full{}

api/v1api/latest.go

@@ -1,12 +0,0 @@
-package v1api
-
-import (
-	"github.com/filecoin-project/lotus/api"
-)
-
-type FullNode = api.FullNode
-type FullNodeStruct = api.FullNodeStruct
-
-func PermissionedFullAPI(a FullNode) FullNode {
-	return api.PermissionedFullAPI(a)
-}

api/version.go

@@ -1,73 +0,0 @@
-package api
-
-import (
-	"fmt"
-
-	xerrors "golang.org/x/xerrors"
-)
-
-type Version uint32
-
-func newVer(major, minor, patch uint8) Version {
-	return Version(uint32(major)<<16 | uint32(minor)<<8 | uint32(patch))
-}
-
-// Ints returns (major, minor, patch) versions
-func (ve Version) Ints() (uint32, uint32, uint32) {
-	v := uint32(ve)
-	return (v & majorOnlyMask) >> 16, (v & minorOnlyMask) >> 8, v & patchOnlyMask
-}
-
-func (ve Version) String() string {
-	vmj, vmi, vp := ve.Ints()
-	return fmt.Sprintf("%d.%d.%d", vmj, vmi, vp)
-}
-
-func (ve Version) EqMajorMinor(v2 Version) bool {
-	return ve&minorMask == v2&minorMask
-}
-
-type NodeType int
-
-const (
-	NodeUnknown NodeType = iota
-
-	NodeFull
-	NodeMiner
-	NodeWorker
-)
-
-var RunningNodeType NodeType
-
-func VersionForType(nodeType NodeType) (Version, error) {
-	switch nodeType {
-	case NodeFull:
-		return FullAPIVersion1, nil
-	case NodeMiner:
-		return MinerAPIVersion0, nil
-	case NodeWorker:
-		return WorkerAPIVersion0, nil
-	default:
-		return Version(0), xerrors.Errorf("unknown node type %d", nodeType)
-	}
-}
-
-// semver versions of the rpc api exposed
-var (
-	FullAPIVersion0 = newVer(1, 3, 1)
-	FullAPIVersion1 = newVer(2, 1, 1)
-
-	MinerAPIVersion0  = newVer(1, 2, 0)
-	WorkerAPIVersion0 = newVer(1, 1, 0)
-)
-
-//nolint:varcheck,deadcode
-const (
-	majorMask = 0xff0000
-	minorMask = 0xffff00
-	patchMask = 0xffffff
-
-	majorOnlyMask = 0xff0000
-	minorOnlyMask = 0x00ff00
-	patchOnlyMask = 0x0000ff
-)

api/wrap.go

@@ -1,53 +0,0 @@
-package api
-
-import (
-	"reflect"
-)
-
-// Wrap adapts partial api impl to another version
-// proxyT is the proxy type used as input in wrapperT
-// Usage: Wrap(new(v1api.FullNodeStruct), new(v0api.WrapperV1Full), eventsApi).(EventAPI)
-func Wrap(proxyT, wrapperT, impl interface{}) interface{} {
-	proxy := reflect.New(reflect.TypeOf(proxyT).Elem())
-	proxyMethods := proxy.Elem().FieldByName("Internal")
-	ri := reflect.ValueOf(impl)
-
-	for i := 0; i < ri.NumMethod(); i++ {
-		mt := ri.Type().Method(i)
-		if proxyMethods.FieldByName(mt.Name).Kind() == reflect.Invalid {
-			continue
-		}
-
-		fn := ri.Method(i)
-		of := proxyMethods.FieldByName(mt.Name)
-
-		proxyMethods.FieldByName(mt.Name).Set(reflect.MakeFunc(of.Type(), func(args []reflect.Value) (results []reflect.Value) {
-			return fn.Call(args)
-		}))
-	}
-
-	for i := 0; i < proxy.Elem().NumField(); i++ {
-		if proxy.Elem().Type().Field(i).Name == "Internal" {
-			continue
-		}
-
-		subProxy := proxy.Elem().Field(i).FieldByName("Internal")
-		for i := 0; i < ri.NumMethod(); i++ {
-			mt := ri.Type().Method(i)
-			if subProxy.FieldByName(mt.Name).Kind() == reflect.Invalid {
-				continue
-			}
-
-			fn := ri.Method(i)
-			of := subProxy.FieldByName(mt.Name)
-
-			subProxy.FieldByName(mt.Name).Set(reflect.MakeFunc(of.Type(), func(args []reflect.Value) (results []reflect.Value) {
-				return fn.Call(args)
-			}))
-		}
-	}
-
-	wp := reflect.New(reflect.TypeOf(wrapperT).Elem())
-	wp.Elem().Field(0).Set(proxy)
-	return wp.Interface()
-}

blockstore/api.go

@@ -1,66 +0,0 @@
-package blockstore
-
-import (
-	"context"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	"golang.org/x/xerrors"
-)
-
-type ChainIO interface {
-	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
-	ChainHasObj(context.Context, cid.Cid) (bool, error)
-}
-
-type apiBlockstore struct {
-	api ChainIO
-}
-
-// This blockstore is adapted in the constructor.
-var _ BasicBlockstore = (*apiBlockstore)(nil)
-
-func NewAPIBlockstore(cio ChainIO) Blockstore {
-	bs := &apiBlockstore{api: cio}
-	return Adapt(bs) // return an adapted blockstore.
-}
-
-func (a *apiBlockstore) DeleteBlock(cid.Cid) error {
-	return xerrors.New("not supported")
-}
-
-func (a *apiBlockstore) Has(c cid.Cid) (bool, error) {
-	return a.api.ChainHasObj(context.TODO(), c)
-}
-
-func (a *apiBlockstore) Get(c cid.Cid) (blocks.Block, error) {
-	bb, err := a.api.ChainReadObj(context.TODO(), c)
-	if err != nil {
-		return nil, err
-	}
-	return blocks.NewBlockWithCid(bb, c)
-}
-
-func (a *apiBlockstore) GetSize(c cid.Cid) (int, error) {
-	bb, err := a.api.ChainReadObj(context.TODO(), c)
-	if err != nil {
-		return 0, err
-	}
-	return len(bb), nil
-}
-
-func (a *apiBlockstore) Put(blocks.Block) error {
-	return xerrors.New("not supported")
-}
-
-func (a *apiBlockstore) PutMany([]blocks.Block) error {
-	return xerrors.New("not supported")
-}
-
-func (a *apiBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return nil, xerrors.New("not supported")
-}
-
-func (a *apiBlockstore) HashOnRead(enabled bool) {
-	return
-}

blockstore/badger/blockstore.go

@@ -1,985 +0,0 @@
-package badgerbs
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"runtime"
-	"sync"
-	"time"
-
-	"github.com/dgraph-io/badger/v2"
-	"github.com/dgraph-io/badger/v2/options"
-	"github.com/dgraph-io/badger/v2/pb"
-	"github.com/multiformats/go-base32"
-	"go.uber.org/zap"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	logger "github.com/ipfs/go-log/v2"
-	pool "github.com/libp2p/go-buffer-pool"
-
-	"github.com/filecoin-project/lotus/blockstore"
-)
-
-var (
-	// KeyPool is the buffer pool we use to compute storage keys.
-	KeyPool *pool.BufferPool = pool.GlobalPool
-)
-
-var (
-	// ErrBlockstoreClosed is returned from blockstore operations after
-	// the blockstore has been closed.
-	ErrBlockstoreClosed = fmt.Errorf("badger blockstore closed")
-
-	log = logger.Logger("badgerbs")
-)
-
-// aliases to mask badger dependencies.
-const (
-	// FileIO is equivalent to badger/options.FileIO.
-	FileIO = options.FileIO
-	// MemoryMap is equivalent to badger/options.MemoryMap.
-	MemoryMap = options.MemoryMap
-	// LoadToRAM is equivalent to badger/options.LoadToRAM.
-	LoadToRAM = options.LoadToRAM
-)
-
-// Options embeds the badger options themselves, and augments them with
-// blockstore-specific options.
-type Options struct {
-	badger.Options
-
-	// Prefix is an optional prefix to prepend to keys. Default: "".
-	Prefix string
-}
-
-func DefaultOptions(path string) Options {
-	return Options{
-		Options: badger.DefaultOptions(path),
-		Prefix:  "",
-	}
-}
-
-// badgerLogger is a local wrapper for go-log to make the interface
-// compatible with badger.Logger (namely, aliasing Warnf to Warningf)
-type badgerLogger struct {
-	*zap.SugaredLogger // skips 1 caller to get useful line info, skipping over badger.Options.
-
-	skip2 *zap.SugaredLogger // skips 2 callers, just like above + this logger.
-}
-
-// Warningf is required by the badger logger APIs.
-func (b *badgerLogger) Warningf(format string, args ...interface{}) {
-	b.skip2.Warnf(format, args...)
-}
-
-// bsState is the current blockstore state
-type bsState int
-
-const (
-	// stateOpen signifies an open blockstore
-	stateOpen bsState = iota
-	// stateClosing signifies a blockstore that is currently closing
-	stateClosing
-	// stateClosed signifies a blockstore that has been colosed
-	stateClosed
-)
-
-type bsMoveState int
-
-const (
-	// moveStateNone signifies that there is no move in progress
-	moveStateNone bsMoveState = iota
-	// moveStateMoving signifies that there is a move  in a progress
-	moveStateMoving
-	// moveStateCleanup signifies that a move has completed or aborted and we are cleaning up
-	moveStateCleanup
-	// moveStateLock signifies that an exclusive lock has been acquired
-	moveStateLock
-)
-
-// Blockstore is a badger-backed IPLD blockstore.
-type Blockstore struct {
-	stateLk sync.RWMutex
-	state   bsState
-	viewers sync.WaitGroup
-
-	moveMx    sync.Mutex
-	moveCond  sync.Cond
-	moveState bsMoveState
-	rlock     int
-
-	db     *badger.DB
-	dbNext *badger.DB // when moving
-	opts   Options
-
-	prefixing bool
-	prefix    []byte
-	prefixLen int
-}
-
-var _ blockstore.Blockstore = (*Blockstore)(nil)
-var _ blockstore.Viewer = (*Blockstore)(nil)
-var _ blockstore.BlockstoreIterator = (*Blockstore)(nil)
-var _ blockstore.BlockstoreGC = (*Blockstore)(nil)
-var _ blockstore.BlockstoreSize = (*Blockstore)(nil)
-var _ io.Closer = (*Blockstore)(nil)
-
-// Open creates a new badger-backed blockstore, with the supplied options.
-func Open(opts Options) (*Blockstore, error) {
-	opts.Logger = &badgerLogger{
-		SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(),
-		skip2:         log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(),
-	}
-
-	db, err := badger.Open(opts.Options)
-	if err != nil {
-		return nil, fmt.Errorf("failed to open badger blockstore: %w", err)
-	}
-
-	bs := &Blockstore{db: db, opts: opts}
-	if p := opts.Prefix; p != "" {
-		bs.prefixing = true
-		bs.prefix = []byte(p)
-		bs.prefixLen = len(bs.prefix)
-	}
-
-	bs.moveCond.L = &bs.moveMx
-
-	return bs, nil
-}
-
-// Close closes the store. If the store has already been closed, this noops and
-// returns an error, even if the first closure resulted in error.
-func (b *Blockstore) Close() error {
-	b.stateLk.Lock()
-	if b.state != stateOpen {
-		b.stateLk.Unlock()
-		return nil
-	}
-	b.state = stateClosing
-	b.stateLk.Unlock()
-
-	defer func() {
-		b.stateLk.Lock()
-		b.state = stateClosed
-		b.stateLk.Unlock()
-	}()
-
-	// wait for all accesses to complete
-	b.viewers.Wait()
-
-	return b.db.Close()
-}
-
-func (b *Blockstore) access() error {
-	b.stateLk.RLock()
-	defer b.stateLk.RUnlock()
-
-	if b.state != stateOpen {
-		return ErrBlockstoreClosed
-	}
-
-	b.viewers.Add(1)
-	return nil
-}
-
-func (b *Blockstore) isOpen() bool {
-	b.stateLk.RLock()
-	defer b.stateLk.RUnlock()
-
-	return b.state == stateOpen
-}
-
-// lockDB/unlockDB implement a recursive lock contingent on move state
-func (b *Blockstore) lockDB() {
-	b.moveMx.Lock()
-	defer b.moveMx.Unlock()
-
-	if b.rlock == 0 {
-		for b.moveState == moveStateLock {
-			b.moveCond.Wait()
-		}
-	}
-
-	b.rlock++
-}
-
-func (b *Blockstore) unlockDB() {
-	b.moveMx.Lock()
-	defer b.moveMx.Unlock()
-
-	b.rlock--
-	if b.rlock == 0 && b.moveState == moveStateLock {
-		b.moveCond.Broadcast()
-	}
-}
-
-// lockMove/unlockMove implement an exclusive lock of move state
-func (b *Blockstore) lockMove() {
-	b.moveMx.Lock()
-	b.moveState = moveStateLock
-	for b.rlock > 0 {
-		b.moveCond.Wait()
-	}
-}
-
-func (b *Blockstore) unlockMove(state bsMoveState) {
-	b.moveState = state
-	b.moveCond.Broadcast()
-	b.moveMx.Unlock()
-}
-
-// movingGC moves the blockstore to a new path, adjacent to the current path, and creates
-// a symlink from the current path to the new path; the old blockstore is deleted.
-//
-// The blockstore MUST accept new writes during the move and ensure that these
-// are persisted to the new blockstore; if a failure occurs aboring the move,
-// then they must be peristed to the old blockstore.
-// In short, the blockstore must not lose data from new writes during the move.
-func (b *Blockstore) movingGC() error {
-	// this inlines moveLock/moveUnlock for the initial state check to prevent a second move
-	// while one is in progress without clobbering state
-	b.moveMx.Lock()
-	if b.moveState != moveStateNone {
-		b.moveMx.Unlock()
-		return fmt.Errorf("move in progress")
-	}
-
-	b.moveState = moveStateLock
-	for b.rlock > 0 {
-		b.moveCond.Wait()
-	}
-
-	b.moveState = moveStateMoving
-	b.moveCond.Broadcast()
-	b.moveMx.Unlock()
-
-	var newPath string
-
-	defer func() {
-		b.lockMove()
-
-		dbNext := b.dbNext
-		b.dbNext = nil
-
-		var state bsMoveState
-		if dbNext != nil {
-			state = moveStateCleanup
-		} else {
-			state = moveStateNone
-		}
-
-		b.unlockMove(state)
-
-		if dbNext != nil {
-			// the move failed and we have a left-over db; delete it.
-			err := dbNext.Close()
-			if err != nil {
-				log.Warnf("error closing badger db: %s", err)
-			}
-			b.deleteDB(newPath)
-
-			b.lockMove()
-			b.unlockMove(moveStateNone)
-		}
-	}()
-
-	// we resolve symlinks to create the new path in the adjacent to the old path.
-	// this allows the user to symlink the db directory into a separate filesystem.
-	basePath := b.opts.Dir
-	linkPath, err := filepath.EvalSymlinks(basePath)
-	if err != nil {
-		return fmt.Errorf("error resolving symlink %s: %w", basePath, err)
-	}
-
-	if basePath == linkPath {
-		newPath = basePath
-	} else {
-		// we do this dance to create a name adjacent to the current one, while avoiding clown
-		// shoes with multiple moves (i.e. we can't just take the basename of the linkPath, as it
-		// could have been created in a previous move and have the timestamp suffix, which would then
-		// perpetuate itself.
-		name := filepath.Base(basePath)
-		dir := filepath.Dir(linkPath)
-		newPath = filepath.Join(dir, name)
-	}
-	newPath = fmt.Sprintf("%s.%d", newPath, time.Now().UnixNano())
-
-	log.Infof("moving blockstore from %s to %s", b.opts.Dir, newPath)
-
-	opts := b.opts
-	opts.Dir = newPath
-	opts.ValueDir = newPath
-
-	dbNew, err := badger.Open(opts.Options)
-	if err != nil {
-		return fmt.Errorf("failed to open badger blockstore in %s: %w", newPath, err)
-	}
-
-	b.lockMove()
-	b.dbNext = dbNew
-	b.unlockMove(moveStateMoving)
-
-	log.Info("copying blockstore")
-	err = b.doCopy(b.db, b.dbNext)
-	if err != nil {
-		return fmt.Errorf("error moving badger blockstore to %s: %w", newPath, err)
-	}
-
-	b.lockMove()
-	dbOld := b.db
-	b.db = b.dbNext
-	b.dbNext = nil
-	b.unlockMove(moveStateCleanup)
-
-	err = dbOld.Close()
-	if err != nil {
-		log.Warnf("error closing old badger db: %s", err)
-	}
-
-	// this is the canonical db path; this is where our db lives.
-	dbPath := b.opts.Dir
-
-	// we first move the existing db out of the way, and only delete it after we have symlinked the
-	// new db to the canonical path
-	backupPath := fmt.Sprintf("%s.old.%d", dbPath, time.Now().Unix())
-	if err = os.Rename(dbPath, backupPath); err != nil {
-		// this is not catastrophic in the sense that we have not lost any data.
-		// but it is pretty bad, as the db path points to the old db, while we are now using to the new
-		// db; we can't continue and leave a ticking bomb for the next restart.
-		// so a panic is appropriate and user can fix.
-		panic(fmt.Errorf("error renaming old badger db dir from %s to %s: %w; USER ACTION REQUIRED", dbPath, backupPath, err)) //nolint
-	}
-
-	if err = symlink(newPath, dbPath); err != nil {
-		// same here; the db path is pointing to the void. panic and let the user fix.
-		panic(fmt.Errorf("error symlinking new badger db dir from %s to %s: %w; USER ACTION REQUIRED", newPath, dbPath, err)) //nolint
-	}
-
-	// the delete follows symlinks
-	b.deleteDB(backupPath)
-
-	log.Info("moving blockstore done")
-	return nil
-}
-
-// symlink creates a symlink from path to linkTo; the link is relative if the two are
-// in the same directory
-func symlink(path, linkTo string) error {
-	resolvedPathDir, err := filepath.EvalSymlinks(filepath.Dir(path))
-	if err != nil {
-		return fmt.Errorf("error resolving links in %s: %w", path, err)
-	}
-
-	resolvedLinkDir, err := filepath.EvalSymlinks(filepath.Dir(linkTo))
-	if err != nil {
-		return fmt.Errorf("error resolving links in %s: %w", linkTo, err)
-	}
-
-	if resolvedPathDir == resolvedLinkDir {
-		path = filepath.Base(path)
-	}
-
-	return os.Symlink(path, linkTo)
-}
-
-// doCopy copies a badger blockstore to another, with an optional filter; if the filter
-// is not nil, then only cids that satisfy the filter will be copied.
-func (b *Blockstore) doCopy(from, to *badger.DB) error {
-	workers := runtime.NumCPU() / 2
-	if workers < 2 {
-		workers = 2
-	}
-
-	stream := from.NewStream()
-	stream.NumGo = workers
-	stream.LogPrefix = "doCopy"
-	stream.Send = func(list *pb.KVList) error {
-		batch := to.NewWriteBatch()
-		defer batch.Cancel()
-
-		for _, kv := range list.Kv {
-			if kv.Key == nil || kv.Value == nil {
-				continue
-			}
-			if err := batch.Set(kv.Key, kv.Value); err != nil {
-				return err
-			}
-		}
-
-		return batch.Flush()
-	}
-
-	return stream.Orchestrate(context.Background())
-}
-
-func (b *Blockstore) deleteDB(path string) {
-	// follow symbolic links, otherwise the data wil be left behind
-	linkPath, err := filepath.EvalSymlinks(path)
-	if err != nil {
-		log.Warnf("error resolving symlinks in %s", path)
-		return
-	}
-
-	log.Infof("removing data directory %s", linkPath)
-	if err := os.RemoveAll(linkPath); err != nil {
-		log.Warnf("error deleting db at %s: %s", linkPath, err)
-		return
-	}
-
-	if path != linkPath {
-		log.Infof("removing link %s", path)
-		if err := os.Remove(path); err != nil {
-			log.Warnf("error removing symbolic link %s", err)
-		}
-	}
-}
-
-func (b *Blockstore) onlineGC() error {
-	b.lockDB()
-	defer b.unlockDB()
-
-	// compact first to gather the necessary statistics for GC
-	nworkers := runtime.NumCPU() / 2
-	if nworkers < 2 {
-		nworkers = 2
-	}
-
-	err := b.db.Flatten(nworkers)
-	if err != nil {
-		return err
-	}
-
-	for err == nil {
-		err = b.db.RunValueLogGC(0.125)
-	}
-
-	if err == badger.ErrNoRewrite {
-		// not really an error in this case, it signals the end of GC
-		return nil
-	}
-
-	return err
-}
-
-// CollectGarbage compacts and runs garbage collection on the value log;
-// implements the BlockstoreGC trait
-func (b *Blockstore) CollectGarbage(opts ...blockstore.BlockstoreGCOption) error {
-	if err := b.access(); err != nil {
-		return err
-	}
-	defer b.viewers.Done()
-
-	var options blockstore.BlockstoreGCOptions
-	for _, opt := range opts {
-		err := opt(&options)
-		if err != nil {
-			return err
-		}
-	}
-
-	if options.FullGC {
-		return b.movingGC()
-	}
-
-	return b.onlineGC()
-}
-
-// Size returns the aggregate size of the blockstore
-func (b *Blockstore) Size() (int64, error) {
-	if err := b.access(); err != nil {
-		return 0, err
-	}
-	defer b.viewers.Done()
-
-	b.lockDB()
-	defer b.unlockDB()
-
-	lsm, vlog := b.db.Size()
-	size := lsm + vlog
-
-	if size == 0 {
-		// badger reports a 0 size on symlinked directories... sigh
-		dir := b.opts.Dir
-		entries, err := os.ReadDir(dir)
-		if err != nil {
-			return 0, err
-		}
-
-		for _, e := range entries {
-			path := filepath.Join(dir, e.Name())
-			finfo, err := os.Stat(path)
-			if err != nil {
-				return 0, err
-			}
-			size += finfo.Size()
-		}
-	}
-
-	return size, nil
-}
-
-// View implements blockstore.Viewer, which leverages zero-copy read-only
-// access to values.
-func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
-	if err := b.access(); err != nil {
-		return err
-	}
-	defer b.viewers.Done()
-
-	b.lockDB()
-	defer b.unlockDB()
-
-	k, pooled := b.PooledStorageKey(cid)
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	return b.db.View(func(txn *badger.Txn) error {
-		switch item, err := txn.Get(k); err {
-		case nil:
-			return item.Value(fn)
-		case badger.ErrKeyNotFound:
-			return blockstore.ErrNotFound
-		default:
-			return fmt.Errorf("failed to view block from badger blockstore: %w", err)
-		}
-	})
-}
-
-// Has implements Blockstore.Has.
-func (b *Blockstore) Has(cid cid.Cid) (bool, error) {
-	if err := b.access(); err != nil {
-		return false, err
-	}
-	defer b.viewers.Done()
-
-	b.lockDB()
-	defer b.unlockDB()
-
-	k, pooled := b.PooledStorageKey(cid)
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	err := b.db.View(func(txn *badger.Txn) error {
-		_, err := txn.Get(k)
-		return err
-	})
-
-	switch err {
-	case badger.ErrKeyNotFound:
-		return false, nil
-	case nil:
-		return true, nil
-	default:
-		return false, fmt.Errorf("failed to check if block exists in badger blockstore: %w", err)
-	}
-}
-
-// Get implements Blockstore.Get.
-func (b *Blockstore) Get(cid cid.Cid) (blocks.Block, error) {
-	if !cid.Defined() {
-		return nil, blockstore.ErrNotFound
-	}
-
-	if err := b.access(); err != nil {
-		return nil, err
-	}
-	defer b.viewers.Done()
-
-	b.lockDB()
-	defer b.unlockDB()
-
-	k, pooled := b.PooledStorageKey(cid)
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	var val []byte
-	err := b.db.View(func(txn *badger.Txn) error {
-		switch item, err := txn.Get(k); err {
-		case nil:
-			val, err = item.ValueCopy(nil)
-			return err
-		case badger.ErrKeyNotFound:
-			return blockstore.ErrNotFound
-		default:
-			return fmt.Errorf("failed to get block from badger blockstore: %w", err)
-		}
-	})
-	if err != nil {
-		return nil, err
-	}
-	return blocks.NewBlockWithCid(val, cid)
-}
-
-// GetSize implements Blockstore.GetSize.
-func (b *Blockstore) GetSize(cid cid.Cid) (int, error) {
-	if err := b.access(); err != nil {
-		return 0, err
-	}
-	defer b.viewers.Done()
-
-	b.lockDB()
-	defer b.unlockDB()
-
-	k, pooled := b.PooledStorageKey(cid)
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	var size int
-	err := b.db.View(func(txn *badger.Txn) error {
-		switch item, err := txn.Get(k); err {
-		case nil:
-			size = int(item.ValueSize())
-		case badger.ErrKeyNotFound:
-			return blockstore.ErrNotFound
-		default:
-			return fmt.Errorf("failed to get block size from badger blockstore: %w", err)
-		}
-		return nil
-	})
-	if err != nil {
-		size = -1
-	}
-	return size, err
-}
-
-// Put implements Blockstore.Put.
-func (b *Blockstore) Put(block blocks.Block) error {
-	if err := b.access(); err != nil {
-		return err
-	}
-	defer b.viewers.Done()
-
-	b.lockDB()
-	defer b.unlockDB()
-
-	k, pooled := b.PooledStorageKey(block.Cid())
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	put := func(db *badger.DB) error {
-		err := db.Update(func(txn *badger.Txn) error {
-			return txn.Set(k, block.RawData())
-		})
-		if err != nil {
-			return fmt.Errorf("failed to put block in badger blockstore: %w", err)
-		}
-
-		return nil
-	}
-
-	if err := put(b.db); err != nil {
-		return err
-	}
-
-	if b.dbNext != nil {
-		if err := put(b.dbNext); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// PutMany implements Blockstore.PutMany.
-func (b *Blockstore) PutMany(blocks []blocks.Block) error {
-	if err := b.access(); err != nil {
-		return err
-	}
-	defer b.viewers.Done()
-
-	b.lockDB()
-	defer b.unlockDB()
-
-	// toReturn tracks the byte slices to return to the pool, if we're using key
-	// prefixing. we can't return each slice to the pool after each Set, because
-	// badger holds on to the slice.
-	var toReturn [][]byte
-	if b.prefixing {
-		toReturn = make([][]byte, 0, len(blocks))
-		defer func() {
-			for _, b := range toReturn {
-				KeyPool.Put(b)
-			}
-		}()
-	}
-
-	keys := make([][]byte, 0, len(blocks))
-	for _, block := range blocks {
-		k, pooled := b.PooledStorageKey(block.Cid())
-		if pooled {
-			toReturn = append(toReturn, k)
-		}
-		keys = append(keys, k)
-	}
-
-	put := func(db *badger.DB) error {
-		batch := db.NewWriteBatch()
-		defer batch.Cancel()
-
-		for i, block := range blocks {
-			k := keys[i]
-			if err := batch.Set(k, block.RawData()); err != nil {
-				return err
-			}
-		}
-
-		err := batch.Flush()
-		if err != nil {
-			return fmt.Errorf("failed to put blocks in badger blockstore: %w", err)
-		}
-
-		return nil
-	}
-
-	if err := put(b.db); err != nil {
-		return err
-	}
-
-	if b.dbNext != nil {
-		if err := put(b.dbNext); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// DeleteBlock implements Blockstore.DeleteBlock.
-func (b *Blockstore) DeleteBlock(cid cid.Cid) error {
-	if err := b.access(); err != nil {
-		return err
-	}
-	defer b.viewers.Done()
-
-	b.lockDB()
-	defer b.unlockDB()
-
-	k, pooled := b.PooledStorageKey(cid)
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	return b.db.Update(func(txn *badger.Txn) error {
-		return txn.Delete(k)
-	})
-}
-
-func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
-	if err := b.access(); err != nil {
-		return err
-	}
-	defer b.viewers.Done()
-
-	b.lockDB()
-	defer b.unlockDB()
-
-	// toReturn tracks the byte slices to return to the pool, if we're using key
-	// prefixing. we can't return each slice to the pool after each Set, because
-	// badger holds on to the slice.
-	var toReturn [][]byte
-	if b.prefixing {
-		toReturn = make([][]byte, 0, len(cids))
-		defer func() {
-			for _, b := range toReturn {
-				KeyPool.Put(b)
-			}
-		}()
-	}
-
-	batch := b.db.NewWriteBatch()
-	defer batch.Cancel()
-
-	for _, cid := range cids {
-		k, pooled := b.PooledStorageKey(cid)
-		if pooled {
-			toReturn = append(toReturn, k)
-		}
-		if err := batch.Delete(k); err != nil {
-			return err
-		}
-	}
-
-	err := batch.Flush()
-	if err != nil {
-		err = fmt.Errorf("failed to delete blocks from badger blockstore: %w", err)
-	}
-	return err
-}
-
-// AllKeysChan implements Blockstore.AllKeysChan.
-func (b *Blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	if err := b.access(); err != nil {
-		return nil, err
-	}
-
-	b.lockDB()
-	defer b.unlockDB()
-
-	txn := b.db.NewTransaction(false)
-	opts := badger.IteratorOptions{PrefetchSize: 100}
-	if b.prefixing {
-		opts.Prefix = b.prefix
-	}
-	iter := txn.NewIterator(opts)
-
-	ch := make(chan cid.Cid)
-	go func() {
-		defer b.viewers.Done()
-		defer close(ch)
-		defer iter.Close()
-
-		// NewCidV1 makes a copy of the multihash buffer, so we can reuse it to
-		// contain allocs.
-		var buf []byte
-		for iter.Rewind(); iter.Valid(); iter.Next() {
-			if ctx.Err() != nil {
-				return // context has fired.
-			}
-			if !b.isOpen() {
-				// open iterators will run even after the database is closed...
-				return // closing, yield.
-			}
-			k := iter.Item().Key()
-			if b.prefixing {
-				k = k[b.prefixLen:]
-			}
-
-			if reqlen := base32.RawStdEncoding.DecodedLen(len(k)); len(buf) < reqlen {
-				buf = make([]byte, reqlen)
-			}
-			if n, err := base32.RawStdEncoding.Decode(buf, k); err == nil {
-				select {
-				case ch <- cid.NewCidV1(cid.Raw, buf[:n]):
-				case <-ctx.Done():
-					return
-				}
-			} else {
-				log.Warnf("failed to decode key %s in badger AllKeysChan; err: %s", k, err)
-			}
-		}
-	}()
-
-	return ch, nil
-}
-
-// Implementation of BlockstoreIterator interface
-func (b *Blockstore) ForEachKey(f func(cid.Cid) error) error {
-	if err := b.access(); err != nil {
-		return err
-	}
-	defer b.viewers.Done()
-
-	b.lockDB()
-	defer b.unlockDB()
-
-	txn := b.db.NewTransaction(false)
-	defer txn.Discard()
-
-	opts := badger.IteratorOptions{PrefetchSize: 100}
-	if b.prefixing {
-		opts.Prefix = b.prefix
-	}
-
-	iter := txn.NewIterator(opts)
-	defer iter.Close()
-
-	var buf []byte
-	for iter.Rewind(); iter.Valid(); iter.Next() {
-		if !b.isOpen() {
-			return ErrBlockstoreClosed
-		}
-
-		k := iter.Item().Key()
-		if b.prefixing {
-			k = k[b.prefixLen:]
-		}
-
-		klen := base32.RawStdEncoding.DecodedLen(len(k))
-		if klen > len(buf) {
-			buf = make([]byte, klen)
-		}
-
-		n, err := base32.RawStdEncoding.Decode(buf, k)
-		if err != nil {
-			return err
-		}
-
-		c := cid.NewCidV1(cid.Raw, buf[:n])
-
-		err = f(c)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// HashOnRead implements Blockstore.HashOnRead. It is not supported by this
-// blockstore.
-func (b *Blockstore) HashOnRead(_ bool) {
-	log.Warnf("called HashOnRead on badger blockstore; function not supported; ignoring")
-}
-
-// PooledStorageKey returns the storage key under which this CID is stored.
-//
-// The key is: prefix + base32_no_padding(cid.Hash)
-//
-// This method may return pooled byte slice, which MUST be returned to the
-// KeyPool if pooled=true, or a leak will occur.
-func (b *Blockstore) PooledStorageKey(cid cid.Cid) (key []byte, pooled bool) {
-	h := cid.Hash()
-	size := base32.RawStdEncoding.EncodedLen(len(h))
-	if !b.prefixing { // optimize for branch prediction.
-		k := pool.Get(size)
-		base32.RawStdEncoding.Encode(k, h)
-		return k, true // slicing upto length unnecessary; the pool has already done this.
-	}
-
-	size += b.prefixLen
-	k := pool.Get(size)
-	copy(k, b.prefix)
-	base32.RawStdEncoding.Encode(k[b.prefixLen:], h)
-	return k, true // slicing upto length unnecessary; the pool has already done this.
-}
-
-// Storage acts like PooledStorageKey, but attempts to write the storage key
-// into the provided slice. If the slice capacity is insufficient, it allocates
-// a new byte slice with enough capacity to accommodate the result. This method
-// returns the resulting slice.
-func (b *Blockstore) StorageKey(dst []byte, cid cid.Cid) []byte {
-	h := cid.Hash()
-	reqsize := base32.RawStdEncoding.EncodedLen(len(h)) + b.prefixLen
-	if reqsize > cap(dst) {
-		// passed slice is smaller than required size; create new.
-		dst = make([]byte, reqsize)
-	} else if reqsize > len(dst) {
-		// passed slice has enough capacity, but its length is
-		// restricted, expand.
-		dst = dst[:cap(dst)]
-	}
-
-	if b.prefixing { // optimize for branch prediction.
-		copy(dst, b.prefix)
-		base32.RawStdEncoding.Encode(dst[b.prefixLen:], h)
-	} else {
-		base32.RawStdEncoding.Encode(dst, h)
-	}
-	return dst[:reqsize]
-}
-
-// this method is added for lotus-shed needs
-// WARNING: THIS IS COMPLETELY UNSAFE; DONT USE THIS IN PRODUCTION CODE
-func (b *Blockstore) DB() *badger.DB {
-	return b.db
-}

blockstore/badger/blockstore_test.go

@@ -1,275 +0,0 @@
-package badgerbs
-
-import (
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-	"golang.org/x/sync/errgroup"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-
-	"github.com/filecoin-project/lotus/blockstore"
-)
-
-func TestBadgerBlockstore(t *testing.T) {
-	(&Suite{
-		NewBlockstore:  newBlockstore(DefaultOptions),
-		OpenBlockstore: openBlockstore(DefaultOptions),
-	}).RunTests(t, "non_prefixed")
-
-	prefixed := func(path string) Options {
-		opts := DefaultOptions(path)
-		opts.Prefix = "/prefixed/"
-		return opts
-	}
-
-	(&Suite{
-		NewBlockstore:  newBlockstore(prefixed),
-		OpenBlockstore: openBlockstore(prefixed),
-	}).RunTests(t, "prefixed")
-}
-
-func TestStorageKey(t *testing.T) {
-	bs, _ := newBlockstore(DefaultOptions)(t)
-	bbs := bs.(*Blockstore)
-	defer bbs.Close() //nolint:errcheck
-
-	cid1 := blocks.NewBlock([]byte("some data")).Cid()
-	cid2 := blocks.NewBlock([]byte("more data")).Cid()
-	cid3 := blocks.NewBlock([]byte("a little more data")).Cid()
-	require.NotEqual(t, cid1, cid2) // sanity check
-	require.NotEqual(t, cid2, cid3) // sanity check
-
-	// nil slice; let StorageKey allocate for us.
-	k1 := bbs.StorageKey(nil, cid1)
-	require.Len(t, k1, 55)
-	require.True(t, cap(k1) == len(k1))
-
-	// k1's backing array is reused.
-	k2 := bbs.StorageKey(k1, cid2)
-	require.Len(t, k2, 55)
-	require.True(t, cap(k2) == len(k1))
-
-	// bring k2 to len=0, and verify that its backing array gets reused
-	// (i.e. k1 and k2 are overwritten)
-	k3 := bbs.StorageKey(k2[:0], cid3)
-	require.Len(t, k3, 55)
-	require.True(t, cap(k3) == len(k3))
-
-	// backing array of k1 and k2 has been modified, i.e. memory is shared.
-	require.Equal(t, k3, k1)
-	require.Equal(t, k3, k2)
-}
-
-func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
-	return func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
-		tb.Helper()
-
-		path, err := ioutil.TempDir("", "")
-		if err != nil {
-			tb.Fatal(err)
-		}
-
-		db, err := Open(optsSupplier(path))
-		if err != nil {
-			tb.Fatal(err)
-		}
-
-		tb.Cleanup(func() {
-			_ = os.RemoveAll(path)
-		})
-
-		return db, path
-	}
-}
-
-func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) {
-	return func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) {
-		tb.Helper()
-		return Open(optsSupplier(path))
-	}
-}
-
-func testMove(t *testing.T, optsF func(string) Options) {
-	basePath, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	dbPath := filepath.Join(basePath, "db")
-
-	t.Cleanup(func() {
-		_ = os.RemoveAll(basePath)
-	})
-
-	db, err := Open(optsF(dbPath))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	defer db.Close() //nolint
-
-	var have []blocks.Block
-	var deleted []cid.Cid
-
-	// add some blocks
-	for i := 0; i < 10; i++ {
-		blk := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
-		err := db.Put(blk)
-		if err != nil {
-			t.Fatal(err)
-		}
-		have = append(have, blk)
-	}
-
-	// delete some of them
-	for i := 5; i < 10; i++ {
-		c := have[i].Cid()
-		err := db.DeleteBlock(c)
-		if err != nil {
-			t.Fatal(err)
-		}
-		deleted = append(deleted, c)
-	}
-	have = have[:5]
-
-	// start a move concurrent with some more puts
-	g := new(errgroup.Group)
-	g.Go(func() error {
-		for i := 10; i < 1000; i++ {
-			blk := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
-			err := db.Put(blk)
-			if err != nil {
-				return err
-			}
-			have = append(have, blk)
-		}
-		return nil
-	})
-	g.Go(func() error {
-		return db.CollectGarbage(blockstore.WithFullGC(true))
-	})
-
-	err = g.Wait()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// now check that we have all the blocks in have and none in the deleted lists
-	checkBlocks := func() {
-		for _, blk := range have {
-			has, err := db.Has(blk.Cid())
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if !has {
-				t.Fatal("missing block")
-			}
-
-			blk2, err := db.Get(blk.Cid())
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if !bytes.Equal(blk.RawData(), blk2.RawData()) {
-				t.Fatal("data mismatch")
-			}
-		}
-
-		for _, c := range deleted {
-			has, err := db.Has(c)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if has {
-				t.Fatal("resurrected block")
-			}
-		}
-	}
-
-	checkBlocks()
-
-	// check the basePath -- it should contain a directory with name db.{timestamp}, soft-linked
-	// to db and nothing else
-	checkPath := func() {
-		entries, err := os.ReadDir(basePath)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if len(entries) != 2 {
-			t.Fatalf("too many entries; expected %d but got %d", 2, len(entries))
-		}
-
-		var haveDB, haveDBLink bool
-		for _, e := range entries {
-			if e.Name() == "db" {
-				if (e.Type() & os.ModeSymlink) == 0 {
-					t.Fatal("found db, but it's not a symlink")
-				}
-				haveDBLink = true
-				continue
-			}
-			if strings.HasPrefix(e.Name(), "db.") {
-				if !e.Type().IsDir() {
-					t.Fatal("found db prefix, but it's not a directory")
-				}
-				haveDB = true
-				continue
-			}
-		}
-
-		if !haveDB {
-			t.Fatal("db directory is missing")
-		}
-		if !haveDBLink {
-			t.Fatal("db link is missing")
-		}
-	}
-
-	checkPath()
-
-	// now do another FullGC to test the double move and following of symlinks
-	if err := db.CollectGarbage(blockstore.WithFullGC(true)); err != nil {
-		t.Fatal(err)
-	}
-
-	checkBlocks()
-	checkPath()
-
-	// reopen the db to make sure our relative link works:
-	err = db.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	db, err = Open(optsF(dbPath))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// db.Close() is already deferred
-
-	checkBlocks()
-}
-
-func TestMoveNoPrefix(t *testing.T) {
-	testMove(t, DefaultOptions)
-}
-
-func TestMoveWithPrefix(t *testing.T) {
-	testMove(t, func(path string) Options {
-		opts := DefaultOptions(path)
-		opts.Prefix = "/prefixed/"
-		return opts
-	})
-}

blockstore/badger/blockstore_test_suite.go

@@ -1,313 +0,0 @@
-package badgerbs
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"reflect"
-	"strings"
-	"testing"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	u "github.com/ipfs/go-ipfs-util"
-
-	"github.com/filecoin-project/lotus/blockstore"
-
-	"github.com/stretchr/testify/require"
-)
-
-// TODO: move this to go-ipfs-blockstore.
-type Suite struct {
-	NewBlockstore  func(tb testing.TB) (bs blockstore.BasicBlockstore, path string)
-	OpenBlockstore func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error)
-}
-
-func (s *Suite) RunTests(t *testing.T, prefix string) {
-	v := reflect.TypeOf(s)
-	f := func(t *testing.T) {
-		for i := 0; i < v.NumMethod(); i++ {
-			if m := v.Method(i); strings.HasPrefix(m.Name, "Test") {
-				f := m.Func.Interface().(func(*Suite, *testing.T))
-				t.Run(m.Name, func(t *testing.T) {
-					f(s, t)
-				})
-			}
-		}
-	}
-
-	if prefix == "" {
-		f(t)
-	} else {
-		t.Run(prefix, f)
-	}
-}
-
-func (s *Suite) TestGetWhenKeyNotPresent(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	c := cid.NewCidV0(u.Hash([]byte("stuff")))
-	bl, err := bs.Get(c)
-	require.Nil(t, bl)
-	require.Equal(t, blockstore.ErrNotFound, err)
-}
-
-func (s *Suite) TestGetWhenKeyIsNil(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	_, err := bs.Get(cid.Undef)
-	require.Equal(t, blockstore.ErrNotFound, err)
-}
-
-func (s *Suite) TestPutThenGetBlock(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	orig := blocks.NewBlock([]byte("some data"))
-
-	err := bs.Put(orig)
-	require.NoError(t, err)
-
-	fetched, err := bs.Get(orig.Cid())
-	require.NoError(t, err)
-	require.Equal(t, orig.RawData(), fetched.RawData())
-}
-
-func (s *Suite) TestHas(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	orig := blocks.NewBlock([]byte("some data"))
-
-	err := bs.Put(orig)
-	require.NoError(t, err)
-
-	ok, err := bs.Has(orig.Cid())
-	require.NoError(t, err)
-	require.True(t, ok)
-
-	ok, err = bs.Has(blocks.NewBlock([]byte("another thing")).Cid())
-	require.NoError(t, err)
-	require.False(t, ok)
-}
-
-func (s *Suite) TestCidv0v1(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	orig := blocks.NewBlock([]byte("some data"))
-
-	err := bs.Put(orig)
-	require.NoError(t, err)
-
-	fetched, err := bs.Get(cid.NewCidV1(cid.DagProtobuf, orig.Cid().Hash()))
-	require.NoError(t, err)
-	require.Equal(t, orig.RawData(), fetched.RawData())
-}
-
-func (s *Suite) TestPutThenGetSizeBlock(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	block := blocks.NewBlock([]byte("some data"))
-	missingBlock := blocks.NewBlock([]byte("missingBlock"))
-	emptyBlock := blocks.NewBlock([]byte{})
-
-	err := bs.Put(block)
-	require.NoError(t, err)
-
-	blockSize, err := bs.GetSize(block.Cid())
-	require.NoError(t, err)
-	require.Len(t, block.RawData(), blockSize)
-
-	err = bs.Put(emptyBlock)
-	require.NoError(t, err)
-
-	emptySize, err := bs.GetSize(emptyBlock.Cid())
-	require.NoError(t, err)
-	require.Zero(t, emptySize)
-
-	missingSize, err := bs.GetSize(missingBlock.Cid())
-	require.Equal(t, blockstore.ErrNotFound, err)
-	require.Equal(t, -1, missingSize)
-}
-
-func (s *Suite) TestAllKeysSimple(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	keys := insertBlocks(t, bs, 100)
-
-	ctx := context.Background()
-	ch, err := bs.AllKeysChan(ctx)
-	require.NoError(t, err)
-	actual := collect(ch)
-
-	require.ElementsMatch(t, keys, actual)
-}
-
-func (s *Suite) TestAllKeysRespectsContext(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	_ = insertBlocks(t, bs, 100)
-
-	ctx, cancel := context.WithCancel(context.Background())
-	ch, err := bs.AllKeysChan(ctx)
-	require.NoError(t, err)
-
-	// consume 2, then cancel context.
-	v, ok := <-ch
-	require.NotEqual(t, cid.Undef, v)
-	require.True(t, ok)
-
-	v, ok = <-ch
-	require.NotEqual(t, cid.Undef, v)
-	require.True(t, ok)
-
-	cancel()
-	// pull one value out to avoid race
-	_, _ = <-ch
-
-	v, ok = <-ch
-	require.Equal(t, cid.Undef, v)
-	require.False(t, ok)
-}
-
-func (s *Suite) TestDoubleClose(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	c, ok := bs.(io.Closer)
-	if !ok {
-		t.SkipNow()
-	}
-	require.NoError(t, c.Close())
-	require.NoError(t, c.Close())
-}
-
-func (s *Suite) TestReopenPutGet(t *testing.T) {
-	bs, path := s.NewBlockstore(t)
-	c, ok := bs.(io.Closer)
-	if !ok {
-		t.SkipNow()
-	}
-
-	orig := blocks.NewBlock([]byte("some data"))
-	err := bs.Put(orig)
-	require.NoError(t, err)
-
-	err = c.Close()
-	require.NoError(t, err)
-
-	bs, err = s.OpenBlockstore(t, path)
-	require.NoError(t, err)
-
-	fetched, err := bs.Get(orig.Cid())
-	require.NoError(t, err)
-	require.Equal(t, orig.RawData(), fetched.RawData())
-
-	err = bs.(io.Closer).Close()
-	require.NoError(t, err)
-}
-
-func (s *Suite) TestPutMany(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	blks := []blocks.Block{
-		blocks.NewBlock([]byte("foo1")),
-		blocks.NewBlock([]byte("foo2")),
-		blocks.NewBlock([]byte("foo3")),
-	}
-	err := bs.PutMany(blks)
-	require.NoError(t, err)
-
-	for _, blk := range blks {
-		fetched, err := bs.Get(blk.Cid())
-		require.NoError(t, err)
-		require.Equal(t, blk.RawData(), fetched.RawData())
-
-		ok, err := bs.Has(blk.Cid())
-		require.NoError(t, err)
-		require.True(t, ok)
-	}
-
-	ch, err := bs.AllKeysChan(context.Background())
-	require.NoError(t, err)
-
-	cids := collect(ch)
-	require.Len(t, cids, 3)
-}
-
-func (s *Suite) TestDelete(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	blks := []blocks.Block{
-		blocks.NewBlock([]byte("foo1")),
-		blocks.NewBlock([]byte("foo2")),
-		blocks.NewBlock([]byte("foo3")),
-	}
-	err := bs.PutMany(blks)
-	require.NoError(t, err)
-
-	err = bs.DeleteBlock(blks[1].Cid())
-	require.NoError(t, err)
-
-	ch, err := bs.AllKeysChan(context.Background())
-	require.NoError(t, err)
-
-	cids := collect(ch)
-	require.Len(t, cids, 2)
-	require.ElementsMatch(t, cids, []cid.Cid{
-		cid.NewCidV1(cid.Raw, blks[0].Cid().Hash()),
-		cid.NewCidV1(cid.Raw, blks[2].Cid().Hash()),
-	})
-
-	has, err := bs.Has(blks[1].Cid())
-	require.NoError(t, err)
-	require.False(t, has)
-
-}
-
-func insertBlocks(t *testing.T, bs blockstore.BasicBlockstore, count int) []cid.Cid {
-	keys := make([]cid.Cid, count)
-	for i := 0; i < count; i++ {
-		block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
-		err := bs.Put(block)
-		require.NoError(t, err)
-		// NewBlock assigns a CIDv0; we convert it to CIDv1 because that's what
-		// the store returns.
-		keys[i] = cid.NewCidV1(cid.Raw, block.Multihash())
-	}
-	return keys
-}
-
-func collect(ch <-chan cid.Cid) []cid.Cid {
-	var keys []cid.Cid
-	for k := range ch {
-		keys = append(keys, k)
-	}
-	return keys
-}

blockstore/blockstore.go

@@ -1,125 +0,0 @@
-package blockstore
-
-import (
-	cid "github.com/ipfs/go-cid"
-	ds "github.com/ipfs/go-datastore"
-	logging "github.com/ipfs/go-log/v2"
-
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
-)
-
-var log = logging.Logger("blockstore")
-
-var ErrNotFound = blockstore.ErrNotFound
-
-// Blockstore is the blockstore interface used by Lotus. It is the union
-// of the basic go-ipfs blockstore, with other capabilities required by Lotus,
-// e.g. View or Sync.
-type Blockstore interface {
-	blockstore.Blockstore
-	blockstore.Viewer
-	BatchDeleter
-}
-
-// BasicBlockstore is an alias to the original IPFS Blockstore.
-type BasicBlockstore = blockstore.Blockstore
-
-type Viewer = blockstore.Viewer
-
-type BatchDeleter interface {
-	DeleteMany(cids []cid.Cid) error
-}
-
-// BlockstoreIterator is a trait for efficient iteration
-type BlockstoreIterator interface {
-	ForEachKey(func(cid.Cid) error) error
-}
-
-// BlockstoreGC is a trait for blockstores that support online garbage collection
-type BlockstoreGC interface {
-	CollectGarbage(options ...BlockstoreGCOption) error
-}
-
-// BlockstoreGCOption is a functional interface for controlling blockstore GC options
-type BlockstoreGCOption = func(*BlockstoreGCOptions) error
-
-// BlockstoreGCOptions is a struct with GC options
-type BlockstoreGCOptions struct {
-	FullGC bool
-}
-
-func WithFullGC(fullgc bool) BlockstoreGCOption {
-	return func(opts *BlockstoreGCOptions) error {
-		opts.FullGC = fullgc
-		return nil
-	}
-}
-
-// BlockstoreSize is a trait for on-disk blockstores that can report their size
-type BlockstoreSize interface {
-	Size() (int64, error)
-}
-
-// WrapIDStore wraps the underlying blockstore in an "identity" blockstore.
-// The ID store filters out all puts for blocks with CIDs using the "identity"
-// hash function. It also extracts inlined blocks from CIDs using the identity
-// hash function and returns them on get/has, ignoring the contents of the
-// blockstore.
-func WrapIDStore(bstore blockstore.Blockstore) Blockstore {
-	if is, ok := bstore.(*idstore); ok {
-		// already wrapped
-		return is
-	}
-
-	if bs, ok := bstore.(Blockstore); ok {
-		// we need to wrap our own because we don't want to neuter the DeleteMany method
-		// the underlying blockstore has implemented an (efficient) DeleteMany
-		return NewIDStore(bs)
-	}
-
-	// The underlying blockstore does not implement DeleteMany, so we need to shim it.
-	// This is less efficient as it'll iterate and perform single deletes.
-	return NewIDStore(Adapt(bstore))
-}
-
-// FromDatastore creates a new blockstore backed by the given datastore.
-func FromDatastore(dstore ds.Batching) Blockstore {
-	return WrapIDStore(blockstore.NewBlockstore(dstore))
-}
-
-type adaptedBlockstore struct {
-	blockstore.Blockstore
-}
-
-var _ Blockstore = (*adaptedBlockstore)(nil)
-
-func (a *adaptedBlockstore) View(cid cid.Cid, callback func([]byte) error) error {
-	blk, err := a.Get(cid)
-	if err != nil {
-		return err
-	}
-	return callback(blk.RawData())
-}
-
-func (a *adaptedBlockstore) DeleteMany(cids []cid.Cid) error {
-	for _, cid := range cids {
-		err := a.DeleteBlock(cid)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// Adapt adapts a standard blockstore to a Lotus blockstore by
-// enriching it with the extra methods that Lotus requires (e.g. View, Sync).
-//
-// View proxies over to Get and calls the callback with the value supplied by Get.
-// Sync noops.
-func Adapt(bs blockstore.Blockstore) Blockstore {
-	if ret, ok := bs.(Blockstore); ok {
-		return ret
-	}
-	return &adaptedBlockstore{bs}
-}

blockstore/buffered.go

@@ -1,174 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"os"
-
-	block "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-)
-
-// buflog is a logger for the buffered blockstore. It is subscoped from the
-// blockstore logger.
-var buflog = log.Named("buf")
-
-type BufferedBlockstore struct {
-	read  Blockstore
-	write Blockstore
-}
-
-func NewBuffered(base Blockstore) *BufferedBlockstore {
-	var buf Blockstore
-	if os.Getenv("LOTUS_DISABLE_VM_BUF") == "iknowitsabadidea" {
-		buflog.Warn("VM BLOCKSTORE BUFFERING IS DISABLED")
-		buf = base
-	} else {
-		buf = NewMemory()
-	}
-
-	bs := &BufferedBlockstore{
-		read:  base,
-		write: buf,
-	}
-	return bs
-}
-
-func NewTieredBstore(r Blockstore, w Blockstore) *BufferedBlockstore {
-	return &BufferedBlockstore{
-		read:  r,
-		write: w,
-	}
-}
-
-var (
-	_ Blockstore = (*BufferedBlockstore)(nil)
-	_ Viewer     = (*BufferedBlockstore)(nil)
-)
-
-func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	a, err := bs.read.AllKeysChan(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	b, err := bs.write.AllKeysChan(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	out := make(chan cid.Cid)
-	go func() {
-		defer close(out)
-		for a != nil || b != nil {
-			select {
-			case val, ok := <-a:
-				if !ok {
-					a = nil
-				} else {
-					select {
-					case out <- val:
-					case <-ctx.Done():
-						return
-					}
-				}
-			case val, ok := <-b:
-				if !ok {
-					b = nil
-				} else {
-					select {
-					case out <- val:
-					case <-ctx.Done():
-						return
-					}
-				}
-			}
-		}
-	}()
-
-	return out, nil
-}
-
-func (bs *BufferedBlockstore) DeleteBlock(c cid.Cid) error {
-	if err := bs.read.DeleteBlock(c); err != nil {
-		return err
-	}
-
-	return bs.write.DeleteBlock(c)
-}
-
-func (bs *BufferedBlockstore) DeleteMany(cids []cid.Cid) error {
-	if err := bs.read.DeleteMany(cids); err != nil {
-		return err
-	}
-
-	return bs.write.DeleteMany(cids)
-}
-
-func (bs *BufferedBlockstore) View(c cid.Cid, callback func([]byte) error) error {
-	// both stores are viewable.
-	if err := bs.write.View(c, callback); err == ErrNotFound {
-		// not found in write blockstore; fall through.
-	} else {
-		return err // propagate errors, or nil, i.e. found.
-	}
-	return bs.read.View(c, callback)
-}
-
-func (bs *BufferedBlockstore) Get(c cid.Cid) (block.Block, error) {
-	if out, err := bs.write.Get(c); err != nil {
-		if err != ErrNotFound {
-			return nil, err
-		}
-	} else {
-		return out, nil
-	}
-
-	return bs.read.Get(c)
-}
-
-func (bs *BufferedBlockstore) GetSize(c cid.Cid) (int, error) {
-	s, err := bs.read.GetSize(c)
-	if err == ErrNotFound || s == 0 {
-		return bs.write.GetSize(c)
-	}
-
-	return s, err
-}
-
-func (bs *BufferedBlockstore) Put(blk block.Block) error {
-	has, err := bs.read.Has(blk.Cid()) // TODO: consider dropping this check
-	if err != nil {
-		return err
-	}
-
-	if has {
-		return nil
-	}
-
-	return bs.write.Put(blk)
-}
-
-func (bs *BufferedBlockstore) Has(c cid.Cid) (bool, error) {
-	has, err := bs.write.Has(c)
-	if err != nil {
-		return false, err
-	}
-	if has {
-		return true, nil
-	}
-
-	return bs.read.Has(c)
-}
-
-func (bs *BufferedBlockstore) HashOnRead(hor bool) {
-	bs.read.HashOnRead(hor)
-	bs.write.HashOnRead(hor)
-}
-
-func (bs *BufferedBlockstore) PutMany(blks []block.Block) error {
-	return bs.write.PutMany(blks)
-}
-
-func (bs *BufferedBlockstore) Read() Blockstore {
-	return bs.read
-}

blockstore/discard.go

@@ -1,66 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"io"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-)
-
-var _ Blockstore = (*discardstore)(nil)
-
-type discardstore struct {
-	bs Blockstore
-}
-
-func NewDiscardStore(bs Blockstore) Blockstore {
-	return &discardstore{bs: bs}
-}
-
-func (b *discardstore) Has(cid cid.Cid) (bool, error) {
-	return b.bs.Has(cid)
-}
-
-func (b *discardstore) HashOnRead(hor bool) {
-	b.bs.HashOnRead(hor)
-}
-
-func (b *discardstore) Get(cid cid.Cid) (blocks.Block, error) {
-	return b.bs.Get(cid)
-}
-
-func (b *discardstore) GetSize(cid cid.Cid) (int, error) {
-	return b.bs.GetSize(cid)
-}
-
-func (b *discardstore) View(cid cid.Cid, f func([]byte) error) error {
-	return b.bs.View(cid, f)
-}
-
-func (b *discardstore) Put(blk blocks.Block) error {
-	return nil
-}
-
-func (b *discardstore) PutMany(blks []blocks.Block) error {
-	return nil
-}
-
-func (b *discardstore) DeleteBlock(cid cid.Cid) error {
-	return nil
-}
-
-func (b *discardstore) DeleteMany(cids []cid.Cid) error {
-	return nil
-}
-
-func (b *discardstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return b.bs.AllKeysChan(ctx)
-}
-
-func (b *discardstore) Close() error {
-	if c, ok := b.bs.(io.Closer); ok {
-		return c.Close()
-	}
-	return nil
-}

blockstore/doc.go

@@ -1,9 +0,0 @@
-// Package blockstore and subpackages contain most of the blockstore
-// implementations used by Lotus.
-//
-// Blockstores not ultimately constructed out of the building blocks in this
-// package may not work properly.
-//
-// This package re-exports parts of the go-ipfs-blockstore package such that
-// no other package needs to import it directly, for ergonomics and traceability.
-package blockstore

blockstore/fallback.go

@@ -1,106 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"sync"
-	"time"
-
-	"golang.org/x/xerrors"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-)
-
-// UnwrapFallbackStore takes a blockstore, and returns the underlying blockstore
-// if it was a FallbackStore. Otherwise, it just returns the supplied store
-// unmodified.
-func UnwrapFallbackStore(bs Blockstore) (Blockstore, bool) {
-	if fbs, ok := bs.(*FallbackStore); ok {
-		return fbs.Blockstore, true
-	}
-	return bs, false
-}
-
-// FallbackStore is a read-through store that queries another (potentially
-// remote) source if the block is not found locally. If the block is found
-// during the fallback, it stores it in the local store.
-type FallbackStore struct {
-	Blockstore
-
-	lk sync.RWMutex
-	// missFn is the function that will be invoked on a local miss to pull the
-	// block from elsewhere.
-	missFn func(context.Context, cid.Cid) (blocks.Block, error)
-}
-
-var _ Blockstore = (*FallbackStore)(nil)
-
-func (fbs *FallbackStore) SetFallback(missFn func(context.Context, cid.Cid) (blocks.Block, error)) {
-	fbs.lk.Lock()
-	defer fbs.lk.Unlock()
-
-	fbs.missFn = missFn
-}
-
-func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
-	log.Warnf("fallbackstore: block not found locally, fetching from the network; cid: %s", c)
-	fbs.lk.RLock()
-	defer fbs.lk.RUnlock()
-
-	if fbs.missFn == nil {
-		// FallbackStore wasn't configured yet (chainstore/bitswap aren't up yet)
-		// Wait for a bit and retry
-		fbs.lk.RUnlock()
-		time.Sleep(5 * time.Second)
-		fbs.lk.RLock()
-
-		if fbs.missFn == nil {
-			log.Errorw("fallbackstore: missFn not configured yet")
-			return nil, ErrNotFound
-		}
-	}
-
-	ctx, cancel := context.WithTimeout(context.TODO(), 120*time.Second)
-	defer cancel()
-
-	b, err := fbs.missFn(ctx, c)
-	if err != nil {
-		return nil, err
-	}
-
-	// chain bitswap puts blocks in temp blockstore which is cleaned up
-	// every few min (to drop any messages we fetched but don't want)
-	// in this case we want to keep this block around
-	if err := fbs.Put(b); err != nil {
-		return nil, xerrors.Errorf("persisting fallback-fetched block: %w", err)
-	}
-	return b, nil
-}
-
-func (fbs *FallbackStore) Get(c cid.Cid) (blocks.Block, error) {
-	b, err := fbs.Blockstore.Get(c)
-	switch err {
-	case nil:
-		return b, nil
-	case ErrNotFound:
-		return fbs.getFallback(c)
-	default:
-		return b, err
-	}
-}
-
-func (fbs *FallbackStore) GetSize(c cid.Cid) (int, error) {
-	sz, err := fbs.Blockstore.GetSize(c)
-	switch err {
-	case nil:
-		return sz, nil
-	case ErrNotFound:
-		b, err := fbs.getFallback(c)
-		if err != nil {
-			return 0, err
-		}
-		return len(b.RawData()), nil
-	default:
-		return sz, err
-	}
-}

blockstore/idstore.go

@@ -1,174 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"io"
-
-	"golang.org/x/xerrors"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-	mh "github.com/multiformats/go-multihash"
-)
-
-var _ Blockstore = (*idstore)(nil)
-
-type idstore struct {
-	bs Blockstore
-}
-
-func NewIDStore(bs Blockstore) Blockstore {
-	return &idstore{bs: bs}
-}
-
-func decodeCid(cid cid.Cid) (inline bool, data []byte, err error) {
-	if cid.Prefix().MhType != mh.IDENTITY {
-		return false, nil, nil
-	}
-
-	dmh, err := mh.Decode(cid.Hash())
-	if err != nil {
-		return false, nil, err
-	}
-
-	if dmh.Code == mh.IDENTITY {
-		return true, dmh.Digest, nil
-	}
-
-	return false, nil, err
-}
-
-func (b *idstore) Has(cid cid.Cid) (bool, error) {
-	inline, _, err := decodeCid(cid)
-	if err != nil {
-		return false, xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return true, nil
-	}
-
-	return b.bs.Has(cid)
-}
-
-func (b *idstore) Get(cid cid.Cid) (blocks.Block, error) {
-	inline, data, err := decodeCid(cid)
-	if err != nil {
-		return nil, xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return blocks.NewBlockWithCid(data, cid)
-	}
-
-	return b.bs.Get(cid)
-}
-
-func (b *idstore) GetSize(cid cid.Cid) (int, error) {
-	inline, data, err := decodeCid(cid)
-	if err != nil {
-		return 0, xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return len(data), err
-	}
-
-	return b.bs.GetSize(cid)
-}
-
-func (b *idstore) View(cid cid.Cid, cb func([]byte) error) error {
-	inline, data, err := decodeCid(cid)
-	if err != nil {
-		return xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return cb(data)
-	}
-
-	return b.bs.View(cid, cb)
-}
-
-func (b *idstore) Put(blk blocks.Block) error {
-	inline, _, err := decodeCid(blk.Cid())
-	if err != nil {
-		return xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return nil
-	}
-
-	return b.bs.Put(blk)
-}
-
-func (b *idstore) PutMany(blks []blocks.Block) error {
-	toPut := make([]blocks.Block, 0, len(blks))
-	for _, blk := range blks {
-		inline, _, err := decodeCid(blk.Cid())
-		if err != nil {
-			return xerrors.Errorf("error decoding Cid: %w", err)
-		}
-
-		if inline {
-			continue
-		}
-		toPut = append(toPut, blk)
-	}
-
-	if len(toPut) > 0 {
-		return b.bs.PutMany(toPut)
-	}
-
-	return nil
-}
-
-func (b *idstore) DeleteBlock(cid cid.Cid) error {
-	inline, _, err := decodeCid(cid)
-	if err != nil {
-		return xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return nil
-	}
-
-	return b.bs.DeleteBlock(cid)
-}
-
-func (b *idstore) DeleteMany(cids []cid.Cid) error {
-	toDelete := make([]cid.Cid, 0, len(cids))
-	for _, cid := range cids {
-		inline, _, err := decodeCid(cid)
-		if err != nil {
-			return xerrors.Errorf("error decoding Cid: %w", err)
-		}
-
-		if inline {
-			continue
-		}
-		toDelete = append(toDelete, cid)
-	}
-
-	if len(toDelete) > 0 {
-		return b.bs.DeleteMany(toDelete)
-	}
-
-	return nil
-}
-
-func (b *idstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return b.bs.AllKeysChan(ctx)
-}
-
-func (b *idstore) HashOnRead(enabled bool) {
-	b.bs.HashOnRead(enabled)
-}
-
-func (b *idstore) Close() error {
-	if c, ok := b.bs.(io.Closer); ok {
-		return c.Close()
-	}
-	return nil
-}

blockstore/ipfs.go

@@ -1,155 +0,0 @@
-package blockstore
-
-import (
-	"bytes"
-	"context"
-	"io/ioutil"
-
-	"golang.org/x/xerrors"
-
-	"github.com/multiformats/go-multiaddr"
-	"github.com/multiformats/go-multihash"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	httpapi "github.com/ipfs/go-ipfs-http-client"
-	iface "github.com/ipfs/interface-go-ipfs-core"
-	"github.com/ipfs/interface-go-ipfs-core/options"
-	"github.com/ipfs/interface-go-ipfs-core/path"
-)
-
-type IPFSBlockstore struct {
-	ctx             context.Context
-	api, offlineAPI iface.CoreAPI
-}
-
-var _ BasicBlockstore = (*IPFSBlockstore)(nil)
-
-func NewLocalIPFSBlockstore(ctx context.Context, onlineMode bool) (Blockstore, error) {
-	localApi, err := httpapi.NewLocalApi()
-	if err != nil {
-		return nil, xerrors.Errorf("getting local ipfs api: %w", err)
-	}
-	api, err := localApi.WithOptions(options.Api.Offline(!onlineMode))
-	if err != nil {
-		return nil, xerrors.Errorf("setting offline mode: %s", err)
-	}
-
-	offlineAPI := api
-	if onlineMode {
-		offlineAPI, err = localApi.WithOptions(options.Api.Offline(true))
-		if err != nil {
-			return nil, xerrors.Errorf("applying offline mode: %s", err)
-		}
-	}
-
-	bs := &IPFSBlockstore{
-		ctx:        ctx,
-		api:        api,
-		offlineAPI: offlineAPI,
-	}
-
-	return Adapt(bs), nil
-}
-
-func NewRemoteIPFSBlockstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineMode bool) (Blockstore, error) {
-	httpApi, err := httpapi.NewApi(maddr)
-	if err != nil {
-		return nil, xerrors.Errorf("setting remote ipfs api: %w", err)
-	}
-	api, err := httpApi.WithOptions(options.Api.Offline(!onlineMode))
-	if err != nil {
-		return nil, xerrors.Errorf("applying offline mode: %s", err)
-	}
-
-	offlineAPI := api
-	if onlineMode {
-		offlineAPI, err = httpApi.WithOptions(options.Api.Offline(true))
-		if err != nil {
-			return nil, xerrors.Errorf("applying offline mode: %s", err)
-		}
-	}
-
-	bs := &IPFSBlockstore{
-		ctx:        ctx,
-		api:        api,
-		offlineAPI: offlineAPI,
-	}
-
-	return Adapt(bs), nil
-}
-
-func (i *IPFSBlockstore) DeleteBlock(cid cid.Cid) error {
-	return xerrors.Errorf("not supported")
-}
-
-func (i *IPFSBlockstore) Has(cid cid.Cid) (bool, error) {
-	_, err := i.offlineAPI.Block().Stat(i.ctx, path.IpldPath(cid))
-	if err != nil {
-		// The underlying client is running in Offline mode.
-		// Stat() will fail with an err if the block isn't in the
-		// blockstore. If that's the case, return false without
-		// an error since that's the original intention of this method.
-		if err.Error() == "blockservice: key not found" {
-			return false, nil
-		}
-		return false, xerrors.Errorf("getting ipfs block: %w", err)
-	}
-
-	return true, nil
-}
-
-func (i *IPFSBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
-	rd, err := i.api.Block().Get(i.ctx, path.IpldPath(cid))
-	if err != nil {
-		return nil, xerrors.Errorf("getting ipfs block: %w", err)
-	}
-
-	data, err := ioutil.ReadAll(rd)
-	if err != nil {
-		return nil, err
-	}
-
-	return blocks.NewBlockWithCid(data, cid)
-}
-
-func (i *IPFSBlockstore) GetSize(cid cid.Cid) (int, error) {
-	st, err := i.api.Block().Stat(i.ctx, path.IpldPath(cid))
-	if err != nil {
-		return 0, xerrors.Errorf("getting ipfs block: %w", err)
-	}
-
-	return st.Size(), nil
-}
-
-func (i *IPFSBlockstore) Put(block blocks.Block) error {
-	mhd, err := multihash.Decode(block.Cid().Hash())
-	if err != nil {
-		return err
-	}
-
-	_, err = i.api.Block().Put(i.ctx, bytes.NewReader(block.RawData()),
-		options.Block.Hash(mhd.Code, mhd.Length),
-		options.Block.Format(cid.CodecToStr[block.Cid().Type()]))
-	return err
-}
-
-func (i *IPFSBlockstore) PutMany(blocks []blocks.Block) error {
-	// TODO: could be done in parallel
-
-	for _, block := range blocks {
-		if err := i.Put(block); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (i *IPFSBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return nil, xerrors.Errorf("not supported")
-}
-
-func (i *IPFSBlockstore) HashOnRead(enabled bool) {
-	return // TODO: We could technically support this, but..
-}

blockstore/mem.go

@@ -1,102 +0,0 @@
-package blockstore
-
-import (
-	"context"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-)
-
-// NewMemory returns a temporary memory-backed blockstore.
-func NewMemory() MemBlockstore {
-	return make(MemBlockstore)
-}
-
-// MemBlockstore is a terminal blockstore that keeps blocks in memory.
-type MemBlockstore map[cid.Cid]blocks.Block
-
-func (m MemBlockstore) DeleteBlock(k cid.Cid) error {
-	delete(m, k)
-	return nil
-}
-
-func (m MemBlockstore) DeleteMany(ks []cid.Cid) error {
-	for _, k := range ks {
-		delete(m, k)
-	}
-	return nil
-}
-
-func (m MemBlockstore) Has(k cid.Cid) (bool, error) {
-	_, ok := m[k]
-	return ok, nil
-}
-
-func (m MemBlockstore) View(k cid.Cid, callback func([]byte) error) error {
-	b, ok := m[k]
-	if !ok {
-		return ErrNotFound
-	}
-	return callback(b.RawData())
-}
-
-func (m MemBlockstore) Get(k cid.Cid) (blocks.Block, error) {
-	b, ok := m[k]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	return b, nil
-}
-
-// GetSize returns the CIDs mapped BlockSize
-func (m MemBlockstore) GetSize(k cid.Cid) (int, error) {
-	b, ok := m[k]
-	if !ok {
-		return 0, ErrNotFound
-	}
-	return len(b.RawData()), nil
-}
-
-// Put puts a given block to the underlying datastore
-func (m MemBlockstore) Put(b blocks.Block) error {
-	// Convert to a basic block for safety, but try to reuse the existing
-	// block if it's already a basic block.
-	k := b.Cid()
-	if _, ok := b.(*blocks.BasicBlock); !ok {
-		// If we already have the block, abort.
-		if _, ok := m[k]; ok {
-			return nil
-		}
-		// the error is only for debugging.
-		b, _ = blocks.NewBlockWithCid(b.RawData(), b.Cid())
-	}
-	m[b.Cid()] = b
-	return nil
-}
-
-// PutMany puts a slice of blocks at the same time using batching
-// capabilities of the underlying datastore whenever possible.
-func (m MemBlockstore) PutMany(bs []blocks.Block) error {
-	for _, b := range bs {
-		_ = m.Put(b) // can't fail
-	}
-	return nil
-}
-
-// AllKeysChan returns a channel from which
-// the CIDs in the Blockstore can be read. It should respect
-// the given context, closing the channel if it becomes Done.
-func (m MemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	ch := make(chan cid.Cid, len(m))
-	for k := range m {
-		ch <- k
-	}
-	close(ch)
-	return ch, nil
-}
-
-// HashOnRead specifies if every read block should be
-// rehashed to make sure it matches its CID.
-func (m MemBlockstore) HashOnRead(enabled bool) {
-	// no-op
-}

blockstore/metrics.go

@@ -1,154 +0,0 @@
-package blockstore
-
-import (
-	"time"
-
-	"go.opencensus.io/stats"
-	"go.opencensus.io/stats/view"
-	"go.opencensus.io/tag"
-)
-
-//
-// Currently unused, but kept in repo in case we introduce one of the candidate
-// cache implementations (Freecache, Ristretto), both of which report these
-// metrics.
-//
-
-// CacheMetricsEmitInterval is the interval at which metrics are emitted onto
-// OpenCensus.
-var CacheMetricsEmitInterval = 5 * time.Second
-
-var (
-	CacheName, _ = tag.NewKey("cache_name")
-)
-
-// CacheMeasures groups all metrics emitted by the blockstore caches.
-var CacheMeasures = struct {
-	HitRatio       *stats.Float64Measure
-	Hits           *stats.Int64Measure
-	Misses         *stats.Int64Measure
-	Entries        *stats.Int64Measure
-	QueriesServed  *stats.Int64Measure
-	Adds           *stats.Int64Measure
-	Updates        *stats.Int64Measure
-	Evictions      *stats.Int64Measure
-	CostAdded      *stats.Int64Measure
-	CostEvicted    *stats.Int64Measure
-	SetsDropped    *stats.Int64Measure
-	SetsRejected   *stats.Int64Measure
-	QueriesDropped *stats.Int64Measure
-}{
-	HitRatio:       stats.Float64("blockstore/cache/hit_ratio", "Hit ratio of blockstore cache", stats.UnitDimensionless),
-	Hits:           stats.Int64("blockstore/cache/hits", "Total number of hits at blockstore cache", stats.UnitDimensionless),
-	Misses:         stats.Int64("blockstore/cache/misses", "Total number of misses at blockstore cache", stats.UnitDimensionless),
-	Entries:        stats.Int64("blockstore/cache/entry_count", "Total number of entries currently in the blockstore cache", stats.UnitDimensionless),
-	QueriesServed:  stats.Int64("blockstore/cache/queries_served", "Total number of queries served by the blockstore cache", stats.UnitDimensionless),
-	Adds:           stats.Int64("blockstore/cache/adds", "Total number of adds to blockstore cache", stats.UnitDimensionless),
-	Updates:        stats.Int64("blockstore/cache/updates", "Total number of updates in blockstore cache", stats.UnitDimensionless),
-	Evictions:      stats.Int64("blockstore/cache/evictions", "Total number of evictions from blockstore cache", stats.UnitDimensionless),
-	CostAdded:      stats.Int64("blockstore/cache/cost_added", "Total cost (byte size) of entries added into blockstore cache", stats.UnitBytes),
-	CostEvicted:    stats.Int64("blockstore/cache/cost_evicted", "Total cost (byte size) of entries evicted by blockstore cache", stats.UnitBytes),
-	SetsDropped:    stats.Int64("blockstore/cache/sets_dropped", "Total number of sets dropped by blockstore cache", stats.UnitDimensionless),
-	SetsRejected:   stats.Int64("blockstore/cache/sets_rejected", "Total number of sets rejected by blockstore cache", stats.UnitDimensionless),
-	QueriesDropped: stats.Int64("blockstore/cache/queries_dropped", "Total number of queries dropped by blockstore cache", stats.UnitDimensionless),
-}
-
-// CacheViews groups all cache-related default views.
-var CacheViews = struct {
-	HitRatio       *view.View
-	Hits           *view.View
-	Misses         *view.View
-	Entries        *view.View
-	QueriesServed  *view.View
-	Adds           *view.View
-	Updates        *view.View
-	Evictions      *view.View
-	CostAdded      *view.View
-	CostEvicted    *view.View
-	SetsDropped    *view.View
-	SetsRejected   *view.View
-	QueriesDropped *view.View
-}{
-	HitRatio: &view.View{
-		Measure:     CacheMeasures.HitRatio,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Hits: &view.View{
-		Measure:     CacheMeasures.Hits,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Misses: &view.View{
-		Measure:     CacheMeasures.Misses,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Entries: &view.View{
-		Measure:     CacheMeasures.Entries,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	QueriesServed: &view.View{
-		Measure:     CacheMeasures.QueriesServed,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Adds: &view.View{
-		Measure:     CacheMeasures.Adds,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Updates: &view.View{
-		Measure:     CacheMeasures.Updates,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Evictions: &view.View{
-		Measure:     CacheMeasures.Evictions,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	CostAdded: &view.View{
-		Measure:     CacheMeasures.CostAdded,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	CostEvicted: &view.View{
-		Measure:     CacheMeasures.CostEvicted,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	SetsDropped: &view.View{
-		Measure:     CacheMeasures.SetsDropped,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	SetsRejected: &view.View{
-		Measure:     CacheMeasures.SetsRejected,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	QueriesDropped: &view.View{
-		Measure:     CacheMeasures.QueriesDropped,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-}
-
-// DefaultViews exports all default views for this package.
-var DefaultViews = []*view.View{
-	CacheViews.HitRatio,
-	CacheViews.Hits,
-	CacheViews.Misses,
-	CacheViews.Entries,
-	CacheViews.QueriesServed,
-	CacheViews.Adds,
-	CacheViews.Updates,
-	CacheViews.Evictions,
-	CacheViews.CostAdded,
-	CacheViews.CostEvicted,
-	CacheViews.SetsDropped,
-	CacheViews.SetsRejected,
-	CacheViews.QueriesDropped,
-}

blockstore/splitstore/README.md

@@ -1,124 +0,0 @@
-# SplitStore: An actively scalable blockstore for the Filecoin chain
-
-The SplitStore was first introduced in lotus v1.5.1, as an experiment
-in reducing the performance impact of large blockstores.
-
-With lotus v1.11.1, we introduce the next iteration in design and
-implementation, which we call SplitStore v1.
-
-The new design (see [#6474](https://github.com/filecoin-project/lotus/pull/6474)
-evolves the splitstore to be a freestanding compacting blockstore that
-allows us to keep a small (60-100GB) working set in a hot blockstore
-and reliably archive out of scope objects in a coldstore.  The
-coldstore can also be a discard store, whereby out of scope objects
-are discarded or a regular badger blockstore (the default), which can
-be periodically garbage collected according to configurable user
-retention policies.
-
-To enable the splitstore, edit `.lotus/config.toml` and add the following:
-```
-[Chainstore]
-  EnableSplitstore = true
-```
-
-If you intend to use the discard coldstore, your also need to add the following:
-```
-  [Chainstore.Splitstore]
-    ColdStoreType = "discard"
-```
-In general you _should not_ have to use the discard store, unless you
-are running a network assistive node (like a bootstrapper or booster)
-or have very constrained hardware with not enough disk space to
-maintain a coldstore, even with garbage collection. It is also appropriate
-for small nodes that are simply watching the chain.
-
-*Warning:* Using the discard store for a general purpose node is discouraged, unless
-you really know what you are doing. Use it at your own risk.
-
-## Configuration Options
-
-These are options in the `[Chainstore.Splitstore]` section of the configuration:
-
-- `HotStoreType` -- specifies the type of hotstore to use.
-  The only currently supported option is `"badger"`.
-- `ColdStoreType` -- specifies the type of coldstore to use.
-  The default value is `"universal"`, which will use the initial monolith blockstore
-  as the coldstore.
-  The other possible value is `"discard"`, as outlined above, which is specialized for
-  running without a coldstore. Note that the discard store wraps the initial monolith
-  blockstore and discards writes; this is necessary to support syncing from a snapshot.
-- `MarkSetType` -- specifies the type of markset to use during compaction.
-  The markset is the data structure used by compaction/gc to track live objects.
-  The default value is `"map"`, which will use an in-memory map; if you are limited
-  in memory (or indeed see compaction run out of memory), you can also specify
-  `"badger"` which will use an disk backed markset, using badger. This will use
-  much less memory, but will also make compaction slower.
-- `HotStoreMessageRetention` -- specifies how many finalities, beyond the 4
-  finalities maintained by default, to maintain messages and message receipts in the
-  hotstore. This is useful for assistive nodes that want to support syncing for other
-  nodes beyond 4 finalities, while running with the discard coldstore option.
-  It is also useful for miners who accept deals and need to lookback messages beyond
-  the 4 finalities, which would otherwise hit the coldstore.
-- `HotStoreFullGCFrequency` -- specifies how frequenty to garbage collect the hotstore
-  using full (moving) GC.
-  The default value is 20, which uses full GC every 20 compactions (about once a week);
-  set to 0 to disable full GC altogether.
-  Rationale: badger supports online GC, and this is used by default. However it has proven to
-  be ineffective in practice with the hotstore size slowly creeping up. In order to address this,
-  we have added moving GC support in our badger wrapper, which can effectively reclaim all space.
-  The downside is that it takes a bit longer to perform a moving GC and you also need enough
-  space to house the new hotstore while the old one is still live.
-
-
-## Operation
-
-When the splitstore is first enabled, the existing blockstore becomes
-the coldstore and a fresh hotstore is initialized.
-
-The hotstore is warmed up on first startup so as to load all chain
-headers and state roots in the current head.  This allows us to
-immediately gain the performance benefits of a smallerblockstore which
-can be substantial for full archival nodes.
-
-All new writes are directed to the hotstore, while reads first hit the
-hotstore, with fallback to the coldstore.
-
-Once 5 finalities have ellapsed, and every finality henceforth, the
-blockstore _compacts_.  Compaction is the process of moving all
-unreachable objects within the last 4 finalities from the hotstore to
-the coldstore. If the system is configured with a discard coldstore,
-these objects are discarded. Note that chain headers, all the way to
-genesis, are considered reachable. Stateroots and messages are
-considered reachable only within the last 4 finalities, unless there
-is a live reference to them.
-
-## Compaction
-
-Compaction works transactionally with the following algorithm:
-- We prepare a transaction, whereby all i/o referenced objects through the API are tracked.
-- We walk the chain and mark reachable objects, keeping 4 finalities of state roots and messages and all headers all the way to genesis.
-- Once the chain walk is complete, we begin full transaction protection with concurrent marking; we walk and mark all references created during the chain walk. On the same time, all I/O through the API concurrently marks objects as live references.
-- We collect cold objects by iterating through the hotstore and checking the mark set; if an object is not marked, then it is candidate for purge.
-- When running with a coldstore, we next copy all cold objects to the coldstore.
-- At this point we are ready to begin purging:
-  - We sort cold objects heaviest first, so as to never delete the consituents of a DAG before the DAG itself (which would leave dangling references)
-  - We delete in small batches taking a lock; each batch is checked again for marks, from the concurrent transactional mark, so as to never delete anything live
-- We then end the transaction and compact/gc the hotstore.
-
-## Garbage Collection
-
-TBD -- see [#6577](https://github.com/filecoin-project/lotus/issues/6577)
-
-## Utilities
-
-`lotus-shed` has a `splitstore` command which provides some utilities:
-
-- `rollback` -- rolls back a splitstore installation.
-  This command copies the hotstore on top of the coldstore, and then deletes the splitstore
-  directory and associated metadata keys.
-  It can also optionally compact/gc the coldstore after the copy (with the `--gc-coldstore` flag)
-  and automatically rewrite the lotus config to disable splitstore (with the `--rewrite-config` flag).
-  Note: the node *must be stopped* before running this command.
-- `check` -- asynchronously runs a basic healthcheck on the splitstore.
-  The results are appended to `<lotus-repo>/datastore/splitstore/check.txt`.
-- `info` -- prints some basic information about the splitstore.

blockstore/splitstore/debug.go

@@ -1,273 +0,0 @@
-package splitstore
-
-import (
-	"crypto/sha256"
-	"encoding/hex"
-	"fmt"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"runtime/debug"
-	"strings"
-	"sync"
-	"time"
-
-	"go.uber.org/multierr"
-	"golang.org/x/xerrors"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-)
-
-type debugLog struct {
-	readLog, writeLog, deleteLog, stackLog *debugLogOp
-
-	stackMx  sync.Mutex
-	stackMap map[string]string
-}
-
-type debugLogOp struct {
-	path  string
-	mx    sync.Mutex
-	log   *os.File
-	count int
-}
-
-func openDebugLog(path string) (*debugLog, error) {
-	basePath := filepath.Join(path, "debug")
-	err := os.MkdirAll(basePath, 0755)
-	if err != nil {
-		return nil, err
-	}
-
-	readLog, err := openDebugLogOp(basePath, "read.log")
-	if err != nil {
-		return nil, err
-	}
-
-	writeLog, err := openDebugLogOp(basePath, "write.log")
-	if err != nil {
-		_ = readLog.Close()
-		return nil, err
-	}
-
-	deleteLog, err := openDebugLogOp(basePath, "delete.log")
-	if err != nil {
-		_ = readLog.Close()
-		_ = writeLog.Close()
-		return nil, err
-	}
-
-	stackLog, err := openDebugLogOp(basePath, "stack.log")
-	if err != nil {
-		_ = readLog.Close()
-		_ = writeLog.Close()
-		_ = deleteLog.Close()
-		return nil, xerrors.Errorf("error opening stack log: %w", err)
-	}
-
-	return &debugLog{
-		readLog:   readLog,
-		writeLog:  writeLog,
-		deleteLog: deleteLog,
-		stackLog:  stackLog,
-		stackMap:  make(map[string]string),
-	}, nil
-}
-
-func (d *debugLog) LogReadMiss(cid cid.Cid) {
-	if d == nil {
-		return
-	}
-
-	stack := d.getStack()
-	err := d.readLog.Log("%s %s %s\n", d.timestamp(), cid, stack)
-	if err != nil {
-		log.Warnf("error writing read log: %s", err)
-	}
-}
-
-func (d *debugLog) LogWrite(blk blocks.Block) {
-	if d == nil {
-		return
-	}
-
-	var stack string
-	if enableDebugLogWriteTraces {
-		stack = " " + d.getStack()
-	}
-
-	err := d.writeLog.Log("%s %s%s\n", d.timestamp(), blk.Cid(), stack)
-	if err != nil {
-		log.Warnf("error writing write log: %s", err)
-	}
-}
-
-func (d *debugLog) LogWriteMany(blks []blocks.Block) {
-	if d == nil {
-		return
-	}
-
-	var stack string
-	if enableDebugLogWriteTraces {
-		stack = " " + d.getStack()
-	}
-
-	now := d.timestamp()
-	for _, blk := range blks {
-		err := d.writeLog.Log("%s %s%s\n", now, blk.Cid(), stack)
-		if err != nil {
-			log.Warnf("error writing write log: %s", err)
-			break
-		}
-	}
-}
-
-func (d *debugLog) LogDelete(cids []cid.Cid) {
-	if d == nil {
-		return
-	}
-
-	now := d.timestamp()
-	for _, c := range cids {
-		err := d.deleteLog.Log("%s %s\n", now, c)
-		if err != nil {
-			log.Warnf("error writing delete log: %s", err)
-			break
-		}
-	}
-}
-
-func (d *debugLog) Flush() {
-	if d == nil {
-		return
-	}
-
-	// rotate non-empty logs
-	d.readLog.Rotate()
-	d.writeLog.Rotate()
-	d.deleteLog.Rotate()
-	d.stackLog.Rotate()
-}
-
-func (d *debugLog) Close() error {
-	if d == nil {
-		return nil
-	}
-
-	err1 := d.readLog.Close()
-	err2 := d.writeLog.Close()
-	err3 := d.deleteLog.Close()
-	err4 := d.stackLog.Close()
-
-	return multierr.Combine(err1, err2, err3, err4)
-}
-
-func (d *debugLog) getStack() string {
-	sk := d.getNormalizedStackTrace()
-	hash := sha256.Sum256([]byte(sk))
-	key := string(hash[:])
-
-	d.stackMx.Lock()
-	repr, ok := d.stackMap[key]
-	if !ok {
-		repr = hex.EncodeToString(hash[:])
-		d.stackMap[key] = repr
-
-		err := d.stackLog.Log("%s\n%s\n", repr, sk)
-		if err != nil {
-			log.Warnf("error writing stack trace for %s: %s", repr, err)
-		}
-	}
-	d.stackMx.Unlock()
-
-	return repr
-}
-
-func (d *debugLog) getNormalizedStackTrace() string {
-	sk := string(debug.Stack())
-
-	// Normalization for deduplication
-	// skip first line -- it's the goroutine
-	// for each line that ends in a ), remove the call args -- these are the registers
-	lines := strings.Split(sk, "\n")[1:]
-	for i, line := range lines {
-		if len(line) > 0 && line[len(line)-1] == ')' {
-			idx := strings.LastIndex(line, "(")
-			if idx < 0 {
-				continue
-			}
-			lines[i] = line[:idx]
-		}
-	}
-
-	return strings.Join(lines, "\n")
-}
-
-func (d *debugLog) timestamp() string {
-	ts, _ := time.Now().MarshalText()
-	return string(ts)
-}
-
-func openDebugLogOp(basePath, name string) (*debugLogOp, error) {
-	path := filepath.Join(basePath, name)
-	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
-	if err != nil {
-		return nil, xerrors.Errorf("error opening %s: %w", name, err)
-	}
-
-	return &debugLogOp{path: path, log: file}, nil
-}
-
-func (d *debugLogOp) Close() error {
-	d.mx.Lock()
-	defer d.mx.Unlock()
-
-	return d.log.Close()
-}
-
-func (d *debugLogOp) Log(template string, arg ...interface{}) error {
-	d.mx.Lock()
-	defer d.mx.Unlock()
-
-	d.count++
-	_, err := fmt.Fprintf(d.log, template, arg...)
-	return err
-}
-
-func (d *debugLogOp) Rotate() {
-	d.mx.Lock()
-	defer d.mx.Unlock()
-
-	if d.count == 0 {
-		return
-	}
-
-	err := d.log.Close()
-	if err != nil {
-		log.Warnf("error closing log (file: %s): %s", d.path, err)
-		return
-	}
-
-	arxivPath := fmt.Sprintf("%s-%d", d.path, time.Now().Unix())
-	err = os.Rename(d.path, arxivPath)
-	if err != nil {
-		log.Warnf("error moving log (file: %s): %s", d.path, err)
-		return
-	}
-
-	go func() {
-		cmd := exec.Command("gzip", arxivPath)
-		err := cmd.Run()
-		if err != nil {
-			log.Warnf("error compressing log (file: %s): %s", arxivPath, err)
-		}
-	}()
-
-	d.count = 0
-	d.log, err = os.OpenFile(d.path, os.O_WRONLY|os.O_CREATE, 0644)
-	if err != nil {
-		log.Warnf("error opening log (file: %s): %s", d.path, err)
-		return
-	}
-}

blockstore/splitstore/markset.go

@@ -1,40 +0,0 @@
-package splitstore
-
-import (
-	"errors"
-
-	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
-)
-
-var errMarkSetClosed = errors.New("markset closed")
-
-// MarkSet is a utility to keep track of seen CID, and later query for them.
-//
-// * If the expected dataset is large, it can be backed by a datastore (e.g. bbolt).
-// * If a probabilistic result is acceptable, it can be backed by a bloom filter
-type MarkSet interface {
-	Mark(cid.Cid) error
-	Has(cid.Cid) (bool, error)
-	Close() error
-	SetConcurrent()
-}
-
-type MarkSetEnv interface {
-	Create(name string, sizeHint int64) (MarkSet, error)
-	Close() error
-}
-
-func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) {
-	switch mtype {
-	case "bloom":
-		return NewBloomMarkSetEnv()
-	case "map":
-		return NewMapMarkSetEnv()
-	case "badger":
-		return NewBadgerMarkSetEnv(path)
-	default:
-		return nil, xerrors.Errorf("unknown mark set type %s", mtype)
-	}
-}

blockstore/splitstore/markset_badger.go

@@ -1,230 +0,0 @@
-package splitstore
-
-import (
-	"os"
-	"path/filepath"
-	"sync"
-
-	"golang.org/x/xerrors"
-
-	"github.com/dgraph-io/badger/v2"
-	"github.com/dgraph-io/badger/v2/options"
-	"go.uber.org/zap"
-
-	cid "github.com/ipfs/go-cid"
-)
-
-type BadgerMarkSetEnv struct {
-	path string
-}
-
-var _ MarkSetEnv = (*BadgerMarkSetEnv)(nil)
-
-type BadgerMarkSet struct {
-	mx      sync.RWMutex
-	cond    sync.Cond
-	pend    map[string]struct{}
-	writing map[int]map[string]struct{}
-	writers int
-	seqno   int
-
-	db   *badger.DB
-	path string
-}
-
-var _ MarkSet = (*BadgerMarkSet)(nil)
-
-var badgerMarkSetBatchSize = 16384
-
-func NewBadgerMarkSetEnv(path string) (MarkSetEnv, error) {
-	msPath := filepath.Join(path, "markset.badger")
-	err := os.MkdirAll(msPath, 0755) //nolint:gosec
-	if err != nil {
-		return nil, xerrors.Errorf("error creating markset directory: %w", err)
-	}
-
-	return &BadgerMarkSetEnv{path: msPath}, nil
-}
-
-func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
-	path := filepath.Join(e.path, name)
-
-	// clean up first
-	err := os.RemoveAll(path)
-	if err != nil {
-		return nil, xerrors.Errorf("error clearing markset directory: %w", err)
-	}
-
-	err = os.MkdirAll(path, 0755) //nolint:gosec
-	if err != nil {
-		return nil, xerrors.Errorf("error creating markset directory: %w", err)
-	}
-
-	opts := badger.DefaultOptions(path)
-	opts.SyncWrites = false
-	opts.CompactL0OnClose = false
-	opts.Compression = options.None
-	// Note: We use FileIO for loading modes to avoid memory thrashing and interference
-	//       between the system blockstore and the markset.
-	//       It was observed that using the default memory mapped option resulted in
-	//       significant interference and unacceptably high block validation times once the markset
-	//       exceeded 1GB in size.
-	opts.TableLoadingMode = options.FileIO
-	opts.ValueLogLoadingMode = options.FileIO
-	opts.Logger = &badgerLogger{
-		SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(),
-		skip2:         log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(),
-	}
-
-	db, err := badger.Open(opts)
-	if err != nil {
-		return nil, xerrors.Errorf("error creating badger markset: %w", err)
-	}
-
-	ms := &BadgerMarkSet{
-		pend:    make(map[string]struct{}),
-		writing: make(map[int]map[string]struct{}),
-		db:      db,
-		path:    path,
-	}
-	ms.cond.L = &ms.mx
-
-	return ms, nil
-}
-
-func (e *BadgerMarkSetEnv) Close() error {
-	return os.RemoveAll(e.path)
-}
-
-func (s *BadgerMarkSet) Mark(c cid.Cid) error {
-	s.mx.Lock()
-
-	if s.pend == nil {
-		s.mx.Unlock()
-		return errMarkSetClosed
-	}
-
-	s.pend[string(c.Hash())] = struct{}{}
-
-	if len(s.pend) < badgerMarkSetBatchSize {
-		s.mx.Unlock()
-		return nil
-	}
-
-	pend := s.pend
-	seqno := s.seqno
-	s.seqno++
-	s.writing[seqno] = pend
-	s.pend = make(map[string]struct{})
-	s.writers++
-	s.mx.Unlock()
-
-	defer func() {
-		s.mx.Lock()
-		defer s.mx.Unlock()
-
-		delete(s.writing, seqno)
-		s.writers--
-		if s.writers == 0 {
-			s.cond.Broadcast()
-		}
-	}()
-
-	empty := []byte{} // not nil
-
-	batch := s.db.NewWriteBatch()
-	defer batch.Cancel()
-
-	for k := range pend {
-		if err := batch.Set([]byte(k), empty); err != nil {
-			return err
-		}
-	}
-
-	err := batch.Flush()
-	if err != nil {
-		return xerrors.Errorf("error flushing batch to badger markset: %w", err)
-	}
-
-	return nil
-}
-
-func (s *BadgerMarkSet) Has(c cid.Cid) (bool, error) {
-	s.mx.RLock()
-	defer s.mx.RUnlock()
-
-	if s.pend == nil {
-		return false, errMarkSetClosed
-	}
-
-	key := c.Hash()
-	pendKey := string(key)
-	_, ok := s.pend[pendKey]
-	if ok {
-		return true, nil
-	}
-
-	for _, wr := range s.writing {
-		_, ok := wr[pendKey]
-		if ok {
-			return true, nil
-		}
-	}
-
-	err := s.db.View(func(txn *badger.Txn) error {
-		_, err := txn.Get(key)
-		return err
-	})
-
-	switch err {
-	case nil:
-		return true, nil
-
-	case badger.ErrKeyNotFound:
-		return false, nil
-
-	default:
-		return false, xerrors.Errorf("error checking badger markset: %w", err)
-	}
-}
-
-func (s *BadgerMarkSet) Close() error {
-	s.mx.Lock()
-	defer s.mx.Unlock()
-
-	if s.pend == nil {
-		return nil
-	}
-
-	for s.writers > 0 {
-		s.cond.Wait()
-	}
-
-	s.pend = nil
-	db := s.db
-	s.db = nil
-
-	err := db.Close()
-	if err != nil {
-		return xerrors.Errorf("error closing badger markset: %w", err)
-	}
-
-	err = os.RemoveAll(s.path)
-	if err != nil {
-		return xerrors.Errorf("error deleting badger markset: %w", err)
-	}
-
-	return nil
-}
-
-func (s *BadgerMarkSet) SetConcurrent() {}
-
-// badger logging through go-log
-type badgerLogger struct {
-	*zap.SugaredLogger
-	skip2 *zap.SugaredLogger
-}
-
-func (b *badgerLogger) Warningf(format string, args ...interface{}) {}
-func (b *badgerLogger) Infof(format string, args ...interface{})    {}
-func (b *badgerLogger) Debugf(format string, args ...interface{})   {}

blockstore/splitstore/markset_bloom.go

@@ -1,107 +0,0 @@
-package splitstore
-
-import (
-	"crypto/rand"
-	"crypto/sha256"
-	"sync"
-
-	"golang.org/x/xerrors"
-
-	bbloom "github.com/ipfs/bbloom"
-	cid "github.com/ipfs/go-cid"
-)
-
-const (
-	BloomFilterMinSize     = 10_000_000
-	BloomFilterProbability = 0.01
-)
-
-type BloomMarkSetEnv struct{}
-
-var _ MarkSetEnv = (*BloomMarkSetEnv)(nil)
-
-type BloomMarkSet struct {
-	salt []byte
-	mx   sync.RWMutex
-	bf   *bbloom.Bloom
-	ts   bool
-}
-
-var _ MarkSet = (*BloomMarkSet)(nil)
-
-func NewBloomMarkSetEnv() (*BloomMarkSetEnv, error) {
-	return &BloomMarkSetEnv{}, nil
-}
-
-func (e *BloomMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
-	size := int64(BloomFilterMinSize)
-	for size < sizeHint {
-		size += BloomFilterMinSize
-	}
-
-	salt := make([]byte, 4)
-	_, err := rand.Read(salt)
-	if err != nil {
-		return nil, xerrors.Errorf("error reading salt: %w", err)
-	}
-
-	bf, err := bbloom.New(float64(size), BloomFilterProbability)
-	if err != nil {
-		return nil, xerrors.Errorf("error creating bloom filter: %w", err)
-	}
-
-	return &BloomMarkSet{salt: salt, bf: bf}, nil
-}
-
-func (e *BloomMarkSetEnv) Close() error {
-	return nil
-}
-
-func (s *BloomMarkSet) saltedKey(cid cid.Cid) []byte {
-	hash := cid.Hash()
-	key := make([]byte, len(s.salt)+len(hash))
-	n := copy(key, s.salt)
-	copy(key[n:], hash)
-	rehash := sha256.Sum256(key)
-	return rehash[:]
-}
-
-func (s *BloomMarkSet) Mark(cid cid.Cid) error {
-	if s.ts {
-		s.mx.Lock()
-		defer s.mx.Unlock()
-	}
-
-	if s.bf == nil {
-		return errMarkSetClosed
-	}
-
-	s.bf.Add(s.saltedKey(cid))
-	return nil
-}
-
-func (s *BloomMarkSet) Has(cid cid.Cid) (bool, error) {
-	if s.ts {
-		s.mx.RLock()
-		defer s.mx.RUnlock()
-	}
-
-	if s.bf == nil {
-		return false, errMarkSetClosed
-	}
-
-	return s.bf.Has(s.saltedKey(cid)), nil
-}
-
-func (s *BloomMarkSet) Close() error {
-	if s.ts {
-		s.mx.Lock()
-		defer s.mx.Unlock()
-	}
-	s.bf = nil
-	return nil
-}
-
-func (s *BloomMarkSet) SetConcurrent() {
-	s.ts = true
-}

blockstore/splitstore/markset_map.go

@@ -1,75 +0,0 @@
-package splitstore
-
-import (
-	"sync"
-
-	cid "github.com/ipfs/go-cid"
-)
-
-type MapMarkSetEnv struct{}
-
-var _ MarkSetEnv = (*MapMarkSetEnv)(nil)
-
-type MapMarkSet struct {
-	mx  sync.RWMutex
-	set map[string]struct{}
-
-	ts bool
-}
-
-var _ MarkSet = (*MapMarkSet)(nil)
-
-func NewMapMarkSetEnv() (*MapMarkSetEnv, error) {
-	return &MapMarkSetEnv{}, nil
-}
-
-func (e *MapMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
-	return &MapMarkSet{
-		set: make(map[string]struct{}, sizeHint),
-	}, nil
-}
-
-func (e *MapMarkSetEnv) Close() error {
-	return nil
-}
-
-func (s *MapMarkSet) Mark(cid cid.Cid) error {
-	if s.ts {
-		s.mx.Lock()
-		defer s.mx.Unlock()
-	}
-
-	if s.set == nil {
-		return errMarkSetClosed
-	}
-
-	s.set[string(cid.Hash())] = struct{}{}
-	return nil
-}
-
-func (s *MapMarkSet) Has(cid cid.Cid) (bool, error) {
-	if s.ts {
-		s.mx.RLock()
-		defer s.mx.RUnlock()
-	}
-
-	if s.set == nil {
-		return false, errMarkSetClosed
-	}
-
-	_, ok := s.set[string(cid.Hash())]
-	return ok, nil
-}
-
-func (s *MapMarkSet) Close() error {
-	if s.ts {
-		s.mx.Lock()
-		defer s.mx.Unlock()
-	}
-	s.set = nil
-	return nil
-}
-
-func (s *MapMarkSet) SetConcurrent() {
-	s.ts = true
-}

blockstore/splitstore/markset_test.go

@@ -1,147 +0,0 @@
-package splitstore
-
-import (
-	"io/ioutil"
-	"testing"
-
-	cid "github.com/ipfs/go-cid"
-	"github.com/multiformats/go-multihash"
-)
-
-func TestMapMarkSet(t *testing.T) {
-	testMarkSet(t, "map")
-}
-
-func TestBloomMarkSet(t *testing.T) {
-	testMarkSet(t, "bloom")
-}
-
-func TestBadgerMarkSet(t *testing.T) {
-	bs := badgerMarkSetBatchSize
-	badgerMarkSetBatchSize = 1
-	t.Cleanup(func() {
-		badgerMarkSetBatchSize = bs
-	})
-	testMarkSet(t, "badger")
-}
-
-func testMarkSet(t *testing.T, lsType string) {
-	t.Helper()
-
-	path, err := ioutil.TempDir("", "sweep-test.*")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	env, err := OpenMarkSetEnv(path, lsType)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer env.Close() //nolint:errcheck
-
-	hotSet, err := env.Create("hot", 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	coldSet, err := env.Create("cold", 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	makeCid := func(key string) cid.Cid {
-		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		return cid.NewCidV1(cid.Raw, h)
-	}
-
-	mustHave := func(s MarkSet, cid cid.Cid) {
-		has, err := s.Has(cid)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if !has {
-			t.Fatal("mark not found")
-		}
-	}
-
-	mustNotHave := func(s MarkSet, cid cid.Cid) {
-		has, err := s.Has(cid)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if has {
-			t.Fatal("unexpected mark")
-		}
-	}
-
-	k1 := makeCid("a")
-	k2 := makeCid("b")
-	k3 := makeCid("c")
-	k4 := makeCid("d")
-
-	hotSet.Mark(k1)  //nolint
-	hotSet.Mark(k2)  //nolint
-	coldSet.Mark(k3) //nolint
-
-	mustHave(hotSet, k1)
-	mustHave(hotSet, k2)
-	mustNotHave(hotSet, k3)
-	mustNotHave(hotSet, k4)
-
-	mustNotHave(coldSet, k1)
-	mustNotHave(coldSet, k2)
-	mustHave(coldSet, k3)
-	mustNotHave(coldSet, k4)
-
-	// close them and reopen to redo the dance
-
-	err = hotSet.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = coldSet.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	hotSet, err = env.Create("hot", 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	coldSet, err = env.Create("cold", 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	hotSet.Mark(k3)  //nolint
-	hotSet.Mark(k4)  //nolint
-	coldSet.Mark(k1) //nolint
-
-	mustNotHave(hotSet, k1)
-	mustNotHave(hotSet, k2)
-	mustHave(hotSet, k3)
-	mustHave(hotSet, k4)
-
-	mustHave(coldSet, k1)
-	mustNotHave(coldSet, k2)
-	mustNotHave(coldSet, k3)
-	mustNotHave(coldSet, k4)
-
-	err = hotSet.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = coldSet.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-}

blockstore/splitstore/splitstore.go

@@ -1,580 +0,0 @@
-package splitstore
-
-import (
-	"context"
-	"errors"
-	"os"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"go.uber.org/multierr"
-	"golang.org/x/xerrors"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-	dstore "github.com/ipfs/go-datastore"
-	logging "github.com/ipfs/go-log/v2"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	bstore "github.com/filecoin-project/lotus/blockstore"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/metrics"
-
-	"go.opencensus.io/stats"
-)
-
-var (
-	// baseEpochKey stores the base epoch (last compaction epoch) in the
-	// metadata store.
-	baseEpochKey = dstore.NewKey("/splitstore/baseEpoch")
-
-	// warmupEpochKey stores whether a hot store warmup has been performed.
-	// On first start, the splitstore will walk the state tree and will copy
-	// all active blocks into the hotstore.
-	warmupEpochKey = dstore.NewKey("/splitstore/warmupEpoch")
-
-	// markSetSizeKey stores the current estimate for the mark set size.
-	// this is first computed at warmup and updated in every compaction
-	markSetSizeKey = dstore.NewKey("/splitstore/markSetSize")
-
-	// compactionIndexKey stores the compaction index (serial number)
-	compactionIndexKey = dstore.NewKey("/splitstore/compactionIndex")
-
-	log = logging.Logger("splitstore")
-
-	// set this to true if you are debugging the splitstore to enable debug logging
-	enableDebugLog = false
-	// set this to true if you want to track origin stack traces in the write log
-	enableDebugLogWriteTraces = false
-)
-
-func init() {
-	if os.Getenv("LOTUS_SPLITSTORE_DEBUG_LOG") == "1" {
-		enableDebugLog = true
-	}
-
-	if os.Getenv("LOTUS_SPLITSTORE_DEBUG_LOG_WRITE_TRACES") == "1" {
-		enableDebugLogWriteTraces = true
-	}
-}
-
-type Config struct {
-	// MarkSetType is the type of mark set to use.
-	//
-	// The default value is "map", which uses an in-memory map-backed markset.
-	// If you are constrained in memory (i.e. compaction runs out of memory), you
-	// can use "badger", which will use a disk-backed markset using badger.
-	// Note that compaction will take quite a bit longer when using the "badger" option,
-	// but that shouldn't really matter (as long as it is under 7.5hrs).
-	MarkSetType string
-
-	// DiscardColdBlocks indicates whether to skip moving cold blocks to the coldstore.
-	// If the splitstore is running with a noop coldstore then this option is set to true
-	// which skips moving (as it is a noop, but still takes time to read all the cold objects)
-	// and directly purges cold blocks.
-	DiscardColdBlocks bool
-
-	// HotstoreMessageRetention indicates the hotstore retention policy for messages.
-	// It has the following semantics:
-	// - a value of 0 will only retain messages within the compaction boundary (4 finalities)
-	// - a positive integer indicates the number of finalities, outside the compaction boundary,
-	//   for which messages will be retained in the hotstore.
-	HotStoreMessageRetention uint64
-
-	// HotstoreFullGCFrequency indicates how frequently (in terms of compactions) to garbage collect
-	// the hotstore using full (moving) GC if supported by the hotstore.
-	// A value of 0 disables full GC entirely.
-	// A positive value is the number of compactions before a full GC is performed;
-	// a value of 1 will perform full GC in every compaction.
-	HotStoreFullGCFrequency uint64
-}
-
-// ChainAccessor allows the Splitstore to access the chain. It will most likely
-// be a ChainStore at runtime.
-type ChainAccessor interface {
-	GetTipsetByHeight(context.Context, abi.ChainEpoch, *types.TipSet, bool) (*types.TipSet, error)
-	GetHeaviestTipSet() *types.TipSet
-	SubscribeHeadChanges(change func(revert []*types.TipSet, apply []*types.TipSet) error)
-}
-
-// hotstore is the interface that must be satisfied by the hot blockstore; it is an extension
-// of the Blockstore interface with the traits we need for compaction.
-type hotstore interface {
-	bstore.Blockstore
-	bstore.BlockstoreIterator
-}
-
-type SplitStore struct {
-	compacting int32 // compaction/prune/warmup in progress
-	closing    int32 // the splitstore is closing
-
-	cfg  *Config
-	path string
-
-	mx          sync.Mutex
-	warmupEpoch abi.ChainEpoch // protected by mx
-	baseEpoch   abi.ChainEpoch // protected by compaction lock
-
-	headChangeMx sync.Mutex
-
-	coldPurgeSize int
-
-	chain ChainAccessor
-	ds    dstore.Datastore
-	cold  bstore.Blockstore
-	hot   hotstore
-
-	markSetEnv  MarkSetEnv
-	markSetSize int64
-
-	compactionIndex int64
-
-	ctx    context.Context
-	cancel func()
-
-	debug *debugLog
-
-	// transactional protection for concurrent read/writes during compaction
-	txnLk           sync.RWMutex
-	txnViewsMx      sync.Mutex
-	txnViewsCond    sync.Cond
-	txnViews        int
-	txnViewsWaiting bool
-	txnActive       bool
-	txnProtect      MarkSet
-	txnRefsMx       sync.Mutex
-	txnRefs         map[cid.Cid]struct{}
-	txnMissing      map[cid.Cid]struct{}
-
-	// registered protectors
-	protectors []func(func(cid.Cid) error) error
-}
-
-var _ bstore.Blockstore = (*SplitStore)(nil)
-
-// Open opens an existing splistore, or creates a new splitstore. The splitstore
-// is backed by the provided hot and cold stores. The returned SplitStore MUST be
-// attached to the ChainStore with Start in order to trigger compaction.
-func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Config) (*SplitStore, error) {
-	// hot blockstore must support the hotstore interface
-	hots, ok := hot.(hotstore)
-	if !ok {
-		// be specific about what is missing
-		if _, ok := hot.(bstore.BlockstoreIterator); !ok {
-			return nil, xerrors.Errorf("hot blockstore does not support efficient iteration: %T", hot)
-		}
-
-		return nil, xerrors.Errorf("hot blockstore does not support the necessary traits: %T", hot)
-	}
-
-	// the markset env
-	markSetEnv, err := OpenMarkSetEnv(path, cfg.MarkSetType)
-	if err != nil {
-		return nil, err
-	}
-
-	// and now we can make a SplitStore
-	ss := &SplitStore{
-		cfg:        cfg,
-		path:       path,
-		ds:         ds,
-		cold:       cold,
-		hot:        hots,
-		markSetEnv: markSetEnv,
-
-		coldPurgeSize: defaultColdPurgeSize,
-	}
-
-	ss.txnViewsCond.L = &ss.txnViewsMx
-	ss.ctx, ss.cancel = context.WithCancel(context.Background())
-
-	if enableDebugLog {
-		ss.debug, err = openDebugLog(path)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return ss, nil
-}
-
-// Blockstore interface
-func (s *SplitStore) DeleteBlock(_ cid.Cid) error {
-	// afaict we don't seem to be using this method, so it's not implemented
-	return errors.New("DeleteBlock not implemented on SplitStore; don't do this Luke!") //nolint
-}
-
-func (s *SplitStore) DeleteMany(_ []cid.Cid) error {
-	// afaict we don't seem to be using this method, so it's not implemented
-	return errors.New("DeleteMany not implemented on SplitStore; don't do this Luke!") //nolint
-}
-
-func (s *SplitStore) Has(cid cid.Cid) (bool, error) {
-	if isIdentiyCid(cid) {
-		return true, nil
-	}
-
-	s.txnLk.RLock()
-	defer s.txnLk.RUnlock()
-
-	has, err := s.hot.Has(cid)
-
-	if err != nil {
-		return has, err
-	}
-
-	if has {
-		s.trackTxnRef(cid)
-		return true, nil
-	}
-
-	return s.cold.Has(cid)
-}
-
-func (s *SplitStore) Get(cid cid.Cid) (blocks.Block, error) {
-	if isIdentiyCid(cid) {
-		data, err := decodeIdentityCid(cid)
-		if err != nil {
-			return nil, err
-		}
-
-		return blocks.NewBlockWithCid(data, cid)
-	}
-
-	s.txnLk.RLock()
-	defer s.txnLk.RUnlock()
-
-	blk, err := s.hot.Get(cid)
-
-	switch err {
-	case nil:
-		s.trackTxnRef(cid)
-		return blk, nil
-
-	case bstore.ErrNotFound:
-		if s.isWarm() {
-			s.debug.LogReadMiss(cid)
-		}
-
-		blk, err = s.cold.Get(cid)
-		if err == nil {
-			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
-
-		}
-		return blk, err
-
-	default:
-		return nil, err
-	}
-}
-
-func (s *SplitStore) GetSize(cid cid.Cid) (int, error) {
-	if isIdentiyCid(cid) {
-		data, err := decodeIdentityCid(cid)
-		if err != nil {
-			return 0, err
-		}
-
-		return len(data), nil
-	}
-
-	s.txnLk.RLock()
-	defer s.txnLk.RUnlock()
-
-	size, err := s.hot.GetSize(cid)
-
-	switch err {
-	case nil:
-		s.trackTxnRef(cid)
-		return size, nil
-
-	case bstore.ErrNotFound:
-		if s.isWarm() {
-			s.debug.LogReadMiss(cid)
-		}
-
-		size, err = s.cold.GetSize(cid)
-		if err == nil {
-			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
-		}
-		return size, err
-
-	default:
-		return 0, err
-	}
-}
-
-func (s *SplitStore) Put(blk blocks.Block) error {
-	if isIdentiyCid(blk.Cid()) {
-		return nil
-	}
-
-	s.txnLk.RLock()
-	defer s.txnLk.RUnlock()
-
-	err := s.hot.Put(blk)
-	if err != nil {
-		return err
-	}
-
-	s.debug.LogWrite(blk)
-
-	s.trackTxnRef(blk.Cid())
-	return nil
-}
-
-func (s *SplitStore) PutMany(blks []blocks.Block) error {
-	// filter identites
-	idcids := 0
-	for _, blk := range blks {
-		if isIdentiyCid(blk.Cid()) {
-			idcids++
-		}
-	}
-
-	if idcids > 0 {
-		if idcids == len(blks) {
-			// it's all identities
-			return nil
-		}
-
-		filtered := make([]blocks.Block, 0, len(blks)-idcids)
-		for _, blk := range blks {
-			if isIdentiyCid(blk.Cid()) {
-				continue
-			}
-			filtered = append(filtered, blk)
-		}
-
-		blks = filtered
-	}
-
-	batch := make([]cid.Cid, 0, len(blks))
-	for _, blk := range blks {
-		batch = append(batch, blk.Cid())
-	}
-
-	s.txnLk.RLock()
-	defer s.txnLk.RUnlock()
-
-	err := s.hot.PutMany(blks)
-	if err != nil {
-		return err
-	}
-
-	s.debug.LogWriteMany(blks)
-
-	s.trackTxnRefMany(batch)
-	return nil
-}
-
-func (s *SplitStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	ctx, cancel := context.WithCancel(ctx)
-
-	chHot, err := s.hot.AllKeysChan(ctx)
-	if err != nil {
-		cancel()
-		return nil, err
-	}
-
-	chCold, err := s.cold.AllKeysChan(ctx)
-	if err != nil {
-		cancel()
-		return nil, err
-	}
-
-	seen := cid.NewSet()
-	ch := make(chan cid.Cid, 8) // buffer is arbitrary, just enough to avoid context switches
-	go func() {
-		defer cancel()
-		defer close(ch)
-
-		for _, in := range []<-chan cid.Cid{chHot, chCold} {
-			for c := range in {
-				// ensure we only emit each key once
-				if !seen.Visit(c) {
-					continue
-				}
-
-				select {
-				case ch <- c:
-				case <-ctx.Done():
-					return
-				}
-			}
-		}
-	}()
-
-	return ch, nil
-}
-
-func (s *SplitStore) HashOnRead(enabled bool) {
-	s.hot.HashOnRead(enabled)
-	s.cold.HashOnRead(enabled)
-}
-
-func (s *SplitStore) View(cid cid.Cid, cb func([]byte) error) error {
-	if isIdentiyCid(cid) {
-		data, err := decodeIdentityCid(cid)
-		if err != nil {
-			return err
-		}
-
-		return cb(data)
-	}
-
-	// views are (optimistically) protected two-fold:
-	// - if there is an active transaction, then the reference is protected.
-	// - if there is no active transaction, active views are tracked in a
-	//   wait group and compaction is inhibited from starting until they
-	//   have all completed.  this is necessary to ensure that a (very) long-running
-	//   view can't have its data pointer deleted, which would be catastrophic.
-	//   Note that we can't just RLock for the duration of the view, as this could
-	//    lead to deadlock with recursive views.
-	s.protectView(cid)
-	defer s.viewDone()
-
-	err := s.hot.View(cid, cb)
-	switch err {
-	case bstore.ErrNotFound:
-		if s.isWarm() {
-			s.debug.LogReadMiss(cid)
-		}
-
-		err = s.cold.View(cid, cb)
-		if err == nil {
-			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
-		}
-		return err
-
-	default:
-		return err
-	}
-}
-
-func (s *SplitStore) isWarm() bool {
-	s.mx.Lock()
-	defer s.mx.Unlock()
-	return s.warmupEpoch > 0
-}
-
-// State tracking
-func (s *SplitStore) Start(chain ChainAccessor) error {
-	s.chain = chain
-	curTs := chain.GetHeaviestTipSet()
-
-	// should we warmup
-	warmup := false
-
-	// load base epoch from metadata ds
-	// if none, then use current epoch because it's a fresh start
-	bs, err := s.ds.Get(baseEpochKey)
-	switch err {
-	case nil:
-		s.baseEpoch = bytesToEpoch(bs)
-
-	case dstore.ErrNotFound:
-		if curTs == nil {
-			// this can happen in some tests
-			break
-		}
-
-		err = s.setBaseEpoch(curTs.Height())
-		if err != nil {
-			return xerrors.Errorf("error saving base epoch: %w", err)
-		}
-
-	default:
-		return xerrors.Errorf("error loading base epoch: %w", err)
-	}
-
-	// load warmup epoch from metadata ds
-	bs, err = s.ds.Get(warmupEpochKey)
-	switch err {
-	case nil:
-		s.warmupEpoch = bytesToEpoch(bs)
-
-	case dstore.ErrNotFound:
-		warmup = true
-
-	default:
-		return xerrors.Errorf("error loading warmup epoch: %w", err)
-	}
-
-	// load markSetSize from metadata ds to provide a size hint for marksets
-	bs, err = s.ds.Get(markSetSizeKey)
-	switch err {
-	case nil:
-		s.markSetSize = bytesToInt64(bs)
-
-	case dstore.ErrNotFound:
-	default:
-		return xerrors.Errorf("error loading mark set size: %w", err)
-	}
-
-	// load compactionIndex from metadata ds to provide a hint as to when to perform moving gc
-	bs, err = s.ds.Get(compactionIndexKey)
-	switch err {
-	case nil:
-		s.compactionIndex = bytesToInt64(bs)
-
-	case dstore.ErrNotFound:
-		// this is potentially an upgrade from splitstore v0; schedule a warmup as v0 has
-		// some issues with hot references leaking into the coldstore.
-		warmup = true
-	default:
-		return xerrors.Errorf("error loading compaction index: %w", err)
-	}
-
-	log.Infow("starting splitstore", "baseEpoch", s.baseEpoch, "warmupEpoch", s.warmupEpoch)
-
-	if warmup {
-		err = s.warmup(curTs)
-		if err != nil {
-			return xerrors.Errorf("error starting warmup: %w", err)
-		}
-	}
-
-	// watch the chain
-	chain.SubscribeHeadChanges(s.HeadChange)
-
-	return nil
-}
-
-func (s *SplitStore) AddProtector(protector func(func(cid.Cid) error) error) {
-	s.mx.Lock()
-	defer s.mx.Unlock()
-
-	s.protectors = append(s.protectors, protector)
-}
-
-func (s *SplitStore) Close() error {
-	if !atomic.CompareAndSwapInt32(&s.closing, 0, 1) {
-		// already closing
-		return nil
-	}
-
-	if atomic.LoadInt32(&s.compacting) == 1 {
-		log.Warn("close with ongoing compaction in progress; waiting for it to finish...")
-		for atomic.LoadInt32(&s.compacting) == 1 {
-			time.Sleep(time.Second)
-		}
-	}
-
-	s.cancel()
-	return multierr.Combine(s.markSetEnv.Close(), s.debug.Close())
-}
-
-func (s *SplitStore) checkClosing() error {
-	if atomic.LoadInt32(&s.closing) == 1 {
-		return xerrors.Errorf("splitstore is closing")
-	}
-
-	return nil
-}
-
-func (s *SplitStore) setBaseEpoch(epoch abi.ChainEpoch) error {
-	s.baseEpoch = epoch
-	return s.ds.Put(baseEpochKey, epochToBytes(epoch))
-}

blockstore/splitstore/splitstore_check.go

@@ -1,150 +0,0 @@
-package splitstore
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"sync/atomic"
-	"time"
-
-	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
-
-	bstore "github.com/filecoin-project/lotus/blockstore"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-// performs an asynchronous health-check on the splitstore; results are appended to
-// <splitstore-path>/check.txt
-func (s *SplitStore) Check() error {
-	s.headChangeMx.Lock()
-	defer s.headChangeMx.Unlock()
-
-	// try to take compaction lock and inhibit compaction while the health-check is running
-	if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
-		return xerrors.Errorf("can't acquire compaction lock; compacting operation in progress")
-	}
-
-	if s.compactionIndex == 0 {
-		atomic.StoreInt32(&s.compacting, 0)
-		return xerrors.Errorf("splitstore hasn't compacted yet; health check is not meaningful")
-	}
-
-	// check if we are actually closing first
-	if err := s.checkClosing(); err != nil {
-		atomic.StoreInt32(&s.compacting, 0)
-		return err
-	}
-
-	curTs := s.chain.GetHeaviestTipSet()
-	go func() {
-		defer atomic.StoreInt32(&s.compacting, 0)
-
-		log.Info("checking splitstore health")
-		start := time.Now()
-
-		err := s.doCheck(curTs)
-		if err != nil {
-			log.Errorf("error checking splitstore health: %s", err)
-			return
-		}
-
-		log.Infow("health check done", "took", time.Since(start))
-	}()
-
-	return nil
-}
-
-func (s *SplitStore) doCheck(curTs *types.TipSet) error {
-	currentEpoch := curTs.Height()
-	boundaryEpoch := currentEpoch - CompactionBoundary
-
-	outputPath := filepath.Join(s.path, "check.txt")
-	output, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
-	if err != nil {
-		return xerrors.Errorf("error opening check output file %s: %w", outputPath, err)
-	}
-	defer output.Close() //nolint:errcheck
-
-	write := func(format string, args ...interface{}) {
-		_, err := fmt.Fprintf(output, format+"\n", args...)
-		if err != nil {
-			log.Warnf("error writing check output: %s", err)
-		}
-	}
-
-	ts, _ := time.Now().MarshalText()
-	write("---------------------------------------------")
-	write("start check at %s", ts)
-	write("current epoch: %d", currentEpoch)
-	write("boundary epoch: %d", boundaryEpoch)
-	write("compaction index: %d", s.compactionIndex)
-	write("--")
-
-	var coldCnt, missingCnt int64
-	err = s.walkChain(curTs, boundaryEpoch, boundaryEpoch,
-		func(c cid.Cid) error {
-			if isUnitaryObject(c) {
-				return errStopWalk
-			}
-
-			has, err := s.hot.Has(c)
-			if err != nil {
-				return xerrors.Errorf("error checking hotstore: %w", err)
-			}
-
-			if has {
-				return nil
-			}
-
-			has, err = s.cold.Has(c)
-			if err != nil {
-				return xerrors.Errorf("error checking coldstore: %w", err)
-			}
-
-			if has {
-				coldCnt++
-				write("cold object reference: %s", c)
-			} else {
-				missingCnt++
-				write("missing object reference: %s", c)
-				return errStopWalk
-			}
-
-			return nil
-		})
-
-	if err != nil {
-		err = xerrors.Errorf("error walking chain: %w", err)
-		write("ERROR: %s", err)
-		return err
-	}
-
-	log.Infow("check done", "cold", coldCnt, "missing", missingCnt)
-	write("--")
-	write("cold: %d missing: %d", coldCnt, missingCnt)
-	write("DONE")
-
-	return nil
-}
-
-// provides some basic information about the splitstore
-func (s *SplitStore) Info() map[string]interface{} {
-	info := make(map[string]interface{})
-	info["base epoch"] = s.baseEpoch
-	info["warmup epoch"] = s.warmupEpoch
-	info["compactions"] = s.compactionIndex
-
-	sizer, ok := s.hot.(bstore.BlockstoreSize)
-	if ok {
-		size, err := sizer.Size()
-		if err != nil {
-			log.Warnf("error getting hotstore size: %s", err)
-		} else {
-			info["hotstore size"] = size
-		}
-	}
-
-	return info
-}

blockstore/splitstore/splitstore_expose.go

@@ -1,114 +0,0 @@
-package splitstore
-
-import (
-	"context"
-	"errors"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-
-	bstore "github.com/filecoin-project/lotus/blockstore"
-)
-
-type exposedSplitStore struct {
-	s *SplitStore
-}
-
-var _ bstore.Blockstore = (*exposedSplitStore)(nil)
-
-func (s *SplitStore) Expose() bstore.Blockstore {
-	return &exposedSplitStore{s: s}
-}
-
-func (es *exposedSplitStore) DeleteBlock(_ cid.Cid) error {
-	return errors.New("DeleteBlock: operation not supported")
-}
-
-func (es *exposedSplitStore) DeleteMany(_ []cid.Cid) error {
-	return errors.New("DeleteMany: operation not supported")
-}
-
-func (es *exposedSplitStore) Has(c cid.Cid) (bool, error) {
-	if isIdentiyCid(c) {
-		return true, nil
-	}
-
-	has, err := es.s.hot.Has(c)
-	if has || err != nil {
-		return has, err
-	}
-
-	return es.s.cold.Has(c)
-}
-
-func (es *exposedSplitStore) Get(c cid.Cid) (blocks.Block, error) {
-	if isIdentiyCid(c) {
-		data, err := decodeIdentityCid(c)
-		if err != nil {
-			return nil, err
-		}
-
-		return blocks.NewBlockWithCid(data, c)
-	}
-
-	blk, err := es.s.hot.Get(c)
-	switch err {
-	case bstore.ErrNotFound:
-		return es.s.cold.Get(c)
-	default:
-		return blk, err
-	}
-}
-
-func (es *exposedSplitStore) GetSize(c cid.Cid) (int, error) {
-	if isIdentiyCid(c) {
-		data, err := decodeIdentityCid(c)
-		if err != nil {
-			return 0, err
-		}
-
-		return len(data), nil
-	}
-
-	size, err := es.s.hot.GetSize(c)
-	switch err {
-	case bstore.ErrNotFound:
-		return es.s.cold.GetSize(c)
-	default:
-		return size, err
-	}
-}
-
-func (es *exposedSplitStore) Put(blk blocks.Block) error {
-	return es.s.Put(blk)
-}
-
-func (es *exposedSplitStore) PutMany(blks []blocks.Block) error {
-	return es.s.PutMany(blks)
-}
-
-func (es *exposedSplitStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return es.s.AllKeysChan(ctx)
-}
-
-func (es *exposedSplitStore) HashOnRead(enabled bool) {}
-
-func (es *exposedSplitStore) View(c cid.Cid, f func([]byte) error) error {
-	if isIdentiyCid(c) {
-		data, err := decodeIdentityCid(c)
-		if err != nil {
-			return err
-		}
-
-		return f(data)
-	}
-
-	err := es.s.hot.View(c, f)
-	switch err {
-	case bstore.ErrNotFound:
-		return es.s.cold.View(c, f)
-
-	default:
-		return err
-	}
-}

blockstore/splitstore/splitstore_gc.go

@@ -1,35 +0,0 @@
-package splitstore
-
-import (
-	"fmt"
-	"time"
-
-	bstore "github.com/filecoin-project/lotus/blockstore"
-)
-
-func (s *SplitStore) gcHotstore() {
-	var opts []bstore.BlockstoreGCOption
-	if s.cfg.HotStoreFullGCFrequency > 0 && s.compactionIndex%int64(s.cfg.HotStoreFullGCFrequency) == 0 {
-		opts = append(opts, bstore.WithFullGC(true))
-	}
-
-	if err := s.gcBlockstore(s.hot, opts); err != nil {
-		log.Warnf("error garbage collecting hostore: %s", err)
-	}
-}
-
-func (s *SplitStore) gcBlockstore(b bstore.Blockstore, opts []bstore.BlockstoreGCOption) error {
-	if gc, ok := b.(bstore.BlockstoreGC); ok {
-		log.Info("garbage collecting blockstore")
-		startGC := time.Now()
-
-		if err := gc.CollectGarbage(opts...); err != nil {
-			return err
-		}
-
-		log.Infow("garbage collecting hotstore done", "took", time.Since(startGC))
-		return nil
-	}
-
-	return fmt.Errorf("blockstore doesn't support garbage collection: %T", b)
-}

blockstore/splitstore/splitstore_test.go

@@ -1,391 +0,0 @@
-package splitstore
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/blockstore"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/chain/types/mock"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-	datastore "github.com/ipfs/go-datastore"
-	dssync "github.com/ipfs/go-datastore/sync"
-	logging "github.com/ipfs/go-log/v2"
-)
-
-func init() {
-	CompactionThreshold = 5
-	CompactionBoundary = 2
-	WarmupBoundary = 0
-	logging.SetLogLevel("splitstore", "DEBUG")
-}
-
-func testSplitStore(t *testing.T, cfg *Config) {
-	chain := &mockChain{t: t}
-
-	// the myriads of stores
-	ds := dssync.MutexWrap(datastore.NewMapDatastore())
-	hot := newMockStore()
-	cold := newMockStore()
-
-	// this is necessary to avoid the garbage mock puts in the blocks
-	garbage := blocks.NewBlock([]byte{1, 2, 3})
-	err := cold.Put(garbage)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// genesis
-	genBlock := mock.MkBlock(nil, 0, 0)
-	genBlock.Messages = garbage.Cid()
-	genBlock.ParentMessageReceipts = garbage.Cid()
-	genBlock.ParentStateRoot = garbage.Cid()
-	genBlock.Timestamp = uint64(time.Now().Unix())
-
-	genTs := mock.TipSet(genBlock)
-	chain.push(genTs)
-
-	// put the genesis block to cold store
-	blk, err := genBlock.ToStorageBlock()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = cold.Put(blk)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// create a garbage block that is protected with a rgistered protector
-	protected := blocks.NewBlock([]byte("protected!"))
-	err = hot.Put(protected)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// and another one that is not protected
-	unprotected := blocks.NewBlock([]byte("unprotected!"))
-	err = hot.Put(unprotected)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// open the splitstore
-	ss, err := Open("", ds, hot, cold, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer ss.Close() //nolint
-
-	// register our protector
-	ss.AddProtector(func(protect func(cid.Cid) error) error {
-		return protect(protected.Cid())
-	})
-
-	err = ss.Start(chain)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// make some tipsets, but not enough to cause compaction
-	mkBlock := func(curTs *types.TipSet, i int, stateRoot blocks.Block) *types.TipSet {
-		blk := mock.MkBlock(curTs, uint64(i), uint64(i))
-
-		blk.Messages = garbage.Cid()
-		blk.ParentMessageReceipts = garbage.Cid()
-		blk.ParentStateRoot = stateRoot.Cid()
-		blk.Timestamp = uint64(time.Now().Unix())
-
-		sblk, err := blk.ToStorageBlock()
-		if err != nil {
-			t.Fatal(err)
-		}
-		err = ss.Put(stateRoot)
-		if err != nil {
-			t.Fatal(err)
-		}
-		err = ss.Put(sblk)
-		if err != nil {
-			t.Fatal(err)
-		}
-		ts := mock.TipSet(blk)
-		chain.push(ts)
-
-		return ts
-	}
-
-	waitForCompaction := func() {
-		for atomic.LoadInt32(&ss.compacting) == 1 {
-			time.Sleep(100 * time.Millisecond)
-		}
-	}
-
-	curTs := genTs
-	for i := 1; i < 5; i++ {
-		stateRoot := blocks.NewBlock([]byte{byte(i), 3, 3, 7})
-		curTs = mkBlock(curTs, i, stateRoot)
-		waitForCompaction()
-	}
-
-	// count objects in the cold and hot stores
-	countBlocks := func(bs blockstore.Blockstore) int {
-		count := 0
-		_ = bs.(blockstore.BlockstoreIterator).ForEachKey(func(_ cid.Cid) error {
-			count++
-			return nil
-		})
-		return count
-	}
-
-	coldCnt := countBlocks(cold)
-	hotCnt := countBlocks(hot)
-
-	if coldCnt != 2 {
-		t.Errorf("expected %d blocks, but got %d", 2, coldCnt)
-	}
-
-	if hotCnt != 12 {
-		t.Errorf("expected %d blocks, but got %d", 12, hotCnt)
-	}
-
-	// trigger a compaction
-	for i := 5; i < 10; i++ {
-		stateRoot := blocks.NewBlock([]byte{byte(i), 3, 3, 7})
-		curTs = mkBlock(curTs, i, stateRoot)
-		waitForCompaction()
-	}
-
-	coldCnt = countBlocks(cold)
-	hotCnt = countBlocks(hot)
-
-	if coldCnt != 6 {
-		t.Errorf("expected %d cold blocks, but got %d", 6, coldCnt)
-	}
-
-	if hotCnt != 18 {
-		t.Errorf("expected %d hot blocks, but got %d", 18, hotCnt)
-	}
-
-	// ensure our protected block is still there
-	has, err := hot.Has(protected.Cid())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !has {
-		t.Fatal("protected block is missing from hotstore")
-	}
-
-	// ensure our unprotected block is in the coldstore now
-	has, err = hot.Has(unprotected.Cid())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if has {
-		t.Fatal("unprotected block is still in hotstore")
-	}
-
-	has, err = cold.Has(unprotected.Cid())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !has {
-		t.Fatal("unprotected block is missing from coldstore")
-	}
-
-	// Make sure we can revert without panicking.
-	chain.revert(2)
-}
-
-func TestSplitStoreCompaction(t *testing.T) {
-	testSplitStore(t, &Config{MarkSetType: "map"})
-}
-
-func TestSplitStoreCompactionWithBadger(t *testing.T) {
-	bs := badgerMarkSetBatchSize
-	badgerMarkSetBatchSize = 1
-	t.Cleanup(func() {
-		badgerMarkSetBatchSize = bs
-	})
-	testSplitStore(t, &Config{MarkSetType: "badger"})
-}
-
-type mockChain struct {
-	t testing.TB
-
-	sync.Mutex
-	genesis  *types.BlockHeader
-	tipsets  []*types.TipSet
-	listener func(revert []*types.TipSet, apply []*types.TipSet) error
-}
-
-func (c *mockChain) push(ts *types.TipSet) {
-	c.Lock()
-	c.tipsets = append(c.tipsets, ts)
-	if c.genesis == nil {
-		c.genesis = ts.Blocks()[0]
-	}
-	c.Unlock()
-
-	if c.listener != nil {
-		err := c.listener(nil, []*types.TipSet{ts})
-		if err != nil {
-			c.t.Errorf("mockchain: error dispatching listener: %s", err)
-		}
-	}
-}
-
-func (c *mockChain) revert(count int) {
-	c.Lock()
-	revert := make([]*types.TipSet, count)
-	if count > len(c.tipsets) {
-		c.Unlock()
-		c.t.Fatalf("not enough tipsets to revert")
-	}
-	copy(revert, c.tipsets[len(c.tipsets)-count:])
-	c.tipsets = c.tipsets[:len(c.tipsets)-count]
-	c.Unlock()
-
-	if c.listener != nil {
-		err := c.listener(revert, nil)
-		if err != nil {
-			c.t.Errorf("mockchain: error dispatching listener: %s", err)
-		}
-	}
-}
-
-func (c *mockChain) GetTipsetByHeight(_ context.Context, epoch abi.ChainEpoch, _ *types.TipSet, _ bool) (*types.TipSet, error) {
-	c.Lock()
-	defer c.Unlock()
-
-	iEpoch := int(epoch)
-	if iEpoch > len(c.tipsets) {
-		return nil, fmt.Errorf("bad epoch %d", epoch)
-	}
-
-	return c.tipsets[iEpoch], nil
-}
-
-func (c *mockChain) GetHeaviestTipSet() *types.TipSet {
-	c.Lock()
-	defer c.Unlock()
-
-	return c.tipsets[len(c.tipsets)-1]
-}
-
-func (c *mockChain) SubscribeHeadChanges(change func(revert []*types.TipSet, apply []*types.TipSet) error) {
-	c.listener = change
-}
-
-type mockStore struct {
-	mx  sync.Mutex
-	set map[cid.Cid]blocks.Block
-}
-
-func newMockStore() *mockStore {
-	return &mockStore{set: make(map[cid.Cid]blocks.Block)}
-}
-
-func (b *mockStore) Has(cid cid.Cid) (bool, error) {
-	b.mx.Lock()
-	defer b.mx.Unlock()
-	_, ok := b.set[cid]
-	return ok, nil
-}
-
-func (b *mockStore) HashOnRead(hor bool) {}
-
-func (b *mockStore) Get(cid cid.Cid) (blocks.Block, error) {
-	b.mx.Lock()
-	defer b.mx.Unlock()
-
-	blk, ok := b.set[cid]
-	if !ok {
-		return nil, blockstore.ErrNotFound
-	}
-	return blk, nil
-}
-
-func (b *mockStore) GetSize(cid cid.Cid) (int, error) {
-	blk, err := b.Get(cid)
-	if err != nil {
-		return 0, err
-	}
-
-	return len(blk.RawData()), nil
-}
-
-func (b *mockStore) View(cid cid.Cid, f func([]byte) error) error {
-	blk, err := b.Get(cid)
-	if err != nil {
-		return err
-	}
-	return f(blk.RawData())
-}
-
-func (b *mockStore) Put(blk blocks.Block) error {
-	b.mx.Lock()
-	defer b.mx.Unlock()
-
-	b.set[blk.Cid()] = blk
-	return nil
-}
-
-func (b *mockStore) PutMany(blks []blocks.Block) error {
-	b.mx.Lock()
-	defer b.mx.Unlock()
-
-	for _, blk := range blks {
-		b.set[blk.Cid()] = blk
-	}
-	return nil
-}
-
-func (b *mockStore) DeleteBlock(cid cid.Cid) error {
-	b.mx.Lock()
-	defer b.mx.Unlock()
-
-	delete(b.set, cid)
-	return nil
-}
-
-func (b *mockStore) DeleteMany(cids []cid.Cid) error {
-	b.mx.Lock()
-	defer b.mx.Unlock()
-
-	for _, c := range cids {
-		delete(b.set, c)
-	}
-	return nil
-}
-
-func (b *mockStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return nil, errors.New("not implemented")
-}
-
-func (b *mockStore) ForEachKey(f func(cid.Cid) error) error {
-	b.mx.Lock()
-	defer b.mx.Unlock()
-
-	for c := range b.set {
-		err := f(c)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (b *mockStore) Close() error {
-	return nil
-}

blockstore/splitstore/splitstore_util.go

@@ -1,67 +0,0 @@
-package splitstore
-
-import (
-	"encoding/binary"
-
-	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
-	mh "github.com/multiformats/go-multihash"
-
-	"github.com/filecoin-project/go-state-types/abi"
-)
-
-func epochToBytes(epoch abi.ChainEpoch) []byte {
-	return uint64ToBytes(uint64(epoch))
-}
-
-func bytesToEpoch(buf []byte) abi.ChainEpoch {
-	return abi.ChainEpoch(bytesToUint64(buf))
-}
-
-func int64ToBytes(i int64) []byte {
-	return uint64ToBytes(uint64(i))
-}
-
-func bytesToInt64(buf []byte) int64 {
-	return int64(bytesToUint64(buf))
-}
-
-func uint64ToBytes(i uint64) []byte {
-	buf := make([]byte, 16)
-	n := binary.PutUvarint(buf, i)
-	return buf[:n]
-}
-
-func bytesToUint64(buf []byte) uint64 {
-	i, _ := binary.Uvarint(buf)
-	return i
-}
-
-func isUnitaryObject(c cid.Cid) bool {
-	pre := c.Prefix()
-	switch pre.Codec {
-	case cid.FilCommitmentSealed, cid.FilCommitmentUnsealed:
-		return true
-	default:
-		return pre.MhType == mh.IDENTITY
-	}
-}
-
-func isIdentiyCid(c cid.Cid) bool {
-	return c.Prefix().MhType == mh.IDENTITY
-}
-
-func decodeIdentityCid(c cid.Cid) ([]byte, error) {
-	dmh, err := mh.Decode(c.Hash())
-	if err != nil {
-		return nil, xerrors.Errorf("error decoding identity cid %s: %w", c, err)
-	}
-
-	// sanity check
-	if dmh.Code != mh.IDENTITY {
-		return nil, xerrors.Errorf("error decoding identity cid %s: hash type is not identity", c)
-	}
-
-	return dmh.Digest, nil
-}

blockstore/splitstore/splitstore_warmup.go

@@ -1,137 +0,0 @@
-package splitstore
-
-import (
-	"sync/atomic"
-	"time"
-
-	"golang.org/x/xerrors"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	bstore "github.com/filecoin-project/lotus/blockstore"
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-var (
-	// WarmupBoundary is the number of epochs to load state during warmup.
-	WarmupBoundary = build.Finality
-)
-
-// warmup acuiqres the compaction lock and spawns a goroutine to warm up the hotstore;
-// this is necessary when we sync from a snapshot or when we enable the splitstore
-// on top of an existing blockstore (which becomes the coldstore).
-func (s *SplitStore) warmup(curTs *types.TipSet) error {
-	if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
-		return xerrors.Errorf("error locking compaction")
-	}
-
-	go func() {
-		defer atomic.StoreInt32(&s.compacting, 0)
-
-		log.Info("warming up hotstore")
-		start := time.Now()
-
-		err := s.doWarmup(curTs)
-		if err != nil {
-			log.Errorf("error warming up hotstore: %s", err)
-			return
-		}
-
-		log.Infow("warm up done", "took", time.Since(start))
-	}()
-
-	return nil
-}
-
-// the actual warmup procedure; it walks the chain loading all state roots at the boundary
-// and headers all the way up to genesis.
-// objects are written in batches so as to minimize overhead.
-func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
-	var boundaryEpoch abi.ChainEpoch
-	epoch := curTs.Height()
-	if WarmupBoundary < epoch {
-		boundaryEpoch = epoch - WarmupBoundary
-	}
-	batchHot := make([]blocks.Block, 0, batchSize)
-	count := int64(0)
-	xcount := int64(0)
-	missing := int64(0)
-	err := s.walkChain(curTs, boundaryEpoch, epoch+1, // we don't load messages/receipts in warmup
-		func(c cid.Cid) error {
-			if isUnitaryObject(c) {
-				return errStopWalk
-			}
-
-			count++
-
-			has, err := s.hot.Has(c)
-			if err != nil {
-				return err
-			}
-
-			if has {
-				return nil
-			}
-
-			blk, err := s.cold.Get(c)
-			if err != nil {
-				if err == bstore.ErrNotFound {
-					missing++
-					return errStopWalk
-				}
-				return err
-			}
-
-			xcount++
-
-			batchHot = append(batchHot, blk)
-			if len(batchHot) == batchSize {
-				err = s.hot.PutMany(batchHot)
-				if err != nil {
-					return err
-				}
-				batchHot = batchHot[:0]
-			}
-
-			return nil
-		})
-
-	if err != nil {
-		return err
-	}
-
-	if len(batchHot) > 0 {
-		err = s.hot.PutMany(batchHot)
-		if err != nil {
-			return err
-		}
-	}
-
-	log.Infow("warmup stats", "visited", count, "warm", xcount, "missing", missing)
-
-	s.markSetSize = count + count>>2 // overestimate a bit
-	err = s.ds.Put(markSetSizeKey, int64ToBytes(s.markSetSize))
-	if err != nil {
-		log.Warnf("error saving mark set size: %s", err)
-	}
-
-	// save the warmup epoch
-	err = s.ds.Put(warmupEpochKey, epochToBytes(epoch))
-	if err != nil {
-		return xerrors.Errorf("error saving warm up epoch: %w", err)
-	}
-	s.mx.Lock()
-	s.warmupEpoch = epoch
-	s.mx.Unlock()
-
-	// also save the compactionIndex, as this is used as an indicator of warmup for upgraded nodes
-	err = s.ds.Put(compactionIndexKey, int64ToBytes(s.compactionIndex))
-	if err != nil {
-		return xerrors.Errorf("error saving compaction index: %w", err)
-	}
-
-	return nil
-}

blockstore/sync.go

@@ -1,81 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"sync"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-)
-
-// NewMemorySync returns a thread-safe in-memory blockstore.
-func NewMemorySync() *SyncBlockstore {
-	return &SyncBlockstore{bs: make(MemBlockstore)}
-}
-
-// SyncBlockstore is a terminal blockstore that is a synchronized version
-// of MemBlockstore.
-type SyncBlockstore struct {
-	mu sync.RWMutex
-	bs MemBlockstore // specifically use a memStore to save indirection overhead.
-}
-
-func (m *SyncBlockstore) DeleteBlock(k cid.Cid) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return m.bs.DeleteBlock(k)
-}
-
-func (m *SyncBlockstore) DeleteMany(ks []cid.Cid) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return m.bs.DeleteMany(ks)
-}
-
-func (m *SyncBlockstore) Has(k cid.Cid) (bool, error) {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	return m.bs.Has(k)
-}
-
-func (m *SyncBlockstore) View(k cid.Cid, callback func([]byte) error) error {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-
-	return m.bs.View(k, callback)
-}
-
-func (m *SyncBlockstore) Get(k cid.Cid) (blocks.Block, error) {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	return m.bs.Get(k)
-}
-
-func (m *SyncBlockstore) GetSize(k cid.Cid) (int, error) {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	return m.bs.GetSize(k)
-}
-
-func (m *SyncBlockstore) Put(b blocks.Block) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return m.bs.Put(b)
-}
-
-func (m *SyncBlockstore) PutMany(bs []blocks.Block) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return m.bs.PutMany(bs)
-}
-
-func (m *SyncBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	// this blockstore implementation doesn't do any async work.
-	return m.bs.AllKeysChan(ctx)
-}
-
-func (m *SyncBlockstore) HashOnRead(enabled bool) {
-	// noop
-}