New interop network info

.circleci/config.yml

@@ -1,17 +1,15 @@
 version: 2.1
 orbs:
   go: gotest/tools@0.0.13
-  aws-cli: circleci/aws-cli@1.3.2
-  packer: salaxander/packer@0.0.3
 
 executors:
   golang:
     docker:
-      - image: circleci/golang:1.16.4
+      - image: circleci/golang:1.14.2
     resource_class: 2xlarge
   ubuntu:
     docker:
-      - image: ubuntu:20.04
+      - image: ubuntu:19.10
 
 commands:
   install-deps:
@@ -36,7 +34,7 @@ commands:
           condition: << parameters.linux >>
           steps:
             - run: sudo apt-get update
-            - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
+            - run: sudo apt-get install ocl-icd-opencl-dev
       - run: git submodule sync
       - run: git submodule update --init
   download-params:
@@ -47,7 +45,7 @@ commands:
             - 'v25-2k-lotus-params'
           paths:
             - /var/tmp/filecoin-proof-parameters/
-      - run:  ./lotus fetch-params 2048
+      - run:  ./lotus fetch-params --proving-params 2048
       - save_cache:
           name: Save parameters cache
           key: 'v25-2k-lotus-params'
@@ -95,10 +93,10 @@ jobs:
       - store_artifacts:
           path: lotus
       - store_artifacts:
-          path: lotus-miner
+          path: lotus-storage-miner
       - store_artifacts:
-          path: lotus-worker
+          path: lotus-seal-worker
-      - run: mkdir linux && mv lotus lotus-miner lotus-worker linux/
+      - run: mkdir linux && mv lotus lotus-storage-miner lotus-seal-worker linux/
       - persist_to_workspace:
           root: "."
           paths:
@@ -112,10 +110,10 @@ jobs:
       - run:
           command: make debug
 
-  test:
+  test: &test
     description: |
       Run tests with gotestsum.
-    parameters: &test-params
+    parameters:
       executor:
         type: executor
         default: golang
@@ -123,20 +121,20 @@ jobs:
         type: string
         default: "-timeout 30m"
         description: Flags passed to go test.
-      target:
+      packages:
         type: string
         default: "./..."
         description: Import paths of packages to be tested.
-      proofs-log-test:
+      winpost-test:
         type: string
         default: "0"
-      suite:
+      test-suite-name:
         type: string
         default: unit
         description: Test suite name to report to CircleCI.
       gotestsum-format:
         type: string
-        default: standard-verbose
+        default: short
         description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
       coverage:
         type: string
@@ -144,7 +142,7 @@ jobs:
         description: Coverage flag. Set to the empty string to disable.
       codecov-upload:
         type: boolean
-        default: true
+        default: false
         description: |
           Upload coverage report to https://codecov.io/. Requires the codecov API token to be
           set as an environment variable for private projects.
@@ -158,28 +156,21 @@ jobs:
       - download-params
       - go/install-gotestsum:
           gobin: $HOME/.local/bin
-          version: 0.5.2
       - run:
           name: go test
           environment:
-            TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >>
+            GOTESTSUM_JUNITFILE: /tmp/test-reports/<< parameters.test-suite-name >>/junit.xml
-            SKIP_CONFORMANCE: "1"
+            GOTESTSUM_FORMAT: << parameters.gotestsum-format >>
+            LOTUS_TEST_WINDOW_POST: << parameters.winpost-test >>
           command: |
-            mkdir -p /tmp/test-reports/<< parameters.suite >>
+            mkdir -p /tmp/test-reports/<< parameters.test-suite-name >>
-            mkdir -p /tmp/test-artifacts
+            gotestsum -- \
-            gotestsum \
-              --format << parameters.gotestsum-format >> \
-              --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
-              --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \
-              -- \
               << parameters.coverage >> \
               << parameters.go-test-flags >> \
-              << parameters.target >>
+              << parameters.packages >>
           no_output_timeout: 30m
       - store_test_results:
           path: /tmp/test-reports
-      - store_artifacts:
-          path: /tmp/test-artifacts/<< parameters.suite >>.json
       - when:
           condition: << parameters.codecov-upload >>
           steps:
@@ -190,160 +181,10 @@ jobs:
                 command: |
                   bash <(curl -s https://codecov.io/bash)
 
-  test-conformance:
+  test-short:
-    description: |
+    <<: *test
-      Run tests using a corpus of interoperable test vectors for Filecoin 
+  test-window-post:
-      implementations to test their correctness and compliance with the Filecoin
+    <<: *test
-      specifications.
-    parameters:
-      <<: *test-params
-      vectors-branch:
-        type: string
-        default: ""
-        description: |
-          Branch on github.com/filecoin-project/test-vectors to checkout and
-          test with. If empty (the default) the commit defined by the git
-          submodule is used.
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          command: make deps lotus
-          no_output_timeout: 30m
-      - download-params
-      - when:
-          condition:
-            not:
-              equal: [ "", << parameters.vectors-branch >> ]
-          steps:
-            - run:
-                name: checkout vectors branch
-                command: |
-                  cd extern/test-vectors
-                  git fetch
-                  git checkout origin/<< parameters.vectors-branch >>
-      - go/install-gotestsum:
-          gobin: $HOME/.local/bin
-          version: 0.5.2
-      - run:
-          name: install statediff globally
-          command: |
-            ## statediff is optional; we succeed even if compilation fails.
-            mkdir -p /tmp/statediff
-            git clone https://github.com/filecoin-project/statediff.git /tmp/statediff
-            cd /tmp/statediff
-            go install ./cmd/statediff || exit 0
-      - run:
-          name: go test
-          environment:
-            SKIP_CONFORMANCE: "0"
-          command: |
-            mkdir -p /tmp/test-reports
-            mkdir -p /tmp/test-artifacts
-            gotestsum \
-              --format pkgname-and-test-fails \
-              --junitfile /tmp/test-reports/junit.xml \
-              -- \
-              -v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/
-            go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html
-          no_output_timeout: 30m
-      - store_test_results:
-          path: /tmp/test-reports
-      - store_artifacts:
-          path: /tmp/test-artifacts/conformance-coverage.html
-  build-ntwk-calibration:
-    description: |
-      Compile lotus binaries for the calibration network
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: make calibnet
-      - run: mkdir linux-calibrationnet && mv lotus lotus-miner lotus-worker linux-calibrationnet
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux-calibrationnet
-  build-ntwk-butterfly:
-    description: |
-      Compile lotus binaries for the butterfly network
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: make butterflynet
-      - run: mkdir linux-butterflynet && mv lotus lotus-miner lotus-worker linux-butterflynet
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux-butterflynet
-  build-ntwk-nerpa:
-    description: |
-      Compile lotus binaries for the nerpa network
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: make nerpanet
-      - run: mkdir linux-nerpanet && mv lotus lotus-miner lotus-worker linux-nerpanet
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux-nerpanet
-  build-lotus-soup:
-    description: |
-      Compile `lotus-soup` Testground test plan
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: cd extern/filecoin-ffi && make
-      - run:
-          name: "go get lotus@master"
-          command: cd testplans/lotus-soup && go mod edit -replace=github.com/filecoin-project/lotus=../.. && go mod tidy
-      - run:
-          name: "build lotus-soup testplan"
-          command: pushd testplans/lotus-soup && go build -tags=testground .
-  trigger-testplans:
-    description: |
-      Trigger `lotus-soup` test cases on TaaS
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          name: "download testground"
-          command: wget https://gist.github.com/nonsense/5fbf3167cac79945f658771aed32fc44/raw/2e17eb0debf7ec6bdf027c1bdafc2c92dd97273b/testground-d3e9603 -O ~/testground-cli && chmod +x ~/testground-cli
-      - run:
-          name: "prepare .env.toml"
-          command: pushd testplans/lotus-soup && mkdir -p $HOME/testground && cp env-ci.toml $HOME/testground/.env.toml && echo 'endpoint="https://ci.testground.ipfs.team"' >> $HOME/testground/.env.toml && echo 'user="circleci"' >> $HOME/testground/.env.toml
-      - run:
-          name: "prepare testground home dir and link test plans"
-          command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup && ln -s $(pwd)/testplans/graphsync $HOME/testground/plans/graphsync
-      - run:
-          name: "go get lotus@master"
-          command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master
-      - run:
-          name: "trigger deals baseline testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/baseline-k8s-3-1.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-      - run:
-          name: "trigger payment channel stress testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-      - run:
-          name: "trigger graphsync testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/graphsync/_compositions/stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-
 
   build-macos:
     description: build darwin lotus binary
@@ -357,8 +198,8 @@ jobs:
       - run:
           name: Install go
           command: |
-            curl -O https://dl.google.com/go/go1.16.4.darwin-amd64.pkg && \
+            curl -O https://dl.google.com/go/go1.14.2.darwin-amd64.pkg && \
-            sudo installer -pkg go1.16.4.darwin-amd64.pkg -target /
+            sudo installer -pkg go1.14.2.darwin-amd64.pkg -target /
       - run:
           name: Install pkg-config
           command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
@@ -372,15 +213,6 @@ jobs:
           command: |
             curl --location https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 --output /usr/local/bin/jq
             chmod +x /usr/local/bin/jq
-      - run:
-          name: Install hwloc
-          command: |
-            mkdir ~/hwloc
-            curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-            cd ~/hwloc
-            tar -xvzpf hwloc-2.4.1.tar.gz
-            cd hwloc-2.4.1
-            ./configure && make && sudo make install
       - restore_cache:
           name: restore cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
@@ -391,10 +223,10 @@ jobs:
       - store_artifacts:
           path: lotus
       - store_artifacts:
-          path: lotus-miner
+          path: lotus-storage-miner
       - store_artifacts:
-          path: lotus-worker
+          path: lotus-seal-worker
-      - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
+      - run: mkdir darwin && mv lotus lotus-storage-miner lotus-seal-worker darwin/
       - persist_to_workspace:
           root: "."
           paths:
@@ -406,41 +238,6 @@ jobs:
             - "~/.rustup"
             - "~/.cargo"
 
-  build-appimage:
-    machine:
-      image: ubuntu-2004:202104-01
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - run:
-          name: install appimage-builder
-          command: |
-            # docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html
-            sudo apt update
-            sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace
-            sudo curl -Lo /usr/local/bin/appimagetool https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage
-            sudo chmod +x /usr/local/bin/appimagetool
-            sudo pip3 install appimage-builder
-      - run:
-          name: install lotus dependencies
-          command: sudo apt install ocl-icd-opencl-dev libhwloc-dev
-      - run:
-          name: build appimage
-          command: |
-            sed -i "s/version: latest/version: ${CIRCLE_TAG:-latest}/" AppImageBuilder.yml
-            make appimage
-      - run:
-          name: prepare workspace
-          command: |
-            mkdir appimage
-            mv Lotus-*.AppImage appimage
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - appimage
-
-
   gofmt:
     executor: golang
     steps:
@@ -449,41 +246,6 @@ jobs:
       - run:
           command: "! go fmt ./... 2>&1 | read"
 
-  gen-check:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run: make deps
-      - run: go install golang.org/x/tools/cmd/goimports
-      - run: go install github.com/hannahhoward/cbor-gen-for
-      - run: make gen
-      - run: git --no-pager diff
-      - run: git --no-pager diff --quiet
-      - run: make docsgen-cli
-      - run: git --no-pager diff
-      - run: git --no-pager diff --quiet
-
-  docs-check:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run: go install golang.org/x/tools/cmd/goimports
-      - run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full
-      - run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner
-      - run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker
-      - run: make deps
-      - run: make docsgen
-      - run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full
-      - run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner
-      - run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker
-      - run: git --no-pager diff
-      - run: diff ../pre-openrpc-full ../post-openrpc-full
-      - run: diff ../pre-openrpc-miner ../post-openrpc-miner
-      - run: diff ../pre-openrpc-worker ../post-openrpc-worker
-      - run: git --no-pager diff --quiet
-
   lint: &lint
     description: |
       Run golangci-lint.
@@ -520,6 +282,9 @@ jobs:
           command: |
             $HOME/.local/bin/golangci-lint run -v --timeout 2m \
               --concurrency << parameters.concurrency >> << parameters.args >>
+  lint-changes:
+    <<: *lint
+
   lint-all:
     <<: *lint
 
@@ -543,490 +308,52 @@ jobs:
           name: Publish release
           command: ./scripts/publish-release.sh
 
-  publish-snapcraft:
-    description: build and push snapcraft
-    machine:
-      image: ubuntu-2004:202104-01
-    resource_class: 2xlarge
-    parameters:
-      channel:
-        type: string
-        default: "edge"
-        description: snapcraft channel
-    steps:
-      - checkout
-      - run:
-          name: install snapcraft
-          command: sudo snap install snapcraft --classic
-      - run:
-          name: create snapcraft config file
-          command: |
-            mkdir -p ~/.config/snapcraft
-            echo "$SNAPCRAFT_LOGIN_FILE" | base64 -d > ~/.config/snapcraft/snapcraft.cfg
-      - run:
-          name: build snap
-          command: snapcraft --use-lxd
-      - run:
-          name: publish snap
-          command: snapcraft push *.snap --release << parameters.channel >>
-
-  build-and-push-image:
-    description: build and push docker images to public AWS ECR registry
-    executor: aws-cli/default
-    parameters:
-      profile-name:
-        type: string
-        default: "default"
-        description: AWS profile name to be configured.
-
-      aws-access-key-id:
-        type: env_var_name
-        default: AWS_ACCESS_KEY_ID
-        description: >
-          AWS access key id for IAM role. Set this to the name of
-          the environment variable you will set to hold this
-          value, i.e. AWS_ACCESS_KEY.
-
-      aws-secret-access-key:
-        type: env_var_name
-        default: AWS_SECRET_ACCESS_KEY
-        description: >
-          AWS secret key for IAM role. Set this to the name of
-          the environment variable you will set to hold this
-          value, i.e. AWS_SECRET_ACCESS_KEY.
-
-      region:
-        type: env_var_name
-        default: AWS_REGION
-        description: >
-          Name of env var storing your AWS region information,
-          defaults to AWS_REGION
-
-      account-url:
-        type: env_var_name
-        default: AWS_ECR_ACCOUNT_URL
-        description: >
-          Env var storing Amazon ECR account URL that maps to an AWS account,
-          e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com
-          defaults to AWS_ECR_ACCOUNT_URL
-
-      dockerfile:
-        type: string
-        default: Dockerfile
-        description: Name of dockerfile to use. Defaults to Dockerfile.
-
-      path:
-        type: string
-        default: .
-        description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory).
-
-      extra-build-args:
-        type: string
-        default: ""
-        description: >
-          Extra flags to pass to docker build. For examples, see
-          https://docs.docker.com/engine/reference/commandline/build
-
-      repo:
-        type: string
-        description: Name of an Amazon ECR repository
-
-      tag:
-        type: string
-        default: "latest"
-        description: A comma-separated string containing docker image tags to build and push (default = latest)
-
-    steps:
-      - run:
-          name: Confirm that environment variables are set
-          command: |
-            if [ -z "$AWS_ACCESS_KEY_ID" ]; then
-              echo "No AWS_ACCESS_KEY_ID is set. Skipping build-and-push job ..."
-              circleci-agent step halt
-            fi
-
-      - aws-cli/setup:
-          profile-name: <<parameters.profile-name>>
-          aws-access-key-id: <<parameters.aws-access-key-id>>
-          aws-secret-access-key: <<parameters.aws-secret-access-key>>
-          aws-region: <<parameters.region>>
-
-      - run:
-          name: Log into Amazon ECR
-          command: |
-            aws ecr-public get-login-password --region $<<parameters.region>> --profile <<parameters.profile-name>> | docker login --username AWS --password-stdin $<<parameters.account-url>>
-
-      - checkout
-
-      - setup_remote_docker:
-          version: 19.03.13
-          docker_layer_caching: false
-
-      - run:
-          name: Build docker image
-          command: |
-            registry_id=$(echo $<<parameters.account-url>> | sed "s;\..*;;g")
-
-            docker_tag_args=""
-            IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
-            for tag in "${DOCKER_TAGS[@]}"; do
-              docker_tag_args="$docker_tag_args -t $<<parameters.account-url>>/<<parameters.repo>>:$tag"
-            done
-
-            docker build \
-              <<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
-              -f <<parameters.path>>/<<parameters.dockerfile>> \
-              $docker_tag_args \
-              <<parameters.path>>
-
-      - run:
-          name: Push image to Amazon ECR
-          command: |
-            IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
-            for tag in "${DOCKER_TAGS[@]}"; do
-              docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
-            done
-
-  publish-packer-mainnet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
-  publish-packer-calibrationnet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
-  publish-packer-butterflynet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG"
-  publish-packer-nerpanet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG"
 
 workflows:
   version: 2.1
   ci:
     jobs:
-      - lint-all:
+      - lint-changes:
-          concurrency: "16"   # expend all docker 2xlarge CPUs.
+          args: "--new-from-rev origin/next" 
       - mod-tidy-check
       - gofmt
-      - gen-check
-      - docs-check
       - test:
-          name: test-itest-api
+          codecov-upload: true
-          suite: itest-api
+      - test-window-post:
-          target: "./itests/api_test.go"
+          go-test-flags: "-run=TestWindowedPost"
-      
+          winpost-test: "1"
-      - test:
+      - test-short:
-          name: test-itest-batch_deal
+          go-test-flags: "--timeout 10m --short"
-          suite: itest-batch_deal
-          target: "./itests/batch_deal_test.go"
-      
-      - test:
-          name: test-itest-ccupgrade
-          suite: itest-ccupgrade
-          target: "./itests/ccupgrade_test.go"
-      
-      - test:
-          name: test-itest-cli
-          suite: itest-cli
-          target: "./itests/cli_test.go"
-      
-      - test:
-          name: test-itest-deadlines
-          suite: itest-deadlines
-          target: "./itests/deadlines_test.go"
-      
-      - test:
-          name: test-itest-deals_concurrent
-          suite: itest-deals_concurrent
-          target: "./itests/deals_concurrent_test.go"
-      
-      - test:
-          name: test-itest-deals_offline
-          suite: itest-deals_offline
-          target: "./itests/deals_offline_test.go"
-      
-      - test:
-          name: test-itest-deals_power
-          suite: itest-deals_power
-          target: "./itests/deals_power_test.go"
-      
-      - test:
-          name: test-itest-deals_pricing
-          suite: itest-deals_pricing
-          target: "./itests/deals_pricing_test.go"
-      
-      - test:
-          name: test-itest-deals_publish
-          suite: itest-deals_publish
-          target: "./itests/deals_publish_test.go"
-      
-      - test:
-          name: test-itest-deals
-          suite: itest-deals
-          target: "./itests/deals_test.go"
-      
-      - test:
-          name: test-itest-gateway
-          suite: itest-gateway
-          target: "./itests/gateway_test.go"
-      
-      - test:
-          name: test-itest-get_messages_in_ts
-          suite: itest-get_messages_in_ts
-          target: "./itests/get_messages_in_ts_test.go"
-      
-      - test:
-          name: test-itest-multisig
-          suite: itest-multisig
-          target: "./itests/multisig_test.go"
-      
-      - test:
-          name: test-itest-nonce
-          suite: itest-nonce
-          target: "./itests/nonce_test.go"
-      
-      - test:
-          name: test-itest-paych_api
-          suite: itest-paych_api
-          target: "./itests/paych_api_test.go"
-      
-      - test:
-          name: test-itest-paych_cli
-          suite: itest-paych_cli
-          target: "./itests/paych_cli_test.go"
-      
-      - test:
-          name: test-itest-sdr_upgrade
-          suite: itest-sdr_upgrade
-          target: "./itests/sdr_upgrade_test.go"
-      
-      - test:
-          name: test-itest-sector_finalize_early
-          suite: itest-sector_finalize_early
-          target: "./itests/sector_finalize_early_test.go"
-      
-      - test:
-          name: test-itest-sector_miner_collateral
-          suite: itest-sector_miner_collateral
-          target: "./itests/sector_miner_collateral_test.go"
-      
-      - test:
-          name: test-itest-sector_pledge
-          suite: itest-sector_pledge
-          target: "./itests/sector_pledge_test.go"
-      
-      - test:
-          name: test-itest-sector_terminate
-          suite: itest-sector_terminate
-          target: "./itests/sector_terminate_test.go"
-      
-      - test:
-          name: test-itest-tape
-          suite: itest-tape
-          target: "./itests/tape_test.go"
-      
-      - test:
-          name: test-itest-verifreg
-          suite: itest-verifreg
-          target: "./itests/verifreg_test.go"
-      
-      - test:
-          name: test-itest-wdpost_dispute
-          suite: itest-wdpost_dispute
-          target: "./itests/wdpost_dispute_test.go"
-      
-      - test:
-          name: test-itest-wdpost
-          suite: itest-wdpost
-          target: "./itests/wdpost_test.go"
-      
-      - test:
-          name: test-unit-cli
-          suite: utest-unit-cli
-          target: "./cli/... ./cmd/... ./api/..."
-      - test:
-          name: test-unit-node
-          suite: utest-unit-node
-          target: "./node/..."
-      - test:
-          name: test-unit-rest
-          suite: utest-unit-rest
-          target: "./api/... ./blockstore/... ./build/... ./chain/... ./cli/... ./cmd/... ./conformance/... ./extern/... ./gateway/... ./journal/... ./lib/... ./markets/... ./node/... ./paychmgr/... ./storage/... ./tools/..."
-      - test:
-          name: test-unit-storage
-          suite: utest-unit-storage
-          target: "./storage/... ./extern/..."
-      - test:
-          go-test-flags: "-run=TestMulticoreSDR"
-          suite: multicore-sdr-check
-          target: "./extern/sector-storage/ffiwrapper"
-          proofs-log-test: "1"
-      - test-conformance:
-          suite: conformance
-          codecov-upload: false
-          target: "./conformance"
-      - test-conformance:
-          name: test-conformance-bleeding-edge
-          codecov-upload: false
-          suite: conformance-bleeding-edge
-          target: "./conformance"
-          vectors-branch: master
-      - trigger-testplans:
           filters:
-            branches:
+            tags:
               only:
-                - master
+                - /^v\d+\.\d+\.\d+$/
       - build-debug
       - build-all:
+          requires:
+            - test-short
           filters:
             tags:
               only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+                - /^v\d+\.\d+\.\d+$/
-      - build-ntwk-calibration:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-ntwk-butterfly:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-ntwk-nerpa:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-lotus-soup
       - build-macos:
+          requires:
+            - test-short
           filters:
             branches:
               ignore:
                 - /.*/
             tags:
               only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+                - /^v\d+\.\d+\.\d+$/
-      - build-appimage:
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - publish:
           requires:
             - build-all
             - build-macos
-            - build-appimage
           filters:
             branches:
               ignore:
                 - /.*/
             tags:
               only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+                - /^v\d+\.\d+\.\d+$/
-      - build-and-push-image:
-          dockerfile: Dockerfile.lotus
-          path: .
-          repo: lotus-dev
-          tag: '${CIRCLE_SHA1:0:8}'
-      - publish-packer-mainnet:
-          requires:
-            - build-all
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-calibrationnet:
-          requires:
-            - build-ntwk-calibration
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-butterflynet:
-          requires:
-            - build-ntwk-butterfly
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-nerpanet:
-          requires:
-            - build-ntwk-nerpa
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-snapcraft:
-          name: publish-snapcraft-stable
-          channel: stable
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-
-  nightly:
-    triggers:
-      - schedule:
-          cron: "0 0 * * *"
-          filters:
-            branches:
-              only:
-                - master
-    jobs:
-      - publish-snapcraft:
-          name: publish-snapcraft-nightly
-          channel: edge

.circleci/gen.go

@@ -1,136 +0,0 @@
-package main
-
-import (
-	"embed"
-	"fmt"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-	"text/template"
-)
-
-//go:generate go run ./gen.go ..
-
-//go:embed template.yml
-var templateFile embed.FS
-
-type (
-	dirs  = []string
-	suite = string
-)
-
-// groupedUnitTests maps suite names to top-level directories that should be
-// included in that suite. The program adds an implicit group "rest" that
-// includes all other top-level directories.
-var groupedUnitTests = map[suite]dirs{
-	"unit-node":    {"node"},
-	"unit-storage": {"storage", "extern"},
-	"unit-cli":     {"cli", "cmd", "api"},
-}
-
-func main() {
-	if len(os.Args) != 2 {
-		panic("expected path to repo as argument")
-	}
-
-	repo := os.Args[1]
-
-	tmpl := template.New("template.yml")
-	tmpl.Delims("[[", "]]")
-	tmpl.Funcs(template.FuncMap{
-		"stripSuffix": func(in string) string {
-			return strings.TrimSuffix(in, "_test.go")
-		},
-	})
-	tmpl = template.Must(tmpl.ParseFS(templateFile, "*"))
-
-	// list all itests.
-	itests, err := filepath.Glob(filepath.Join(repo, "./itests/*_test.go"))
-	if err != nil {
-		panic(err)
-	}
-
-	// strip the dir from all entries.
-	for i, f := range itests {
-		itests[i] = filepath.Base(f)
-	}
-
-	// calculate the exclusion set of unit test directories to exclude because
-	// they are already included in a grouped suite.
-	var excluded = map[string]struct{}{}
-	for _, ss := range groupedUnitTests {
-		for _, s := range ss {
-			e, err := filepath.Abs(filepath.Join(repo, s))
-			if err != nil {
-				panic(err)
-			}
-			excluded[e] = struct{}{}
-		}
-	}
-
-	// all unit tests top-level dirs that are not itests, nor included in other suites.
-	var rest = map[string]struct{}{}
-	err = filepath.Walk(repo, func(path string, f os.FileInfo, err error) error {
-		// include all tests that aren't in the itests directory.
-		if strings.Contains(path, "itests") {
-			return filepath.SkipDir
-		}
-		// exclude all tests included in other suites
-		if f.IsDir() {
-			if _, ok := excluded[path]; ok {
-				return filepath.SkipDir
-			}
-		}
-		if strings.HasSuffix(path, "_test.go") {
-			rel, err := filepath.Rel(repo, path)
-			if err != nil {
-				panic(err)
-			}
-			// take the first directory
-			rest[strings.Split(rel, string(os.PathSeparator))[0]] = struct{}{}
-		}
-		return err
-	})
-	if err != nil {
-		panic(err)
-	}
-
-	// add other directories to a 'rest' suite.
-	for k := range rest {
-		groupedUnitTests["unit-rest"] = append(groupedUnitTests["unit-rest"], k)
-	}
-
-	// map iteration guarantees no order, so sort the array in-place.
-	sort.Strings(groupedUnitTests["unit-rest"])
-
-	// form the input data.
-	type data struct {
-		ItestFiles []string
-		UnitSuites map[string]string
-	}
-	in := data{
-		ItestFiles: itests,
-		UnitSuites: func() map[string]string {
-			ret := make(map[string]string)
-			for name, dirs := range groupedUnitTests {
-				for i, d := range dirs {
-					dirs[i] = fmt.Sprintf("./%s/...", d) // turn into package
-				}
-				ret[name] = strings.Join(dirs, " ")
-			}
-			return ret
-		}(),
-	}
-
-	out, err := os.Create("./config.yml")
-	if err != nil {
-		panic(err)
-	}
-	defer out.Close()
-
-	// execute the template.
-	if err := tmpl.Execute(out, in); err != nil {
-		panic(err)
-	}
-}

.circleci/template.yml

@@ -1,902 +0,0 @@
-version: 2.1
-orbs:
-  go: gotest/tools@0.0.13
-  aws-cli: circleci/aws-cli@1.3.2
-  packer: salaxander/packer@0.0.3
-
-executors:
-  golang:
-    docker:
-      - image: circleci/golang:1.16.4
-    resource_class: 2xlarge
-  ubuntu:
-    docker:
-      - image: ubuntu:20.04
-
-commands:
-  install-deps:
-    steps:
-      - go/install-ssh
-      - go/install: {package: git}
-  prepare:
-    parameters:
-      linux:
-        default: true
-        description: is a linux build environment?
-        type: boolean
-      darwin:
-        default: false
-        description: is a darwin build environment?
-        type: boolean
-    steps:
-      - checkout
-      - git_fetch_all_tags
-      - checkout
-      - when:
-          condition: << parameters.linux >>
-          steps:
-            - run: sudo apt-get update
-            - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
-      - run: git submodule sync
-      - run: git submodule update --init
-  download-params:
-    steps:
-      - restore_cache:
-          name: Restore parameters cache
-          keys:
-            - 'v25-2k-lotus-params'
-          paths:
-            - /var/tmp/filecoin-proof-parameters/
-      - run:  ./lotus fetch-params 2048
-      - save_cache:
-          name: Save parameters cache
-          key: 'v25-2k-lotus-params'
-          paths:
-            - /var/tmp/filecoin-proof-parameters/
-  install_ipfs:
-    steps:
-      - run: |
-          apt update
-          apt install -y wget
-          wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz
-          wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512
-          if [ "$(sha512sum go-ipfs_v0.4.22_linux-amd64.tar.gz)" != "$(cat go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512)" ]
-          then
-            echo "ipfs failed checksum check"
-            exit 1
-          fi
-          tar -xf go-ipfs_v0.4.22_linux-amd64.tar.gz
-          mv go-ipfs/ipfs /usr/local/bin/ipfs
-          chmod +x /usr/local/bin/ipfs
-  git_fetch_all_tags:
-    steps:
-      - run:
-          name: fetch all tags
-          command: |
-            git fetch --all
-
-jobs:
-  mod-tidy-check:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - go/mod-tidy-check
-
-  build-all:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run: sudo apt-get update
-      - run: sudo apt-get install npm
-      - run:
-          command: make buildall
-      - store_artifacts:
-          path: lotus
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir linux && mv lotus lotus-miner lotus-worker linux/
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux
-
-  build-debug:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          command: make debug
-
-  test:
-    description: |
-      Run tests with gotestsum.
-    parameters: &test-params
-      executor:
-        type: executor
-        default: golang
-      go-test-flags:
-        type: string
-        default: "-timeout 30m"
-        description: Flags passed to go test.
-      target:
-        type: string
-        default: "./..."
-        description: Import paths of packages to be tested.
-      proofs-log-test:
-        type: string
-        default: "0"
-      suite:
-        type: string
-        default: unit
-        description: Test suite name to report to CircleCI.
-      gotestsum-format:
-        type: string
-        default: standard-verbose
-        description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
-      coverage:
-        type: string
-        default: -coverprofile=coverage.txt -coverpkg=github.com/filecoin-project/lotus/...
-        description: Coverage flag. Set to the empty string to disable.
-      codecov-upload:
-        type: boolean
-        default: true
-        description: |
-          Upload coverage report to https://codecov.io/. Requires the codecov API token to be
-          set as an environment variable for private projects.
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          command: make deps lotus
-          no_output_timeout: 30m
-      - download-params
-      - go/install-gotestsum:
-          gobin: $HOME/.local/bin
-          version: 0.5.2
-      - run:
-          name: go test
-          environment:
-            TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >>
-            SKIP_CONFORMANCE: "1"
-          command: |
-            mkdir -p /tmp/test-reports/<< parameters.suite >>
-            mkdir -p /tmp/test-artifacts
-            gotestsum \
-              --format << parameters.gotestsum-format >> \
-              --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
-              --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \
-              -- \
-              << parameters.coverage >> \
-              << parameters.go-test-flags >> \
-              << parameters.target >>
-          no_output_timeout: 30m
-      - store_test_results:
-          path: /tmp/test-reports
-      - store_artifacts:
-          path: /tmp/test-artifacts/<< parameters.suite >>.json
-      - when:
-          condition: << parameters.codecov-upload >>
-          steps:
-            - go/install: {package: bash}
-            - go/install: {package: curl}
-            - run:
-                shell: /bin/bash -eo pipefail
-                command: |
-                  bash <(curl -s https://codecov.io/bash)
-
-  test-conformance:
-    description: |
-      Run tests using a corpus of interoperable test vectors for Filecoin 
-      implementations to test their correctness and compliance with the Filecoin
-      specifications.
-    parameters:
-      <<: *test-params
-      vectors-branch:
-        type: string
-        default: ""
-        description: |
-          Branch on github.com/filecoin-project/test-vectors to checkout and
-          test with. If empty (the default) the commit defined by the git
-          submodule is used.
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          command: make deps lotus
-          no_output_timeout: 30m
-      - download-params
-      - when:
-          condition:
-            not:
-              equal: [ "", << parameters.vectors-branch >> ]
-          steps:
-            - run:
-                name: checkout vectors branch
-                command: |
-                  cd extern/test-vectors
-                  git fetch
-                  git checkout origin/<< parameters.vectors-branch >>
-      - go/install-gotestsum:
-          gobin: $HOME/.local/bin
-          version: 0.5.2
-      - run:
-          name: install statediff globally
-          command: |
-            ## statediff is optional; we succeed even if compilation fails.
-            mkdir -p /tmp/statediff
-            git clone https://github.com/filecoin-project/statediff.git /tmp/statediff
-            cd /tmp/statediff
-            go install ./cmd/statediff || exit 0
-      - run:
-          name: go test
-          environment:
-            SKIP_CONFORMANCE: "0"
-          command: |
-            mkdir -p /tmp/test-reports
-            mkdir -p /tmp/test-artifacts
-            gotestsum \
-              --format pkgname-and-test-fails \
-              --junitfile /tmp/test-reports/junit.xml \
-              -- \
-              -v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/
-            go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html
-          no_output_timeout: 30m
-      - store_test_results:
-          path: /tmp/test-reports
-      - store_artifacts:
-          path: /tmp/test-artifacts/conformance-coverage.html
-  build-ntwk-calibration:
-    description: |
-      Compile lotus binaries for the calibration network
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: make calibnet
-      - run: mkdir linux-calibrationnet && mv lotus lotus-miner lotus-worker linux-calibrationnet
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux-calibrationnet
-  build-ntwk-butterfly:
-    description: |
-      Compile lotus binaries for the butterfly network
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: make butterflynet
-      - run: mkdir linux-butterflynet && mv lotus lotus-miner lotus-worker linux-butterflynet
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux-butterflynet
-  build-ntwk-nerpa:
-    description: |
-      Compile lotus binaries for the nerpa network
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: make nerpanet
-      - run: mkdir linux-nerpanet && mv lotus lotus-miner lotus-worker linux-nerpanet
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux-nerpanet
-  build-lotus-soup:
-    description: |
-      Compile `lotus-soup` Testground test plan
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: cd extern/filecoin-ffi && make
-      - run:
-          name: "go get lotus@master"
-          command: cd testplans/lotus-soup && go mod edit -replace=github.com/filecoin-project/lotus=../.. && go mod tidy
-      - run:
-          name: "build lotus-soup testplan"
-          command: pushd testplans/lotus-soup && go build -tags=testground .
-  trigger-testplans:
-    description: |
-      Trigger `lotus-soup` test cases on TaaS
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          name: "download testground"
-          command: wget https://gist.github.com/nonsense/5fbf3167cac79945f658771aed32fc44/raw/2e17eb0debf7ec6bdf027c1bdafc2c92dd97273b/testground-d3e9603 -O ~/testground-cli && chmod +x ~/testground-cli
-      - run:
-          name: "prepare .env.toml"
-          command: pushd testplans/lotus-soup && mkdir -p $HOME/testground && cp env-ci.toml $HOME/testground/.env.toml && echo 'endpoint="https://ci.testground.ipfs.team"' >> $HOME/testground/.env.toml && echo 'user="circleci"' >> $HOME/testground/.env.toml
-      - run:
-          name: "prepare testground home dir and link test plans"
-          command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup && ln -s $(pwd)/testplans/graphsync $HOME/testground/plans/graphsync
-      - run:
-          name: "go get lotus@master"
-          command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master
-      - run:
-          name: "trigger deals baseline testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/baseline-k8s-3-1.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-      - run:
-          name: "trigger payment channel stress testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-      - run:
-          name: "trigger graphsync testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/graphsync/_compositions/stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-
-
-  build-macos:
-    description: build darwin lotus binary
-    macos:
-      xcode: "10.0.0"
-    working_directory: ~/go/src/github.com/filecoin-project/lotus
-    steps:
-      - prepare:
-          linux: false
-          darwin: true
-      - run:
-          name: Install go
-          command: |
-            curl -O https://dl.google.com/go/go1.16.4.darwin-amd64.pkg && \
-            sudo installer -pkg go1.16.4.darwin-amd64.pkg -target /
-      - run:
-          name: Install pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
-      - run: go version
-      - run:
-          name: Install Rust
-          command: |
-            curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - run:
-          name: Install jq
-          command: |
-            curl --location https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 --output /usr/local/bin/jq
-            chmod +x /usr/local/bin/jq
-      - run:
-          name: Install hwloc
-          command: |
-            mkdir ~/hwloc
-            curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-            cd ~/hwloc
-            tar -xvzpf hwloc-2.4.1.tar.gz
-            cd hwloc-2.4.1
-            ./configure && make && sudo make install
-      - restore_cache:
-          name: restore cargo cache
-          key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-      - install-deps
-      - run:
-          command: make build
-          no_output_timeout: 30m
-      - store_artifacts:
-          path: lotus
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - darwin
-      - save_cache:
-          name: save cargo cache
-          key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-          paths:
-            - "~/.rustup"
-            - "~/.cargo"
-
-  build-appimage:
-    machine:
-      image: ubuntu-2004:202104-01
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - run:
-          name: install appimage-builder
-          command: |
-            # docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html
-            sudo apt update
-            sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace
-            sudo curl -Lo /usr/local/bin/appimagetool https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage
-            sudo chmod +x /usr/local/bin/appimagetool
-            sudo pip3 install appimage-builder
-      - run:
-          name: install lotus dependencies
-          command: sudo apt install ocl-icd-opencl-dev libhwloc-dev
-      - run:
-          name: build appimage
-          command: |
-            sed -i "s/version: latest/version: ${CIRCLE_TAG:-latest}/" AppImageBuilder.yml
-            make appimage
-      - run:
-          name: prepare workspace
-          command: |
-            mkdir appimage
-            mv Lotus-*.AppImage appimage
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - appimage
-
-
-  gofmt:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          command: "! go fmt ./... 2>&1 | read"
-
-  gen-check:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run: make deps
-      - run: go install golang.org/x/tools/cmd/goimports
-      - run: go install github.com/hannahhoward/cbor-gen-for
-      - run: make gen
-      - run: git --no-pager diff
-      - run: git --no-pager diff --quiet
-      - run: make docsgen-cli
-      - run: git --no-pager diff
-      - run: git --no-pager diff --quiet
-
-  docs-check:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run: go install golang.org/x/tools/cmd/goimports
-      - run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full
-      - run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner
-      - run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker
-      - run: make deps
-      - run: make docsgen
-      - run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full
-      - run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner
-      - run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker
-      - run: git --no-pager diff
-      - run: diff ../pre-openrpc-full ../post-openrpc-full
-      - run: diff ../pre-openrpc-miner ../post-openrpc-miner
-      - run: diff ../pre-openrpc-worker ../post-openrpc-worker
-      - run: git --no-pager diff --quiet
-
-  lint: &lint
-    description: |
-      Run golangci-lint.
-    parameters:
-      executor:
-        type: executor
-        default: golang
-      golangci-lint-version:
-        type: string
-        default: 1.27.0
-      concurrency:
-        type: string
-        default: '2'
-        description: |
-          Concurrency used to run linters. Defaults to 2 because NumCPU is not
-          aware of container CPU limits.
-      args:
-        type: string
-        default: ''
-        description: |
-          Arguments to pass to golangci-lint
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          command: make deps
-          no_output_timeout: 30m
-      - go/install-golangci-lint:
-          gobin: $HOME/.local/bin
-          version: << parameters.golangci-lint-version >>
-      - run:
-          name: Lint
-          command: |
-            $HOME/.local/bin/golangci-lint run -v --timeout 2m \
-              --concurrency << parameters.concurrency >> << parameters.args >>
-  lint-all:
-    <<: *lint
-
-  publish:
-    description: publish binary artifacts
-    executor: ubuntu
-    steps:
-      - run:
-          name: Install git jq curl
-          command: apt update && apt install -y git jq curl
-      - checkout
-      - git_fetch_all_tags
-      - checkout
-      - install_ipfs
-      - attach_workspace:
-          at: "."
-      - run:
-          name: Create bundles
-          command: ./scripts/build-bundle.sh
-      - run:
-          name: Publish release
-          command: ./scripts/publish-release.sh
-
-  publish-snapcraft:
-    description: build and push snapcraft
-    machine:
-      image: ubuntu-2004:202104-01
-    resource_class: 2xlarge
-    parameters:
-      channel:
-        type: string
-        default: "edge"
-        description: snapcraft channel
-    steps:
-      - checkout
-      - run:
-          name: install snapcraft
-          command: sudo snap install snapcraft --classic
-      - run:
-          name: create snapcraft config file
-          command: |
-            mkdir -p ~/.config/snapcraft
-            echo "$SNAPCRAFT_LOGIN_FILE" | base64 -d > ~/.config/snapcraft/snapcraft.cfg
-      - run:
-          name: build snap
-          command: snapcraft --use-lxd
-      - run:
-          name: publish snap
-          command: snapcraft push *.snap --release << parameters.channel >>
-
-  build-and-push-image:
-    description: build and push docker images to public AWS ECR registry
-    executor: aws-cli/default
-    parameters:
-      profile-name:
-        type: string
-        default: "default"
-        description: AWS profile name to be configured.
-
-      aws-access-key-id:
-        type: env_var_name
-        default: AWS_ACCESS_KEY_ID
-        description: >
-          AWS access key id for IAM role. Set this to the name of
-          the environment variable you will set to hold this
-          value, i.e. AWS_ACCESS_KEY.
-
-      aws-secret-access-key:
-        type: env_var_name
-        default: AWS_SECRET_ACCESS_KEY
-        description: >
-          AWS secret key for IAM role. Set this to the name of
-          the environment variable you will set to hold this
-          value, i.e. AWS_SECRET_ACCESS_KEY.
-
-      region:
-        type: env_var_name
-        default: AWS_REGION
-        description: >
-          Name of env var storing your AWS region information,
-          defaults to AWS_REGION
-
-      account-url:
-        type: env_var_name
-        default: AWS_ECR_ACCOUNT_URL
-        description: >
-          Env var storing Amazon ECR account URL that maps to an AWS account,
-          e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com
-          defaults to AWS_ECR_ACCOUNT_URL
-
-      dockerfile:
-        type: string
-        default: Dockerfile
-        description: Name of dockerfile to use. Defaults to Dockerfile.
-
-      path:
-        type: string
-        default: .
-        description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory).
-
-      extra-build-args:
-        type: string
-        default: ""
-        description: >
-          Extra flags to pass to docker build. For examples, see
-          https://docs.docker.com/engine/reference/commandline/build
-
-      repo:
-        type: string
-        description: Name of an Amazon ECR repository
-
-      tag:
-        type: string
-        default: "latest"
-        description: A comma-separated string containing docker image tags to build and push (default = latest)
-
-    steps:
-      - run:
-          name: Confirm that environment variables are set
-          command: |
-            if [ -z "$AWS_ACCESS_KEY_ID" ]; then
-              echo "No AWS_ACCESS_KEY_ID is set. Skipping build-and-push job ..."
-              circleci-agent step halt
-            fi
-
-      - aws-cli/setup:
-          profile-name: <<parameters.profile-name>>
-          aws-access-key-id: <<parameters.aws-access-key-id>>
-          aws-secret-access-key: <<parameters.aws-secret-access-key>>
-          aws-region: <<parameters.region>>
-
-      - run:
-          name: Log into Amazon ECR
-          command: |
-            aws ecr-public get-login-password --region $<<parameters.region>> --profile <<parameters.profile-name>> | docker login --username AWS --password-stdin $<<parameters.account-url>>
-
-      - checkout
-
-      - setup_remote_docker:
-          version: 19.03.13
-          docker_layer_caching: false
-
-      - run:
-          name: Build docker image
-          command: |
-            registry_id=$(echo $<<parameters.account-url>> | sed "s;\..*;;g")
-
-            docker_tag_args=""
-            IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
-            for tag in "${DOCKER_TAGS[@]}"; do
-              docker_tag_args="$docker_tag_args -t $<<parameters.account-url>>/<<parameters.repo>>:$tag"
-            done
-
-            docker build \
-              <<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
-              -f <<parameters.path>>/<<parameters.dockerfile>> \
-              $docker_tag_args \
-              <<parameters.path>>
-
-      - run:
-          name: Push image to Amazon ECR
-          command: |
-            IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
-            for tag in "${DOCKER_TAGS[@]}"; do
-              docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
-            done
-
-  publish-packer-mainnet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
-  publish-packer-calibrationnet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
-  publish-packer-butterflynet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG"
-  publish-packer-nerpanet:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG"
-
-workflows:
-  version: 2.1
-  ci:
-    jobs:
-      - lint-all:
-          concurrency: "16"   # expend all docker 2xlarge CPUs.
-      - mod-tidy-check
-      - gofmt
-      - gen-check
-      - docs-check
-
-      [[- range $file := .ItestFiles -]]
-      [[ with $name := $file | stripSuffix ]]
-      - test:
-          name: test-itest-[[ $name ]]
-          suite: itest-[[ $name ]]
-          target: "./itests/[[ $file ]]"
-      [[ end ]]
-      [[- end -]]
-
-      [[range $suite, $pkgs := .UnitSuites]]
-      - test:
-          name: test-[[ $suite ]]
-          suite: utest-[[ $suite ]]
-          target: "[[ $pkgs ]]"
-      [[- end]]
-      - test:
-          go-test-flags: "-run=TestMulticoreSDR"
-          suite: multicore-sdr-check
-          target: "./extern/sector-storage/ffiwrapper"
-          proofs-log-test: "1"
-      - test-conformance:
-          suite: conformance
-          codecov-upload: false
-          target: "./conformance"
-      - test-conformance:
-          name: test-conformance-bleeding-edge
-          codecov-upload: false
-          suite: conformance-bleeding-edge
-          target: "./conformance"
-          vectors-branch: master
-      - trigger-testplans:
-          filters:
-            branches:
-              only:
-                - master
-      - build-debug
-      - build-all:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-ntwk-calibration:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-ntwk-butterfly:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-ntwk-nerpa:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-lotus-soup
-      - build-macos:
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-appimage:
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish:
-          requires:
-            - build-all
-            - build-macos
-            - build-appimage
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-and-push-image:
-          dockerfile: Dockerfile.lotus
-          path: .
-          repo: lotus-dev
-          tag: '${CIRCLE_SHA1:0:8}'
-      - publish-packer-mainnet:
-          requires:
-            - build-all
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-calibrationnet:
-          requires:
-            - build-ntwk-calibration
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-butterflynet:
-          requires:
-            - build-ntwk-butterfly
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-packer-nerpanet:
-          requires:
-            - build-ntwk-nerpa
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-snapcraft:
-          name: publish-snapcraft-stable
-          channel: stable
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-
-  nightly:
-    triggers:
-      - schedule:
-          cron: "0 0 * * *"
-          filters:
-            branches:
-              only:
-                - master
-    jobs:
-      - publish-snapcraft:
-          name: publish-snapcraft-nightly
-          channel: edge

.codecov.yml

@@ -1,19 +1,3 @@
 comment: off
 ignore:
-  - "**/cbor_gen.go"
+  - "cbor_gen.go"
-  - "api/test/**/*"
-  - "api/test/*"
-  - "gen/**/*"
-  - "gen/*"
-  - "cmd/lotus-shed/*"
-  - "cmd/tvx/*"
-  - "cmd/lotus-pcr/*"
-  - "cmd/tvx/*"
-  - "cmd/lotus-chainwatch/*"
-  - "cmd/lotus-health/*"
-  - "cmd/lotus-fountain/*"
-  - "cmd/lotus-townhall/*"
-  - "cmd/lotus-stats/*"
-  - "cmd/lotus-pcr/*"
-github_checks:
-  annotations: false

.github/CODEOWNERS

@@ -1,6 +0,0 @@
-# Reference
-# https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/creating-a-repository-on-github/about-code-owners
-
-# Global owners
-# Ensure maintainers team is a requested reviewer for non-draft PRs
-*                       @filecoin-project/lotus-maintainers

.github/ISSUE_TEMPLATE/bug-report.md

@@ -1,33 +0,0 @@
----
-name: Bug Report
-about: Create a report to help us improve
-title: "[BUG] "
-labels: hint/needs-triaging, kind/bug
-assignees: ''
-
----
-
-> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
-
-**Describe the bug**
-A clear and concise description of what the bug is. 
-(If you are not sure what the bug is, try to figure it out via a [discussion](https://github.com/filecoin-project/lotus/discussions/new) first!
-
-**Version (run `lotus version`):**
-
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Run '...'
-2. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Logs**
-Provide daemon/miner/worker logs, and goroutines(if available) for troubleshooting.
- 
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,27 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Run '...'
+2. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Version (run `lotus --version`):**
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/deal-making-issues.md

@@ -1,49 +0,0 @@
----
-name: Deal Making Issues
-about: Create a report for help with deal making failures.
-title: "[Deal Making Issue]"
-labels: hint/needs-triaging, area/markets
-assignees: ''
-
----
-
-> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
-
-Please provide all the information requested here to help us troubleshoot "deal making failed" issues.
-If the information requested is missing, we will probably have to just ask you to provide it anyway,
-before we can help debug.
-
-**Basic Information**
-Including information like, Are you the client or the miner? Is this a storage deal or a retrieval deal? Is it an offline deal?
-
-**Describe the problem**
-
-A brief description of the problem you encountered while trying to make a deal. 
-
-**Version**
-
-The output of `lotus --version`.
-
-**Setup**
-
-You miner(if applicable) and daemon setup, i.e: What hardware do you use, how much ram and etc.
-
-**To Reproduce**
- Steps to reproduce the behavior:
- 1. Run '...'
- 2. See error
-
-**Deal status**
-
-The output of `lotus client list-deals -v` and/or `lotus-miner storage-deals|retrieval-deals|data-transfers list [-v]` commands for the deal(s) in question.
-
-**Lotus daemon and miner logs**
-
-Please go through the logs of your daemon and miner(if applicable), and include screenshots of any error/warning-like messages you find.
-
-Alternatively please upload full log files and share a link here
-
-** Code modifications **
-
-If you have modified parts of lotus, please describe which areas were modified,
-and the scope of those modifications

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,20 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-title: "[Feature Request]"
-labels: hint/needs-triaging
-assignees: ''
-
----
-
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/mining-issues.md

@@ -1,35 +0,0 @@
----
-name: Mining Issues
-about: Create a report for help with mining failures.
-title: "[Mining Issue]"
-labels: hint/needs-triaging, area/mining
-assignees: ''
-
----
-
-> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
-
-Please provide all the information requested here to help us troubleshoot "mining/WinningPoSt failed" issues.
-If the information requested is missing, you may be asked you to provide it.
-
-**Describe the problem**
-A brief description of the problem you encountered while mining new blocks.
-
-**Version**
-
-The output of `lotus --version`.
-
-**Setup**
-
-You miner and daemon setup, including what hardware do you use, your environment variable settings, how do you run your miner and worker, do you use GPU and etc.
-
-**Lotus daemon and miner logs**
-
-Please go through the logs of your daemon and miner, and include screenshots of any error/warning-like messages you find, highlighting the one has "winning post" in it.
-
-Alternatively please upload full log files and share a link here
-
-** Code modifications **
-
-If you have modified parts of lotus, please describe which areas were modified,
-and the scope of those modifications

.github/ISSUE_TEMPLATE/proving-issues.md

@@ -1,46 +0,0 @@
----
-name: Proving Issues
-about: Create a report for help with proving failures.
-title: "[Proving Issue]"
-labels: area/proving, hint/needs-triaging
-assignees: ''
-
----
-
-> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
-
-Please provide all the information requested here to help us troubleshoot "proving/window PoSt failed" issues.
-If the information requested is missing, we will probably have to just ask you to provide it anyway,
-before we can help debug.
-
-**Describe the problem**
-A brief description of the problem you encountered while proving the storage.
-
-**Version**
-
-The output of `lotus --version`.
-
-**Setup**
-
-You miner and daemon setup, including what hardware do you use, your environment variable settings, how do you run your miner and worker, do you use GPU and etc.
-
-**Proving status**
-
-The output of `lotus-miner proving` info.
-
-**Lotus miner logs**
-
-Please go through the logs of your miner, and include screenshots of any error-like messages you find, highlighting the one has "window post" in it.
-
-Alternatively please upload full log files and share a link here
-
-**Lotus miner diagnostic info**
-
-Please collect the following diagnostic information, and share a link here
-
-* lotus-miner diagnostic info `lotus-miner info all > allinfo.txt`
-
-** Code modifications **
-
-If you have modified parts of lotus, please describe which areas were modified,
-and the scope of those modifications

.github/ISSUE_TEMPLATE/sealing-issues.md

@@ -1,50 +0,0 @@
----
-name: Sealing Issues
-about: Create a report for help with sealing (commit) failures.
-title: "[Sealing Issue]"
-labels: hint/needs-triaging, area/sealing
-assignees: ''
-
----
-
-> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
-
-Please provide all the information requested here to help us troubleshoot "commit failed" issues.
-If the information requested is missing, we will probably have to just ask you to provide it anyway,
-before we can help debug.
-
-**Describe the problem**
-A brief description of the problem you encountered while sealing a sector.
-
-**Version**
-
-The output of `lotus --version`.
-
-**Setup**
-
-You miner and daemon setup, including what hardware do you use, your environment variable settings, how do you run your miner and worker, do you use GPU and etc.
-
-**Commands**
-
-Commands you ran.
-
-**Sectors status**
-
-The output of `lotus-miner sectors status --log <sectorId>` for the failed sector(s).
-
-**Lotus miner logs**
-
-Please go through the logs of your miner, and include screenshots of any error-like messages you find.
-
-Alternatively please upload full log files and share a link here
-
-**Lotus miner diagnostic info**
-
-Please collect the following diagnostic information, and share a link here
-
-* lotus-miner diagnostic info `lotus-miner info all > allinfo`
-
-** Code modifications **
-
-If you have modified parts of lotus, please describe which areas were modified,
-and the scope of those modifications

.github/ISSUE_TEMPLATE/sealingfailed.md

@@ -0,0 +1,34 @@
+---
+name: Sealing Issues
+about: Create a report for help with sealing (commit) failures.
+title: ''
+labels: 'sealing'
+assignees: ''
+
+---
+
+Please provide all the information requested here to help us troubleshoot "commit failed" issues.
+If the information requested is missing, we will probably have to just ask you to provide it anyway,
+before we can help debug.
+
+**Describe the problem**
+
+A brief description of the problem you encountered while proving (sealing) a sector.
+
+Including what commands you ran, and a description of your setup, is very helpful.
+
+**Sectors list**
+
+The output of `./lotus-storage-miner sectors list`.
+
+**Sectors status**
+
+The output of `./lotus-storage-miner sectors status --log <sectorId>` for the failed sector(s).
+
+**Lotus storage miner logs**
+
+Please go through the logs of your storage miner, and include screenshots of any error-like messages you find.
+
+**Version**
+
+The output of `./lotus --version`.

.github/labels.yml

@@ -1,248 +0,0 @@
-###
-### Special magic GitHub labels
-### https://help.github.com/en/github/building-a-strong-community/encouraging-helpful-contributions-to-your-project-with-labels
-#
-- name: "good first issue"
-  color: 7057ff
-  description: "Good for newcomers"
-- name: "help wanted"
-  color: 008672
-  description: "Extra attention is needed"
-
-###
-### Goals
-#
-- name: goal/incentives
-  color: ff004d
-  description: "Incentinet"
-
-###
-### Areas
-#
-- name: area/ux
-  color: 00A4E0
-  description: "Area: UX"
-- name: area/chain/vm
-  color: 00A4E2
-  description: "Area: Chain/VM"
-- name: area/chain/sync
-  color: 00A4E4
-  description: "Area: Chain/Sync"
-- name: area/chain/misc
-  color: 00A4E6
-  description: "Area: Chain/Misc"
-- name: area/markets
-  color: 00A4E8
-  description: "Area: Markets"  
-- name: area/sealing/fsm
-  color: 0bb1ed
-  description: "Area: Sealing/FSM"
-- name: area/sealing/storage
-  color: 0EB4F0
-  description: "Area: Sealing/Storage"
-- name: area/proving
-  color: 0EB4F0
-  description: "Area: Proving"
-- name: area/mining
-  color: 10B6F2
-  description: "Area: Mining"
-- name: area/client/storage
-  color: 13B9F5
-  description: "Area: Client/Storage"
-- name: area/client/retrieval
-  color: 15BBF7
-  description: "Area: Client/Retrieval"
-- name: area/wallet
-  color: 15BBF7
-  description: "Area: Wallet"
-- name: area/payment-channel
-  color: ff6767
-  description: "Area: Payment Channel"
-- name: area/multisig
-  color: fff0ff
-  description: "Area: Multisig"
-- name: area/networking
-  color: 273f8a
-  description: "Area: Networking"
-
-###
-### Kinds
-#
-- name: kind/bug
-  color: c92712
-  description: "Kind: Bug"
-- name: kind/chore
-  color: fcf0b5
-  description: "Kind: Chore"
-- name: kind/feature
-  color: FFF3B8
-  description: "Kind: Feature"
-- name: kind/improvement
-  color: FFF5BA
-  description: "Kind: Improvement"
-- name: kind/test
-  color: FFF8BD
-  description: "Kind: Test"
-- name: kind/question
-  color: FFFDC2
-  description: "Kind: Question"
-- name: kind/enhancement
-  color: FFFFC5
-  description: "Kind: Enhancement"
-- name: kind/discussion
-  color: FFFFC7
-  description: "Kind: Discussion"
-
-###
-### Difficulties
-#
-- name: dif/trivial
-  color: b2b7ff
-  description: "Can be confidently tackled by newcomers, who are widely unfamiliar with lotus"
-- name: dif/easy
-  color: 7886d7
-  description: "An existing lotus user should be able to pick this up"
-- name: dif/medium
-  color: 6574cd
-  description: "Prior development experience with lotus is likely helpful"
-- name: dif/hard
-  color: 5661b3
-  description: "Suggests that having worked on the specific component affected by this issue is important"
-- name: dif/expert
-  color: 2f365f
-  description: "Requires extensive knowledge of the history, implications, ramifications of the issue"
-
-###
-### Efforts
-#
-- name: effort/minutes
-  color: e8fffe
-  description: "Effort: Minutes"
-- name: effort/hours
-  color: a0f0ed
-  description: "Effort: Hours"
-- name: effort/day
-  color: 64d5ca
-  description: "Effort: One Day"
-- name: effort/days
-  color: 4dc0b5
-  description: "Effort: Multiple Days"
-- name: effort/week
-  color: 38a89d
-  description: "Effort: One Week"
-- name: effort/weeks
-  color: 20504f
-  description: "Effort: Multiple Weeks"
-
-###
-### Impacts
-#
-- name: impact/regression
-  color: f1f5f8
-  description: "Impact: Regression"
-- name: impact/api-breakage
-  color: ECF0F3
-  description: "Impact: API Breakage"
-- name: impact/quality
-  color: E7EBEE
-  description: "Impact: Quality"
-- name: impact/dx
-  color: E2E6E9
-  description: "Impact: Developer Experience"
-- name: impact/test-flakiness
-  color: DDE1E4
-  description: "Impact: Test Flakiness"
-- name: impact/consensus
-  color: b20014
-  description: "Impact: Consensus"
-
-###
-### Topics
-#
-- name: topic/interoperability
-  color: bf0f73
-  description: "Topic: Interoperability"
-- name: topic/specs
-  color: CC1C80
-  description: "Topic: Specs"
-- name: topic/docs
-  color: D9298D
-  description: "Topic: Documentation"
-- name: topic/architecture
-  color: E53599
-  description: "Topic: Architecture"
-
-###
-### Priorities
-###
-- name: P0
-  color: dd362a
-  description: "P0: Critical Blocker"
-- name: P1
-  color: ce8048
-  description: "P1: Must be resolved"
-- name: P2
-  color: dbd81a
-  description: "P2: Should be resolved"
-- name: P3
-  color: 9fea8f
-  description: "P3: Might get resolved"
-
-###
-### Hints
-#
-#- name: hint/good-first-issue
-#  color: 7057ff
-#  description: "Hint: Good First Issue"
-#- name: hint/help-wanted
-#  color: 008672
-#  description: "Hint: Help Wanted"
-- name: hint/needs-decision
-  color: 33B9A5
-  description: "Hint: Needs Decision"
-- name: hint/needs-triage
-  color: 1AA08C
-  description: "Hint: Needs Triage"
-- name: hint/needs-analysis
-  color: 26AC98
-  description: "Hint: Needs Analysis"
-- name: hint/needs-author-input
-  color: 33B9A5
-  description: "Hint: Needs Author Input"
-- name: hint/needs-team-input
-  color: 40C6B2
-  description: "Hint: Needs Team Input"
-- name: hint/needs-community-input
-  color: 4DD3BF
-  description: "Hint: Needs Community Input"
-- name: hint/needs-review
-  color: 5AE0CC
-  description: "Hint: Needs Review"
-
-###
-### Statuses
-#
-- name: status/done
-  color: edb3a6
-  description: "Status: Done"
-- name: status/deferred
-  color: E0A699
-  description: "Status: Deferred"
-- name: status/in-progress
-  color: D49A8D
-  description: "Status: In Progress"
-- name: status/blocked
-  color: C78D80
-  description: "Status: Blocked"
-- name: status/inactive
-  color: BA8073
-  description: "Status: Inactive"
-- name: status/waiting
-  color: AD7366
-  description: "Status: Waiting"
-- name: status/rotten
-  color: 7A4033
-  description: "Status: Rotten"
-- name: status/discarded
-  color: 6D3326
-  description: "Status: Discarded / Won't fix"

.github/workflows/codeql-analysis.yml

@@ -1,69 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ master ]
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'go' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
-        # Learn more:
-        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-
-    - uses: actions/setup-go@v1
-      with:
-        go-version: '1.16.4'
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.github/workflows/label-syncer.yml

@@ -1,17 +0,0 @@
-
-name: Label syncer
-on:
-  push:
-    paths:
-      - '.github/labels.yml'
-    branches:
-      - master
-jobs:
-  build:
-    name: Sync labels
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@1.0.0
-      - uses: micnncim/action-label-syncer@v1.0.0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/stale.yml

@@ -1,27 +0,0 @@
-name: Close and mark stale issue
-
-on:
-  schedule:
-  - cron: '0 0 * * *'
-
-jobs:
-  stale:
-
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-
-    steps:
-    - uses: actions/stale@v3
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 24 hours.'
-        close-issue-message: 'This issue was closed because it is missing author input.'
-        stale-issue-label: 'kind/stale'
-        any-of-labels: 'hint/needs-author-input'
-        days-before-issue-stale: 5
-        days-before-issue-close: 1
-        enable-statistics: true
-       
-

.github/workflows/testground-on-push.yml

@@ -1,29 +0,0 @@
----
-name: Testground PR Checker
-
-on: [push]
-
-jobs:
-  testground:
-    runs-on: ubuntu-latest
-    name: ${{ matrix.composition_file }}
-    strategy:
-      matrix:
-        include:
-          - backend_addr: ci.testground.ipfs.team
-            backend_proto: https
-            plan_directory: testplans/lotus-soup
-            composition_file: testplans/lotus-soup/_compositions/baseline-k8s-3-1.toml
-          - backend_addr: ci.testground.ipfs.team
-            backend_proto: https
-            plan_directory: testplans/lotus-soup
-            composition_file: testplans/lotus-soup/_compositions/paych-stress-k8s.toml
-    steps:
-      - uses: actions/checkout@v2
-      - name: testground run
-        uses: coryschwartz/testground-github-action@v1.1
-        with:
-          backend_addr: ${{ matrix.backend_addr }}
-          backend_proto: ${{ matrix.backend_proto }}
-          plan_directory: ${{ matrix.plan_directory }}
-          composition_file: ${{ matrix.composition_file }}

.gitignore

@@ -1,31 +1,19 @@
-/AppDir
-/appimage-builder-cache
-*.AppImage
 /lotus
-/lotus-miner
+/lotus-storage-miner
-/lotus-worker
+/lotus-seal-worker
 /lotus-seed
 /lotus-health
-/lotus-chainwatch
 /lotus-shed
-/lotus-sim
+/pond
-/lotus-pond
+/townhall
-/lotus-townhall
+/fountain
-/lotus-fountain
+/stats
-/lotus-stats
+/bench
-/lotus-bench
-/lotus-gateway
-/lotus-pcr
-/lotus-wallet
-/lotus-keygen
-/docgen-md
-/docgen-openrpc
 /bench.json
 /lotuspond/front/node_modules
 /lotuspond/front/build
 /cmd/lotus-townhall/townhall/node_modules
 /cmd/lotus-townhall/townhall/build
-/cmd/lotus-townhall/townhall/package-lock.json
 extern/filecoin-ffi/rust/target
 **/*.a
 **/*.pc
@@ -36,6 +24,7 @@ build/paramfetch.sh
 /vendor
 /blocks.dot
 /blocks.svg
+/chainwatch
 /chainwatch.db
 /bundle
 /darwin
@@ -46,4 +35,3 @@ build/paramfetch.sh
 bin/ipget
 bin/tmp/*
 .idea
-scratchpad

.gitmodules

@@ -1,9 +1,7 @@
 [submodule "extern/filecoin-ffi"]
 	path = extern/filecoin-ffi
 	url = https://github.com/filecoin-project/filecoin-ffi.git
+	branch = master
 [submodule "extern/serialization-vectors"]
 	path = extern/serialization-vectors
-	url = https://github.com/filecoin-project/serialization-vectors.git
+	url = https://github.com/filecoin-project/serialization-vectors
-[submodule "extern/test-vectors"]
-	path = extern/test-vectors
-	url = https://github.com/filecoin-project/test-vectors.git

.golangci.yml

@@ -16,12 +16,6 @@ linters:
     - deadcode
     - scopelint
 
-# We don't want to skip builtin/
-skip-dirs-use-default: false
-skip-dirs:
-  - vendor$
-  - testdata$
-  - examples$
 
 issues:
   exclude:
@@ -29,14 +23,6 @@ issues:
     - "Potential file inclusion via variable"
     - "should have( a package)? comment"
     - "Error return value of `logging.SetLogLevel` is not checked"
-    - "comment on exported"
-    - "(func|method) \\w+ should be \\w+"
-    - "(type|var|struct field|(method|func) parameter) `\\w+` should be `\\w+`"
-    - "(G306|G301|G307|G108|G302|G204|G104)"
-    - "don't use ALL_CAPS in Go names"
-    - "string .* has .* occurrences, make it a constant"
-    - "a blank import should be only in a main or test package, or have a comment justifying it"
-    - "package comment should be of the form"
 
   exclude-use-default: false
   exclude-rules:
@@ -60,19 +46,6 @@ issues:
       linters:
         - gosec
 
-    - path: chain/vectors/gen/.*
-      linters:
-        - gosec
-
-    - path: cmd/lotus-bench/.*
-      linters:
-        - gosec
-
-    - path: api/test/.*
-      text: "context.Context should be the first parameter"
-      linters:
-        - golint
-
 linters-settings:
   goconst:
     min-occurrences: 6

AppDir/usr/share/icons/icon.svg

@@ -1 +0,0 @@
-<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0" y="0" viewBox="0 0 127 127" xml:space="preserve" enable-background="new 0 0 127 127"><style type="text/css">.st0{fill:#00d2d6}.st1{fill:#fff}</style><g><path class="st0" d="M63.5,127C28.5,127.1-0.2,98.4,0,63.2C0.2,28.3,28.6-0.2,63.9,0c34.8,0.2,63.3,28.7,63.1,64 C126.7,98.7,98.5,127.1,63.5,127z M71.4,57.6c5.5,0.8,11,1.5,16.5,2.3c0.5-1.7,0.9-3.1,1.3-4.7c-5.7-0.8-11.2-1.7-17.1-2.5 c2-7,3.7-13.7,5.8-20.2c0.7-2.2,2.3-4.2,3.9-5.9c2.1-2.2,5-1.7,6.8,0.7c0.7,1,1.4,2.1,2.3,2.9c1.1,1.1,2.8,1.6,4,0.6 c0.8-0.7,0.7-2.4,0.8-3.6c0-0.5-0.6-1.1-1-1.6c-2-2.3-4.7-3.1-7.5-3.2c-6.3-0.3-10.9,3-14.5,7.8c-3.5,4.8-5.1,10.5-6.8,16.2 c-0.5,1.6-0.9,3.3-1.4,5.1c-6.2-0.9-12.1-1.7-18.2-2.6c-0.2,1.6-0.4,3.2-0.6,4.8c6,0.9,11.8,1.8,17.8,2.7c-0.8,3.4-1.5,6.5-2.3,9.7 c-5.8-0.8-11.4-1.6-17-2.4c-0.2,1.8-0.4,3.2-0.6,4.8c5.6,0.9,11,1.7,16.5,2.5c0,0.6,0.1,1,0,1.3c-1.7,7.4-3.4,14.8-5.3,22.2 c-0.9,3.5-2.4,6.9-5.3,9.3c-2.4,2-5,1.7-6.8-0.8c-0.8-1.1-1.5-2.5-2.6-3.3c-0.8-0.6-2.5-0.9-3.1-0.5c-0.9,0.7-1.5,2.2-1.4,3.3 c0.1,1,1,2.2,1.9,2.8c3,2.3,6.5,2.6,10,1.9c5.9-1.2,10.1-4.9,12.7-10.1c2-4.1,3.6-8.5,5-12.9c1.3-4,2.2-8,3.3-12.2 c5.8,0.8,11.5,1.7,17.3,2.5c0.5-1.7,0.9-3.2,1.4-4.8c-6.1-0.9-11.9-1.7-17.7-2.6C70.1,64,70.7,60.9,71.4,57.6z"/><path class="st1" d="M71.4,57.6c-0.7,3.3-1.3,6.4-2,9.8c5.9,0.9,11.7,1.7,17.7,2.6c-0.5,1.6-0.9,3.1-1.4,4.8 c-5.8-0.8-11.5-1.7-17.3-2.5c-1.1,4.2-2,8.3-3.3,12.2c-1.4,4.4-3,8.7-5,12.9c-2.6,5.2-6.8,8.9-12.7,10.1c-3.5,0.7-7,0.4-10-1.9 c-0.9-0.7-1.8-1.8-1.9-2.8c-0.1-1.1,0.5-2.7,1.4-3.3c0.6-0.5,2.3-0.1,3.1,0.5c1.1,0.8,1.8,2.1,2.6,3.3c1.8,2.5,4.4,2.9,6.8,0.8 c2.9-2.5,4.4-5.8,5.3-9.3c1.9-7.3,3.6-14.8,5.3-22.2c0.1-0.3,0-0.7,0-1.3c-5.4-0.8-10.8-1.7-16.5-2.5c0.2-1.6,0.4-3,0.6-4.8 c5.6,0.8,11.1,1.6,17,2.4c0.8-3.2,1.5-6.4,2.3-9.7c-6-0.9-11.7-1.8-17.8-2.7c0.2-1.6,0.4-3.2,0.6-4.8c6.1,0.9,12,1.7,18.2,2.6 c0.5-1.8,0.9-3.5,1.4-5.1c1.7-5.6,3.2-11.3,6.8-16.2c3.6-4.9,8.1-8.1,14.5-7.8c2.8,0.1,5.5,0.9,7.5,3.2c0.4,0.5,1,1.1,1,1.6 c-0.1,1.2,0,2.9-0.8,3.6c-1.1,1.1-2.8,0.5-4-0.6c-0.9-0.9-1.6-1.9-2.3-2.9c-1.8-2.4-4.7-2.9-6.8-0.7c-1.6,1.7-3.2,3.7-3.9,5.9 C75.7,39.4,74,46,72,53c5.9,0.9,11.4,1.7,17.1,2.5c-0.5,1.6-0.9,3.1-1.3,4.7C82.3,59.1,76.9,58.3,71.4,57.6z"/></g></svg>

AppImageBuilder.yml

@@ -1,73 +0,0 @@
-version: 1
-AppDir:
-  path: ./AppDir
-  app_info:
-    id: io.filecoin.lotus
-    name: Lotus
-    icon: icon
-    version: latest
-    exec: usr/bin/lotus
-    exec_args: $@
-  apt:
-    arch: amd64
-    allow_unauthenticated: true
-    sources:
-    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal main restricted
-    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-updates main restricted
-    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal universe
-    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-updates universe
-    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal multiverse
-    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-updates multiverse
-    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-backports main restricted
-        universe multiverse
-    - sourceline: deb http://security.ubuntu.com/ubuntu focal-security main restricted
-    - sourceline: deb http://security.ubuntu.com/ubuntu focal-security universe
-    - sourceline: deb http://security.ubuntu.com/ubuntu focal-security multiverse
-    - sourceline: deb https://cli-assets.heroku.com/apt ./
-    - sourceline: deb http://ppa.launchpad.net/openjdk-r/ppa/ubuntu focal main
-    - sourceline: deb http://ppa.launchpad.net/git-core/ppa/ubuntu focal main
-    - sourceline: deb http://archive.canonical.com/ubuntu focal partner
-    include:
-    - ocl-icd-libopencl1
-    - libhwloc15
-    exclude: []
-  files:
-    include:
-    - /usr/lib/x86_64-linux-gnu/libgcc_s.so.1
-    - /usr/lib/x86_64-linux-gnu/libpthread-2.31.so
-    - /usr/lib/x86_64-linux-gnu/libm-2.31.so
-    - /usr/lib/x86_64-linux-gnu/libdl-2.31.so
-    - /usr/lib/x86_64-linux-gnu/libc-2.31.so
-    - /usr/lib/x86_64-linux-gnu/libudev.so.1.6.17
-    exclude:
-    - usr/share/man
-    - usr/share/doc/*/README.*
-    - usr/share/doc/*/changelog.*
-    - usr/share/doc/*/NEWS.*
-    - usr/share/doc/*/TODO.*
-  test:
-    fedora:
-      image: appimagecrafters/tests-env:fedora-30
-      command: ./AppRun
-      use_host_x: true
-    debian:
-      image: appimagecrafters/tests-env:debian-stable
-      command: ./AppRun
-      use_host_x: true
-    arch:
-      image: appimagecrafters/tests-env:archlinux-latest
-      command: ./AppRun
-      use_host_x: true
-    centos:
-      image: appimagecrafters/tests-env:centos-7
-      command: ./AppRun
-      use_host_x: true
-    ubuntu:
-      image: appimagecrafters/tests-env:ubuntu-xenial
-      command: ./AppRun
-      use_host_x: true
-AppImage:
-  arch: x86_64
-  update-information: guess
-  sign-key: None
-

Dockerfile.lotus

@@ -1,74 +0,0 @@
-FROM golang:1.16.4 AS builder-deps
-MAINTAINER Lotus Development Team
-
-RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
-
-ARG RUST_VERSION=nightly
-ENV XDG_CACHE_HOME="/tmp"
-
-ENV RUSTUP_HOME=/usr/local/rustup \
-    CARGO_HOME=/usr/local/cargo \
-    PATH=/usr/local/cargo/bin:$PATH
-
-RUN wget "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init"; \
-    chmod +x rustup-init; \
-    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION; \
-    rm rustup-init; \
-    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
-    rustup --version; \
-    cargo --version; \
-    rustc --version;
-
-
-FROM builder-deps AS builder-local
-MAINTAINER Lotus Development Team
-
-COPY ./ /opt/filecoin
-WORKDIR /opt/filecoin
-RUN make clean deps
-
-
-FROM builder-local AS builder
-MAINTAINER Lotus Development Team
-
-WORKDIR /opt/filecoin
-
-ARG RUSTFLAGS=""
-ARG GOFLAGS=""
-
-RUN make deps lotus lotus-miner lotus-worker lotus-shed lotus-chainwatch lotus-stats
-
-
-FROM ubuntu:20.04 AS base
-MAINTAINER Lotus Development Team
-
-# Base resources
-COPY --from=builder /etc/ssl/certs                           /etc/ssl/certs
-COPY --from=builder /lib/x86_64-linux-gnu/libdl.so.2         /lib/
-COPY --from=builder /lib/x86_64-linux-gnu/librt.so.1         /lib/
-COPY --from=builder /lib/x86_64-linux-gnu/libgcc_s.so.1      /lib/
-COPY --from=builder /lib/x86_64-linux-gnu/libutil.so.1       /lib/
-COPY --from=builder /usr/lib/x86_64-linux-gnu/libltdl.so.7   /lib/
-COPY --from=builder /usr/lib/x86_64-linux-gnu/libnuma.so.1   /lib/
-COPY --from=builder /usr/lib/x86_64-linux-gnu/libhwloc.so.5  /lib/
-COPY --from=builder /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /lib/
-
-RUN useradd -r -u 532 -U fc
-
-
-FROM base AS lotus
-MAINTAINER Lotus Development Team
-
-COPY --from=builder /opt/filecoin/lotus      /usr/local/bin/
-COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/
-
-ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV LOTUS_PATH /var/lib/lotus
-
-RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters && chown fc /var/lib/lotus /var/tmp/filecoin-proof-parameters
-
-USER fc
-
-ENTRYPOINT ["/usr/local/bin/lotus"]
-
-CMD ["-help"]

Makefile

@@ -5,10 +5,10 @@ all: build
 
 unexport GOFLAGS
 
-GOVERSION:=$(shell go version | cut -d' ' -f 3 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
+GOVERSION:=$(shell go version | cut -d' ' -f 3 | cut -d. -f 2)
-ifeq ($(shell expr $(GOVERSION) \< 1016000), 1)
+ifeq ($(shell expr $(GOVERSION) \< 13), 1)
-$(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell expr $(GOVERSION) % 1000000 / 1000).$(shell expr $(GOVERSION) % 1000))
+$(warning Your Golang version is go 1.$(GOVERSION))
-$(error Update Golang to version to at least 1.16.0)
+$(error Update Golang to version $(shell grep '^go' go.mod))
 endif
 
 # git modules that need to be loaded
@@ -41,13 +41,8 @@ MODULES+=$(FFI_PATH)
 BUILD_DEPS+=build/.filecoin-install
 CLEAN+=build/.filecoin-install
 
-ffi-version-check:
-	@[[ "$$(awk '/const Version/{print $$5}' extern/filecoin-ffi/version.go)" -eq 3 ]] || (echo "FFI version mismatch, update submodules"; exit 1)
-BUILD_DEPS+=ffi-version-check
-
-.PHONY: ffi-version-check
-
 $(MODULES): build/.update-modules ;
+
 # dummy file that marks the last time modules were updated
 build/.update-modules:
 	git submodule update --init --recursive
@@ -62,80 +57,71 @@ CLEAN+=build/.update-modules
 deps: $(BUILD_DEPS)
 .PHONY: deps
 
-build-devnets: build lotus-seed lotus-shed lotus-wallet lotus-gateway
-.PHONY: build-devnets
-
 debug: GOFLAGS+=-tags=debug
-debug: build-devnets
+debug: lotus lotus-storage-miner lotus-seal-worker lotus-seed
 
 2k: GOFLAGS+=-tags=2k
-2k: build-devnets
+2k: lotus lotus-storage-miner lotus-seal-worker lotus-seed
-
-calibnet: GOFLAGS+=-tags=calibnet
-calibnet: build-devnets
-
-nerpanet: GOFLAGS+=-tags=nerpanet
-nerpanet: build-devnets
-
-butterflynet: GOFLAGS+=-tags=butterflynet
-butterflynet: build-devnets
-
-interopnet: GOFLAGS+=-tags=interopnet
-interopnet: build-devnets
 
 lotus: $(BUILD_DEPS)
 	rm -f lotus
 	go build $(GOFLAGS) -o lotus ./cmd/lotus
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus -i ./build
 
 .PHONY: lotus
 BINS+=lotus
 
-lotus-miner: $(BUILD_DEPS)
+lotus-storage-miner: $(BUILD_DEPS)
-	rm -f lotus-miner
+	rm -f lotus-storage-miner
-	go build $(GOFLAGS) -o lotus-miner ./cmd/lotus-storage-miner
+	go build $(GOFLAGS) -o lotus-storage-miner ./cmd/lotus-storage-miner
-.PHONY: lotus-miner
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus-storage-miner -i ./build
-BINS+=lotus-miner
+.PHONY: lotus-storage-miner
+BINS+=lotus-storage-miner
 
-lotus-worker: $(BUILD_DEPS)
+lotus-seal-worker: $(BUILD_DEPS)
-	rm -f lotus-worker
+	rm -f lotus-seal-worker
-	go build $(GOFLAGS) -o lotus-worker ./cmd/lotus-seal-worker
+	go build $(GOFLAGS) -o lotus-seal-worker ./cmd/lotus-seal-worker
-.PHONY: lotus-worker
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus-seal-worker -i ./build
-BINS+=lotus-worker
+.PHONY: lotus-seal-worker
+BINS+=lotus-seal-worker
 
 lotus-shed: $(BUILD_DEPS)
 	rm -f lotus-shed
 	go build $(GOFLAGS) -o lotus-shed ./cmd/lotus-shed
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus-shed -i ./build
 .PHONY: lotus-shed
 BINS+=lotus-shed
 
-lotus-gateway: $(BUILD_DEPS)
+build: lotus lotus-storage-miner lotus-seal-worker
-	rm -f lotus-gateway
-	go build $(GOFLAGS) -o lotus-gateway ./cmd/lotus-gateway
-.PHONY: lotus-gateway
-BINS+=lotus-gateway
-
-build: lotus lotus-miner lotus-worker
 	@[[ $$(type -P "lotus") ]] && echo "Caution: you have \
 an existing lotus binary in your PATH. This may cause problems if you don't run 'sudo make install'" || true
 
 .PHONY: build
 
-install: install-daemon install-miner install-worker
+install:
-
-install-daemon:
 	install -C ./lotus /usr/local/bin/lotus
+	install -C ./lotus-storage-miner /usr/local/bin/lotus-storage-miner
+	install -C ./lotus-seal-worker /usr/local/bin/lotus-seal-worker
 
-install-miner:
+install-services: install
-	install -C ./lotus-miner /usr/local/bin/lotus-miner
+	mkdir -p /usr/local/lib/systemd/system
+	install -C -m 0644 ./scripts/lotus-daemon.service /usr/local/lib/systemd/system/lotus-daemon.service
+	install -C -m 0644 ./scripts/lotus-miner.service /usr/local/lib/systemd/system/lotus-miner.service
+	systemctl daemon-reload
+	@echo
+	@echo "lotus and lotus-miner services installed. Don't forget to 'systemctl enable lotus|lotus-miner' for it to be enabled on startup."
 
-install-worker:
+clean-services:
-	install -C ./lotus-worker /usr/local/bin/lotus-worker
+	rm -f /usr/local/lib/systemd/system/lotus-daemon.service
+	rm -f /usr/local/lib/systemd/system/lotus-miner.service
+	systemctl daemon-reload
 
 # TOOLS
 
 lotus-seed: $(BUILD_DEPS)
 	rm -f lotus-seed
 	go build $(GOFLAGS) -o lotus-seed ./cmd/lotus-seed
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus-seed -i ./build
 
 .PHONY: lotus-seed
 BINS+=lotus-seed
@@ -146,164 +132,71 @@ benchmarks:
 	@curl -X POST 'http://benchmark.kittyhawk.wtf/benchmark' -d '@bench.json' -u "${benchmark_http_cred}"
 .PHONY: benchmarks
 
-lotus-pond: 2k
+pond: 2k
-	go build -o lotus-pond ./lotuspond
+	go build -o pond ./lotuspond
-.PHONY: lotus-pond
-BINS+=lotus-pond
-
-lotus-pond-front:
 	(cd lotuspond/front && npm i && CI=false npm run build)
-.PHONY: lotus-pond-front
+.PHONY: pond
+BINS+=pond
 
-lotus-pond-app: lotus-pond-front lotus-pond
+townhall:
-.PHONY: lotus-pond-app
+	rm -f townhall
-
+	go build -o townhall ./cmd/lotus-townhall
-lotus-townhall:
-	rm -f lotus-townhall
-	go build -o lotus-townhall ./cmd/lotus-townhall
-.PHONY: lotus-townhall
-BINS+=lotus-townhall
-
-lotus-townhall-front:
 	(cd ./cmd/lotus-townhall/townhall && npm i && npm run build)
-.PHONY: lotus-townhall-front
+	go run github.com/GeertJohan/go.rice/rice append --exec townhall -i ./cmd/lotus-townhall -i ./build
+.PHONY: townhall
+BINS+=townhall
 
-lotus-townhall-app: lotus-touch lotus-townhall-front
+fountain:
-.PHONY: lotus-townhall-app
+	rm -f fountain
+	go build -o fountain ./cmd/lotus-fountain
+	go run github.com/GeertJohan/go.rice/rice append --exec fountain -i ./cmd/lotus-fountain -i ./build
+.PHONY: fountain
+BINS+=fountain
 
-lotus-fountain:
+chainwatch:
-	rm -f lotus-fountain
+	rm -f chainwatch
-	go build -o lotus-fountain ./cmd/lotus-fountain
+	go build -o chainwatch ./cmd/lotus-chainwatch
-.PHONY: lotus-fountain
+	go run github.com/GeertJohan/go.rice/rice append --exec chainwatch -i ./cmd/lotus-chainwatch -i ./build
-BINS+=lotus-fountain
+.PHONY: chainwatch
+BINS+=chainwatch
 
-lotus-chainwatch:
+bench:
-	rm -f lotus-chainwatch
+	rm -f bench
-	go build $(GOFLAGS) -o lotus-chainwatch ./cmd/lotus-chainwatch
+	go build -o bench ./cmd/lotus-bench
-.PHONY: lotus-chainwatch
+	go run github.com/GeertJohan/go.rice/rice append --exec bench -i ./build
-BINS+=lotus-chainwatch
+.PHONY: bench
+BINS+=bench
 
-lotus-bench:
+stats:
-	rm -f lotus-bench
+	rm -f stats
-	go build -o lotus-bench ./cmd/lotus-bench
+	go build -o stats ./tools/stats
-.PHONY: lotus-bench
+	go run github.com/GeertJohan/go.rice/rice append --exec stats -i ./build
-BINS+=lotus-bench
+.PHONY: stats
+BINS+=stats
 
-lotus-stats:
+health:
-	rm -f lotus-stats
-	go build $(GOFLAGS) -o lotus-stats ./cmd/lotus-stats
-.PHONY: lotus-stats
-BINS+=lotus-stats
-
-lotus-pcr:
-	rm -f lotus-pcr
-	go build $(GOFLAGS) -o lotus-pcr ./cmd/lotus-pcr
-.PHONY: lotus-pcr
-BINS+=lotus-pcr
-
-lotus-health:
 	rm -f lotus-health
 	go build -o lotus-health ./cmd/lotus-health
-.PHONY: lotus-health
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus-health -i ./build
-BINS+=lotus-health
 
-lotus-wallet:
+.PHONY: health
-	rm -f lotus-wallet
+BINS+=health
-	go build -o lotus-wallet ./cmd/lotus-wallet
-.PHONY: lotus-wallet
-BINS+=lotus-wallet
-
-lotus-keygen:
-	rm -f lotus-keygen
-	go build -o lotus-keygen ./cmd/lotus-keygen
-.PHONY: lotus-keygen
-BINS+=lotus-keygen
-
-testground:
-	go build -tags testground -o /dev/null ./cmd/lotus
-.PHONY: testground
-BINS+=testground
-
-
-tvx:
-	rm -f tvx
-	go build -o tvx ./cmd/tvx
-.PHONY: tvx
-BINS+=tvx
-
-install-chainwatch: lotus-chainwatch
-	install -C ./lotus-chainwatch /usr/local/bin/lotus-chainwatch
-
-lotus-sim: $(BUILD_DEPS)
-	rm -f lotus-sim
-	go build $(GOFLAGS) -o lotus-sim ./cmd/lotus-sim
-.PHONY: lotus-sim
-BINS+=lotus-sim
-
-# SYSTEMD
-
-install-daemon-service: install-daemon
-	mkdir -p /etc/systemd/system
-	mkdir -p /var/log/lotus
-	install -C -m 0644 ./scripts/lotus-daemon.service /etc/systemd/system/lotus-daemon.service
-	systemctl daemon-reload
-	@echo
-	@echo "lotus-daemon service installed. Don't forget to run 'sudo systemctl start lotus-daemon' to start it and 'sudo systemctl enable lotus-daemon' for it to be enabled on startup."
-
-install-miner-service: install-miner install-daemon-service
-	mkdir -p /etc/systemd/system
-	mkdir -p /var/log/lotus
-	install -C -m 0644 ./scripts/lotus-miner.service /etc/systemd/system/lotus-miner.service
-	systemctl daemon-reload
-	@echo
-	@echo "lotus-miner service installed. Don't forget to run 'sudo systemctl start lotus-miner' to start it and 'sudo systemctl enable lotus-miner' for it to be enabled on startup."
-
-install-chainwatch-service: install-chainwatch install-daemon-service
-	mkdir -p /etc/systemd/system
-	mkdir -p /var/log/lotus
-	install -C -m 0644 ./scripts/lotus-chainwatch.service /etc/systemd/system/lotus-chainwatch.service
-	systemctl daemon-reload
-	@echo
-	@echo "chainwatch service installed. Don't forget to run 'sudo systemctl start lotus-chainwatch' to start it and 'sudo systemctl enable lotus-chainwatch' for it to be enabled on startup."
-
-install-main-services: install-miner-service
-
-install-all-services: install-main-services install-chainwatch-service
-
-install-services: install-main-services
-
-clean-daemon-service: clean-miner-service clean-chainwatch-service
-	-systemctl stop lotus-daemon
-	-systemctl disable lotus-daemon
-	rm -f /etc/systemd/system/lotus-daemon.service
-	systemctl daemon-reload
-
-clean-miner-service:
-	-systemctl stop lotus-miner
-	-systemctl disable lotus-miner
-	rm -f /etc/systemd/system/lotus-miner.service
-	systemctl daemon-reload
-
-clean-chainwatch-service:
-	-systemctl stop lotus-chainwatch
-	-systemctl disable lotus-chainwatch
-	rm -f /etc/systemd/system/lotus-chainwatch.service
-	systemctl daemon-reload
-
-clean-main-services: clean-daemon-service
-
-clean-all-services: clean-main-services
-
-clean-services: clean-all-services
 
 # MISC
 
 buildall: $(BINS)
 
+completions:
+	./scripts/make-completions.sh lotus
+	./scripts/make-completions.sh lotus-storage-miner
+.PHONY: completions
+
 install-completions:
 	mkdir -p /usr/share/bash-completion/completions /usr/local/share/zsh/site-functions/
 	install -C ./scripts/bash-completion/lotus /usr/share/bash-completion/completions/lotus
+	install -C ./scripts/bash-completion/lotus-storage-miner /usr/share/bash-completion/completions/lotus-storage-miner
 	install -C ./scripts/zsh-completion/lotus /usr/local/share/zsh/site-functions/_lotus
+	install -C ./scripts/zsh-completion/lotus-storage-miner /usr/local/share/zsh/site-functions/_lotus-storage-miner
 
 clean:
 	rm -rf $(CLEAN) $(BINS)
@@ -315,76 +208,13 @@ dist-clean:
 	git submodule deinit --all -f
 .PHONY: dist-clean
 
-type-gen: api-gen
+type-gen:
 	go run ./gen/main.go
-	go generate -x ./...
-	goimports -w api/
 
-method-gen: api-gen
+method-gen:
 	(cd ./lotuspond/front/src/chain && go run ./methodgen.go)
 
-actors-gen:
+gen: type-gen method-gen
-	go run ./chain/actors/agen
-	go fmt ./...
-
-api-gen:
-	go run ./gen/api
-	goimports -w api
-	goimports -w api
-.PHONY: api-gen
-
-appimage: lotus
-	rm -rf appimage-builder-cache || true
-	rm AppDir/io.filecoin.lotus.desktop || true
-	rm AppDir/icon.svg || true
-	rm Appdir/AppRun || true
-	mkdir -p AppDir/usr/bin
-	cp ./lotus AppDir/usr/bin/
-	appimage-builder
-
-docsgen: docsgen-md docsgen-openrpc
-
-docsgen-md-bin: api-gen actors-gen
-	go build $(GOFLAGS) -o docgen-md ./api/docgen/cmd
-docsgen-openrpc-bin: api-gen actors-gen
-	go build $(GOFLAGS) -o docgen-openrpc ./api/docgen-openrpc/cmd
-
-docsgen-md: docsgen-md-full docsgen-md-storage docsgen-md-worker
-
-docsgen-md-full: docsgen-md-bin
-	./docgen-md "api/api_full.go" "FullNode" "api" "./api" > documentation/en/api-v1-unstable-methods.md
-	./docgen-md "api/v0api/full.go" "FullNode" "v0api" "./api/v0api" > documentation/en/api-v0-methods.md
-docsgen-md-storage: docsgen-md-bin
-	./docgen-md "api/api_storage.go" "StorageMiner" "api" "./api" > documentation/en/api-v0-methods-miner.md
-docsgen-md-worker: docsgen-md-bin
-	./docgen-md "api/api_worker.go" "Worker" "api" "./api" > documentation/en/api-v0-methods-worker.md
-
-docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker
-
-docsgen-openrpc-full: docsgen-openrpc-bin
-	./docgen-openrpc "api/api_full.go" "FullNode" "api" "./api" -gzip > build/openrpc/full.json.gz
-docsgen-openrpc-storage: docsgen-openrpc-bin
-	./docgen-openrpc "api/api_storage.go" "StorageMiner" "api" "./api" -gzip > build/openrpc/miner.json.gz
-docsgen-openrpc-worker: docsgen-openrpc-bin
-	./docgen-openrpc "api/api_worker.go" "Worker" "api" "./api" -gzip > build/openrpc/worker.json.gz
-
-.PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin
-
-gen: actors-gen type-gen method-gen docsgen api-gen circleci
-	@echo ">>> IF YOU'VE MODIFIED THE CLI, REMEMBER TO ALSO MAKE docsgen-cli"
-.PHONY: gen
-
-snap: lotus lotus-miner lotus-worker
-	snapcraft
-	# snapcraft upload ./lotus_*.snap
-
-# separate from gen because it needs binaries
-docsgen-cli: lotus lotus-miner lotus-worker
-	python ./scripts/generate-lotus-cli.py
-.PHONY: docsgen-cli
 
 print-%:
 	@echo $*=$($*)
-
-circleci:
-	go generate -x ./.circleci

README.md

@@ -1,138 +1,16 @@
-<p align="center">
+![Lotus](documentation/images/lotus_logo_h.png)
-  <a href="https://docs.filecoin.io/" title="Filecoin Docs">
-    <img src="documentation/images/lotus_logo_h.png" alt="Project Lotus Logo" width="244" />
-  </a>
-</p>
 
-<h1 align="center">Project Lotus - 莲</h1>
+# Project Lotus - 莲
 
-<p align="center">
+Lotus is an implementation of the Filecoin Distributed Storage Network. For more details about Filecoin, check out the [Filecoin Spec](https://github.com/filecoin-project/specs).
-  <a href="https://circleci.com/gh/filecoin-project/lotus"><img src="https://circleci.com/gh/filecoin-project/lotus.svg?style=svg"></a>
-  <a href="https://codecov.io/gh/filecoin-project/lotus"><img src="https://codecov.io/gh/filecoin-project/lotus/branch/master/graph/badge.svg"></a>
-  <a href="https://goreportcard.com/report/github.com/filecoin-project/lotus"><img src="https://goreportcard.com/badge/github.com/filecoin-project/lotus" /></a>  
-  <a href=""><img src="https://img.shields.io/badge/golang-%3E%3D1.16-blue.svg" /></a>
-  <br>
-</p>
 
-Lotus is an implementation of the Filecoin Distributed Storage Network. For more details about Filecoin, check out the [Filecoin Spec](https://spec.filecoin.io).
+## Development
+
+All work is tracked via issues. An attempt at keeping an up-to-date view on remaining work is in the [lotus testnet github project board](https://github.com/filecoin-project/lotus/projects/1).
 
 ## Building & Documentation
 
-> Note: The default `master` branch is the dev branch, please use with caution. For the latest stable version, checkout the most recent [`Latest release`](https://github.com/filecoin-project/lotus/releases).
+For instructions on how to build lotus from source, please visit [https://docs.lotu.sh](https://docs.lotu.sh) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation).
- 
-For complete instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/). Basic build instructions can be found further down in this readme.
-
-## Reporting a Vulnerability
-
-Please send an email to security@filecoin.org. See our [security policy](SECURITY.md) for more details.
-
-## Related packages
-
-These repos are independent and reusable modules, but are tightly integrated into Lotus to make up a fully featured Filecoin implementation:
-
-- [go-fil-markets](https://github.com/filecoin-project/go-fil-markets) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/markets-shared-components-5daa144a7046a60001c6e253/board)
-- [specs-actors](https://github.com/filecoin-project/specs-actors) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/actors-5ee6f3aa87591f0016c05685/board)
-
-## Contribute
-
-Lotus is a universally open project and welcomes contributions of all kinds: code, docs, and more. However, before making a contribution, we ask you to heed these recommendations:
-
-1. If the proposal entails a protocol change, please first submit a [Filecoin Improvement Proposal](https://github.com/filecoin-project/FIPs).
-2. If the change is complex and requires prior discussion, [open an issue](github.com/filecoin-project/lotus/issues) or a [discussion](https://github.com/filecoin-project/lotus/discussions) to request feedback before you start working on a pull request. This is to avoid disappointment and sunk costs, in case the change is not actually needed or accepted.
-3. Please refrain from submitting PRs to adapt existing code to subjective preferences. The changeset should contain functional or technical improvements/enhancements, bug fixes, new features, or some other clear material contribution. Simple stylistic changes are likely to be rejected in order to reduce code churn.
-
-When implementing a change:
-
-1. Adhere to the standard Go formatting guidelines, e.g. [Effective Go](https://golang.org/doc/effective_go.html). Run `go fmt`.
-2. Stick to the idioms and patterns used in the codebase. Familiar-looking code has a higher chance of being accepted than eerie code. Pay attention to commonly used variable and parameter names, avoidance of naked returns, error handling patterns, etc.
-3. Comments: follow the advice on the [Commentary](https://golang.org/doc/effective_go.html#commentary) section of Effective Go.
-4. Minimize code churn. Modify only what is strictly necessary. Well-encapsulated changesets will get a quicker response from maintainers.
-5. Lint your code with [`golangci-lint`](https://golangci-lint.run) (CI will reject your PR if unlinted).
-6. Add tests.
-7. Title the PR in a meaningful way and describe the rationale and the thought process in the PR description.
-8. Write clean, thoughtful, and detailed [commit messages](https://chris.beams.io/posts/git-commit/). This is even more important than the PR description, because commit messages are stored _inside_ the Git history. One good rule is: if you are happy posting the commit message as the PR description, then it's a good commit message.
-
-## Basic Build Instructions
-**System-specific Software Dependencies**:
-
-Building Lotus requires some system dependencies, usually provided by your distribution.
-
-Ubuntu/Debian:
-```
-sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl clang build-essential hwloc libhwloc-dev wget -y && sudo apt upgrade -y
-```
-
-Fedora:
-```
-sudo dnf -y install gcc make git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm wget hwloc libhwloc-dev
-```
-
-For other distributions you can find the required dependencies [here.](https://docs.filecoin.io/get-started/lotus/installation/#system-specific) For instructions specific to macOS, you can find them [here.](https://docs.filecoin.io/get-started/lotus/installation/#macos)
-
-#### Go
-
-To build Lotus, you need a working installation of [Go 1.16.4 or higher](https://golang.org/dl/):
-
-```bash
-wget -c https://golang.org/dl/go1.16.4.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
-```
-
-**TIP:**
-You'll need to add `/usr/local/go/bin` to your path. For most Linux distributions you can run something like:
-
-```shell
-echo "export PATH=$PATH:/usr/local/go/bin" >> ~/.bashrc && source ~/.bashrc
-```
-
-See the [official Golang installation instructions](https://golang.org/doc/install) if you get stuck.
-
-### Build and install Lotus
-
-Once all the dependencies are installed, you can build and install the Lotus suite (`lotus`, `lotus-miner`, and `lotus-worker`).
-
-1. Clone the repository:
-
-   ```sh
-   git clone https://github.com/filecoin-project/lotus.git
-   cd lotus/
-   ```
-   
-Note: The default branch `master` is the dev branch where the latest new features, bug fixes and improvement are in. However, if you want to run lotus on Filecoin mainnet and want to run a production-ready lotus, get the latest release[ here](https://github.com/filecoin-project/lotus/releases).
-
-2. To join mainnet, checkout the [latest release](https://github.com/filecoin-project/lotus/releases).
-
-   If you are changing networks from a previous Lotus installation or there has been a network reset, read the [Switch networks guide](https://docs.filecoin.io/get-started/lotus/switch-networks/) before proceeding.
-
-   For networks other than mainnet, look up the current branch or tag/commit for the network you want to join in the [Filecoin networks dashboard](https://network.filecoin.io), then build Lotus for your specific network below.
-
-   ```sh
-   git checkout <tag_or_branch>
-   # For example:
-   git checkout <vX.X.X> # tag for a release
-   ```
-
-   Currently, the latest code on the _master_ branch corresponds to mainnet.
-
-3. If you are in China, see "[Lotus: tips when running in China](https://docs.filecoin.io/get-started/lotus/tips-running-in-china/)".
-4. This build instruction uses the prebuilt proofs binaries. If you want to build the proof binaries from source check the [complete instructions](https://docs.filecoin.io/get-started/lotus/installation/#build-and-install-lotus). Note, if you are building the proof binaries from source, [installing rustup](https://docs.filecoin.io/get-started/lotus/installation/#rustup) is also needed.
-
-5. Build and install Lotus:
-
-   ```sh
-   make clean all #mainnet
-
-   # Or to join a testnet or devnet:
-   make clean calibnet # Calibration with min 32GiB sectors
-   make clean nerpanet # Nerpa with min 512MiB sectors
-
-   sudo make install
-   ```
-
-   This will put `lotus`, `lotus-miner` and `lotus-worker` in `/usr/local/bin`.
-
-   `lotus` will use the `$HOME/.lotus` folder by default for storage (configuration, chain data, wallets, etc). See [advanced options](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/) for information on how to customize the Lotus folder.
-
-6. You should now have Lotus installed. You can now [start the Lotus daemon and sync the chain](https://docs.filecoin.io/get-started/lotus/installation/#start-the-lotus-daemon-and-sync-the-chain).
 
 ## License
 

SECURITY.md

@@ -1,23 +0,0 @@
-# Security Policy
-
-## Reporting a Vulnerability
-
-For reporting security vulnerabilities/bugs, please consult our Security Policy and Responsible Disclosure Program information at https://github.com/filecoin-project/community/blob/master/SECURITY.md. Security vulnerabilities should be reported via our [Vulnerability Reporting channels](https://github.com/filecoin-project/community/blob/master/SECURITY.md#vulnerability-reporting) and will be eligible for a [Bug Bounty](https://security.filecoin.io/bug-bounty/).
-
-Please try to provide a clear description of any bugs reported, along with how to reproduce the bug if possible. More detailed bug reports (especially those with a PoC included) will help us move forward much faster. Additionally, please avoid reporting bugs that already have open issues. Take a moment to search the issue list of the related GitHub repositories before writing up a new report.
-
-Here are some examples of bugs we would consider to be security vulnerabilities:
-
-* If you can spend from a `multisig` wallet you do not control the keys for.
-* If you can cause a miner to be slashed without them actually misbehaving.
-* If you can maintain power without submitting windowed posts regularly.
-* If you can craft a message that causes lotus nodes to panic.
-* If you can cause your miner to win significantly more blocks than it should.
-* If you can craft a message that causes a persistent fork in the network.
-* If you can cause the total amount of Filecoin in the network to no longer be 2 billion.
-
-This is not an exhaustive list, but should provide some idea of what we consider as a security vulnerability, . 
-
-## Reporting a non security bug
-
-For non-security bugs, please simply file a GitHub [issue](https://github.com/filecoin-project/lotus/issues/new?template=bug_report.md). 

api/README.md

@@ -1,14 +0,0 @@
-## Lotus API
-
-This package contains all lotus API definitions. Interfaces defined here are
-exposed as JsonRPC 2.0 endpoints by lotus programs.
-
-### Versions
-
-| File             | Alias File        | Interface      | Exposed by         | Version | HTTP Endpoint | Status                       | Docs
-|------------------|-------------------|----------------|--------------------|---------|---------------|------------------------------|------
-| `api_common.go`  | `v0api/latest.go` | `Common`       | lotus; lotus-miner | v0      | `/rpc/v0`     | Latest, Stable               | [Methods](../documentation/en/api-v0-methods.md)
-| `api_full.go`    | `v1api/latest.go` | `FullNode`     | lotus              | v1      | `/rpc/v1`     | Latest, **Work in progress** | [Methods](../documentation/en/api-v1-unstable-methods.md)
-| `api_storage.go` | `v0api/latest.go` | `StorageMiner` | lotus-miner        | v0      | `/rpc/v0`     | Latest, Stable               | [Methods](../documentation/en/api-v0-methods-miner.md)
-| `api_worker.go`  | `v0api/latest.go` | `Worker`       | lotus-worker       | v0      | `/rpc/v0`     | Latest, Stable               | [Methods](../documentation/en/api-v0-methods-worker.md)
-| `v0api/full.go`  |                   | `FullNode`     | lotus              | v0      | `/rpc/v0`     | Stable                       | [Methods](../documentation/en/api-v0-methods.md)

api/api_common.go

@@ -4,97 +4,51 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/google/uuid"
-
-	"github.com/filecoin-project/go-jsonrpc/auth"
-	metrics "github.com/libp2p/go-libp2p-core/metrics"
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
-	protocol "github.com/libp2p/go-libp2p-core/protocol"
 
-	apitypes "github.com/filecoin-project/lotus/api/types"
+	"github.com/filecoin-project/go-jsonrpc/auth"
+
+	"github.com/filecoin-project/lotus/build"
 )
 
-//                       MODIFYING THE API INTERFACE
-//
-// When adding / changing methods in this file:
-// * Do the change here
-// * Adjust implementation in `node/impl/`
-// * Run `make gen` - this will:
-//  * Generate proxy structs
-//  * Generate mocks
-//  * Generate markdown docs
-//  * Generate openrpc blobs
-
 type Common interface {
+	// Auth
+	AuthVerify(ctx context.Context, token string) ([]auth.Permission, error)
+	AuthNew(ctx context.Context, perms []auth.Permission) ([]byte, error)
 
-	// MethodGroup: Auth
+	// network
 
-	AuthVerify(ctx context.Context, token string) ([]auth.Permission, error) //perm:read
+	NetConnectedness(context.Context, peer.ID) (network.Connectedness, error)
-	AuthNew(ctx context.Context, perms []auth.Permission) ([]byte, error)    //perm:admin
+	NetPeers(context.Context) ([]peer.AddrInfo, error)
-
+	NetConnect(context.Context, peer.AddrInfo) error
-	// MethodGroup: Net
+	NetAddrsListen(context.Context) (peer.AddrInfo, error)
-
+	NetDisconnect(context.Context, peer.ID) error
-	NetConnectedness(context.Context, peer.ID) (network.Connectedness, error) //perm:read
+	NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error)
-	NetPeers(context.Context) ([]peer.AddrInfo, error)                        //perm:read
+	NetPubsubScores(context.Context) ([]PubsubScore, error)
-	NetConnect(context.Context, peer.AddrInfo) error                          //perm:write
-	NetAddrsListen(context.Context) (peer.AddrInfo, error)                    //perm:read
-	NetDisconnect(context.Context, peer.ID) error                             //perm:write
-	NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error)              //perm:read
-	NetPubsubScores(context.Context) ([]PubsubScore, error)                   //perm:read
-	NetAutoNatStatus(context.Context) (NatInfo, error)                        //perm:read
-	NetAgentVersion(ctx context.Context, p peer.ID) (string, error)           //perm:read
-	NetPeerInfo(context.Context, peer.ID) (*ExtendedPeerInfo, error)          //perm:read
-
-	// NetBandwidthStats returns statistics about the nodes total bandwidth
-	// usage and current rate across all peers and protocols.
-	NetBandwidthStats(ctx context.Context) (metrics.Stats, error) //perm:read
-
-	// NetBandwidthStatsByPeer returns statistics about the nodes bandwidth
-	// usage and current rate per peer
-	NetBandwidthStatsByPeer(ctx context.Context) (map[string]metrics.Stats, error) //perm:read
-
-	// NetBandwidthStatsByProtocol returns statistics about the nodes bandwidth
-	// usage and current rate per protocol
-	NetBandwidthStatsByProtocol(ctx context.Context) (map[protocol.ID]metrics.Stats, error) //perm:read
-
-	// ConnectionGater API
-	NetBlockAdd(ctx context.Context, acl NetBlockList) error    //perm:admin
-	NetBlockRemove(ctx context.Context, acl NetBlockList) error //perm:admin
-	NetBlockList(ctx context.Context) (NetBlockList, error)     //perm:read
-
-	// MethodGroup: Common
-
-	// Discover returns an OpenRPC document describing an RPC API.
-	Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) //perm:read
 
 	// ID returns peerID of libp2p node backing this API
-	ID(context.Context) (peer.ID, error) //perm:read
+	ID(context.Context) (peer.ID, error)
 
 	// Version provides information about API provider
-	Version(context.Context) (APIVersion, error) //perm:read
+	Version(context.Context) (Version, error)
 
-	LogList(context.Context) ([]string, error)         //perm:write
+	LogList(context.Context) ([]string, error)
-	LogSetLevel(context.Context, string, string) error //perm:write
+	LogSetLevel(context.Context, string, string) error
 
 	// trigger graceful shutdown
-	Shutdown(context.Context) error //perm:admin
+	Shutdown(context.Context) error
-
-	// Session returns a random UUID of api provider session
-	Session(context.Context) (uuid.UUID, error) //perm:read
-
-	Closing(context.Context) (<-chan struct{}, error) //perm:read
 }
 
-// APIVersion provides various build-time information
+// Version provides various build-time information
-type APIVersion struct {
+type Version struct {
 	Version string
 
 	// APIVersion is a binary encoded semver version of the remote implementing
 	// this api
 	//
 	// See APIVersion in build/version.go
-	APIVersion Version
+	APIVersion build.Version
 
 	// TODO: git commit / os / genesis cid?
 
@@ -102,11 +56,6 @@ type APIVersion struct {
 	BlockDelay uint64
 }
 
-func (v APIVersion) String() string {
+func (v Version) String() string {
 	return fmt.Sprintf("%s+api%s", v.Version, v.APIVersion.String())
 }
-
-type NatInfo struct {
-	Reachability network.Reachability
-	PublicAddr   string
-}

api/api_gateway.go

@@ -1,62 +0,0 @@
-package api
-
-import (
-	"context"
-
-	"github.com/ipfs/go-cid"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/dline"
-
-	apitypes "github.com/filecoin-project/lotus/api/types"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-//                       MODIFYING THE API INTERFACE
-//
-// NOTE: This is the V1 (Unstable) API - to add methods to the V0 (Stable) API
-// you'll have to add those methods to interfaces in `api/v0api`
-//
-// When adding / changing methods in this file:
-// * Do the change here
-// * Adjust implementation in `node/impl/`
-// * Run `make gen` - this will:
-//  * Generate proxy structs
-//  * Generate mocks
-//  * Generate markdown docs
-//  * Generate openrpc blobs
-
-type Gateway interface {
-	ChainHasObj(context.Context, cid.Cid) (bool, error)
-	ChainHead(ctx context.Context) (*types.TipSet, error)
-	ChainGetBlockMessages(context.Context, cid.Cid) (*BlockMessages, error)
-	ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
-	ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error)
-	ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error)
-	ChainNotify(context.Context) (<-chan []*HeadChange, error)
-	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
-	GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
-	MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
-	MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
-	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
-	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
-	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
-	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
-	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
-	StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
-	StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
-	StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MarketBalance, error)
-	StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*MarketDeal, error)
-	StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error)
-	StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
-	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error)
-	StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error)
-	StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
-	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
-	StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
-	StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
-	WalletBalance(context.Context, address.Address) (types.BigInt, error)
-	Version(context.Context) (APIVersion, error)
-}

api/api_storage.go

@@ -3,206 +3,61 @@ package api
 import (
 	"bytes"
 	"context"
-	"time"
 
-	"github.com/filecoin-project/lotus/chain/actors/builtin"
-
-	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
-	"github.com/libp2p/go-libp2p-core/peer"
 
 	"github.com/filecoin-project/go-address"
-	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-fil-markets/piecestore"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
-	"github.com/filecoin-project/specs-storage/storage"
-
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
+	"github.com/filecoin-project/sector-storage/stores"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
+	"github.com/filecoin-project/sector-storage/storiface"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
 )
 
-//                       MODIFYING THE API INTERFACE
-//
-// When adding / changing methods in this file:
-// * Do the change here
-// * Adjust implementation in `node/impl/`
-// * Run `make gen` - this will:
-//  * Generate proxy structs
-//  * Generate mocks
-//  * Generate markdown docs
-//  * Generate openrpc blobs
-
 // StorageMiner is a low-level interface to the Filecoin network storage miner node
 type StorageMiner interface {
 	Common
 
-	ActorAddress(context.Context) (address.Address, error) //perm:read
+	ActorAddress(context.Context) (address.Address, error)
 
-	ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error) //perm:read
+	ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error)
-	ActorAddressConfig(ctx context.Context) (AddressConfig, error)            //perm:read
 
-	MiningBase(context.Context) (*types.TipSet, error) //perm:read
+	MiningBase(context.Context) (*types.TipSet, error)
 
 	// Temp api for testing
-	PledgeSector(context.Context) (abi.SectorID, error) //perm:write
+	PledgeSector(context.Context) error
 
 	// Get the status of a given sector by ID
-	SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (SectorInfo, error) //perm:read
+	SectorsStatus(context.Context, abi.SectorNumber) (SectorInfo, error)
-
-	// Add piece to an open sector. If no sectors with enough space are open,
-	// either a new sector will be created, or this call will block until more
-	// sectors can be created.
-	SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storage.Data, d PieceDealInfo) (SectorOffset, error) //perm:admin
-
-	SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error //perm:admin
 
 	// List all staged sectors
-	SectorsList(context.Context) ([]abi.SectorNumber, error) //perm:read
+	SectorsList(context.Context) ([]abi.SectorNumber, error)
 
-	// Get summary info of sectors
+	SectorsRefs(context.Context) (map[string][]SealedRef, error)
-	SectorsSummary(ctx context.Context) (map[SectorState]int, error) //perm:read
 
-	// List sectors in particular states
+	SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error
-	SectorsListInStates(context.Context, []SectorState) ([]abi.SectorNumber, error) //perm:read
 
-	SectorsRefs(context.Context) (map[string][]SealedRef, error) //perm:read
+	StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error)
-
+	StorageLocal(ctx context.Context) (map[stores.ID]string, error)
-	// SectorStartSealing can be called on sectors in Empty or WaitDeals states
+	StorageStat(ctx context.Context, id stores.ID) (stores.FsStat, error)
-	// to trigger sealing early
-	SectorStartSealing(context.Context, abi.SectorNumber) error //perm:write
-	// SectorSetSealDelay sets the time that a newly-created sector
-	// waits for more deals before it starts sealing
-	SectorSetSealDelay(context.Context, time.Duration) error //perm:write
-	// SectorGetSealDelay gets the time that a newly-created sector
-	// waits for more deals before it starts sealing
-	SectorGetSealDelay(context.Context) (time.Duration, error) //perm:read
-	// SectorSetExpectedSealDuration sets the expected time for a sector to seal
-	SectorSetExpectedSealDuration(context.Context, time.Duration) error //perm:write
-	// SectorGetExpectedSealDuration gets the expected time for a sector to seal
-	SectorGetExpectedSealDuration(context.Context) (time.Duration, error) //perm:read
-	SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error   //perm:admin
-	// SectorRemove removes the sector from storage. It doesn't terminate it on-chain, which can
-	// be done with SectorTerminate. Removing and not terminating live sectors will cause additional penalties.
-	SectorRemove(context.Context, abi.SectorNumber) error //perm:admin
-	// SectorTerminate terminates the sector on-chain (adding it to a termination batch first), then
-	// automatically removes it from storage
-	SectorTerminate(context.Context, abi.SectorNumber) error //perm:admin
-	// SectorTerminateFlush immediately sends a terminate message with sectors batched for termination.
-	// Returns null if message wasn't sent
-	SectorTerminateFlush(ctx context.Context) (*cid.Cid, error) //perm:admin
-	// SectorTerminatePending returns a list of pending sector terminations to be sent in the next batch message
-	SectorTerminatePending(ctx context.Context) ([]abi.SectorID, error)  //perm:admin
-	SectorMarkForUpgrade(ctx context.Context, id abi.SectorNumber) error //perm:admin
-	// SectorPreCommitFlush immediately sends a PreCommit message with sectors batched for PreCommit.
-	// Returns null if message wasn't sent
-	SectorPreCommitFlush(ctx context.Context) ([]sealiface.PreCommitBatchRes, error) //perm:admin
-	// SectorPreCommitPending returns a list of pending PreCommit sectors to be sent in the next batch message
-	SectorPreCommitPending(ctx context.Context) ([]abi.SectorID, error) //perm:admin
-	// SectorCommitFlush immediately sends a Commit message with sectors aggregated for Commit.
-	// Returns null if message wasn't sent
-	SectorCommitFlush(ctx context.Context) ([]sealiface.CommitBatchRes, error) //perm:admin
-	// SectorCommitPending returns a list of pending Commit sectors to be sent in the next aggregate message
-	SectorCommitPending(ctx context.Context) ([]abi.SectorID, error) //perm:admin
 
 	// WorkerConnect tells the node to connect to workers RPC
-	WorkerConnect(context.Context, string) error                              //perm:admin retry:true
+	WorkerConnect(context.Context, string) error
-	WorkerStats(context.Context) (map[uuid.UUID]storiface.WorkerStats, error) //perm:admin
+	WorkerStats(context.Context) (map[uint64]storiface.WorkerStats, error)
-	WorkerJobs(context.Context) (map[uuid.UUID][]storiface.WorkerJob, error)  //perm:admin
 
-	//storiface.WorkerReturn
+	stores.SectorIndex
-	ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error                //perm:admin retry:true
-	ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storage.PreCommit1Out, err *storiface.CallError) error //perm:admin retry:true
-	ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storage.SectorCids, err *storiface.CallError) error //perm:admin retry:true
-	ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storage.Commit1Out, err *storiface.CallError) error       //perm:admin retry:true
-	ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storage.Proof, err *storiface.CallError) error          //perm:admin retry:true
-	ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                            //perm:admin retry:true
-	ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                           //perm:admin retry:true
-	ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                               //perm:admin retry:true
-	ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                               //perm:admin retry:true
-	ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err *storiface.CallError) error                        //perm:admin retry:true
-	ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                     //perm:admin retry:true
 
-	// SealingSchedDiag dumps internal sealing scheduler state
+	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error
-	SealingSchedDiag(ctx context.Context, doSched bool) (interface{}, error) //perm:admin
+	MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error)
-	SealingAbort(ctx context.Context, call storiface.CallID) error           //perm:admin
+	MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error)
+	MarketSetPrice(context.Context, types.BigInt) error
 
-	//stores.SectorIndex
+	DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error
-	StorageAttach(context.Context, stores.StorageInfo, fsutil.FsStat) error                                                                                             //perm:admin
+	DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error)
-	StorageInfo(context.Context, stores.ID) (stores.StorageInfo, error)                                                                                                 //perm:admin
-	StorageReportHealth(context.Context, stores.ID, stores.HealthReport) error                                                                                          //perm:admin
-	StorageDeclareSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error                                     //perm:admin
-	StorageDropSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType) error                                                      //perm:admin
-	StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]stores.SectorStorageInfo, error) //perm:admin
-	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]stores.StorageInfo, error)           //perm:admin
-	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                                          //perm:admin
-	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                               //perm:admin
-	StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error)                                                                                               //perm:admin
 
-	StorageLocal(ctx context.Context) (map[stores.ID]string, error)       //perm:admin
+	StorageAddLocal(ctx context.Context, path string) error
-	StorageStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) //perm:admin
-
-	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error                                                                                                        //perm:write
-	MarketListDeals(ctx context.Context) ([]MarketDeal, error)                                                                                                                           //perm:read
-	MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error)                                                                                           //perm:read
-	MarketGetDealUpdates(ctx context.Context) (<-chan storagemarket.MinerDeal, error)                                                                                                    //perm:read
-	MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error)                                                                                                    //perm:read
-	MarketSetAsk(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error //perm:admin
-	MarketGetAsk(ctx context.Context) (*storagemarket.SignedStorageAsk, error)                                                                                                           //perm:read
-	MarketSetRetrievalAsk(ctx context.Context, rask *retrievalmarket.Ask) error                                                                                                          //perm:admin
-	MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error)                                                                                                             //perm:read
-	MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)                                                                                                          //perm:write
-	MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)                                                                                                   //perm:write
-	// MarketRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer
-	MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
-	// MarketCancelDataTransfer cancels a data transfer with the given transfer ID and other peer
-	MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
-	MarketPendingDeals(ctx context.Context) (PendingDealInfo, error)                                                             //perm:write
-	MarketPublishPendingDeals(ctx context.Context) error                                                                         //perm:admin
-
-	DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error //perm:admin
-	DealsList(ctx context.Context) ([]MarketDeal, error)                         //perm:admin
-	DealsConsiderOnlineStorageDeals(context.Context) (bool, error)               //perm:admin
-	DealsSetConsiderOnlineStorageDeals(context.Context, bool) error              //perm:admin
-	DealsConsiderOnlineRetrievalDeals(context.Context) (bool, error)             //perm:admin
-	DealsSetConsiderOnlineRetrievalDeals(context.Context, bool) error            //perm:admin
-	DealsPieceCidBlocklist(context.Context) ([]cid.Cid, error)                   //perm:admin
-	DealsSetPieceCidBlocklist(context.Context, []cid.Cid) error                  //perm:admin
-	DealsConsiderOfflineStorageDeals(context.Context) (bool, error)              //perm:admin
-	DealsSetConsiderOfflineStorageDeals(context.Context, bool) error             //perm:admin
-	DealsConsiderOfflineRetrievalDeals(context.Context) (bool, error)            //perm:admin
-	DealsSetConsiderOfflineRetrievalDeals(context.Context, bool) error           //perm:admin
-	DealsConsiderVerifiedStorageDeals(context.Context) (bool, error)             //perm:admin
-	DealsSetConsiderVerifiedStorageDeals(context.Context, bool) error            //perm:admin
-	DealsConsiderUnverifiedStorageDeals(context.Context) (bool, error)           //perm:admin
-	DealsSetConsiderUnverifiedStorageDeals(context.Context, bool) error          //perm:admin
-
-	StorageAddLocal(ctx context.Context, path string) error //perm:admin
-
-	PiecesListPieces(ctx context.Context) ([]cid.Cid, error)                                 //perm:read
-	PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error)                               //perm:read
-	PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) //perm:read
-	PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error)   //perm:read
-
-	// CreateBackup creates node backup onder the specified file name. The
-	// method requires that the lotus-miner is running with the
-	// LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that
-	// the path specified when calling CreateBackup is within the base path
-	CreateBackup(ctx context.Context, fpath string) error //perm:admin
-
-	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
-
-	ComputeProof(ctx context.Context, ssi []builtin.SectorInfo, rand abi.PoStRandomness) ([]builtin.PoStProof, error) //perm:read
 }
 
-var _ storiface.WorkerReturn = *new(StorageMiner)
-var _ stores.SectorIndex = *new(StorageMiner)
-
 type SealRes struct {
 	Err   string
 	GoErr error `json:"-"`
@@ -220,40 +75,24 @@ type SectorLog struct {
 }
 
 type SectorInfo struct {
-	SectorID     abi.SectorNumber
+	SectorID abi.SectorNumber
-	State        SectorState
+	State    SectorState
-	CommD        *cid.Cid
+	CommD    *cid.Cid
-	CommR        *cid.Cid
+	CommR    *cid.Cid
-	Proof        []byte
+	Proof    []byte
-	Deals        []abi.DealID
+	Deals    []abi.DealID
-	Ticket       SealTicket
+	Ticket   SealTicket
-	Seed         SealSeed
+	Seed     SealSeed
-	PreCommitMsg *cid.Cid
+	Retries  uint64
-	CommitMsg    *cid.Cid
-	Retries      uint64
-	ToUpgrade    bool
 
 	LastErr string
 
 	Log []SectorLog
-
-	// On Chain Info
-	SealProof          abi.RegisteredSealProof // The seal proof type implies the PoSt proof/s
-	Activation         abi.ChainEpoch          // Epoch during which the sector proof was accepted
-	Expiration         abi.ChainEpoch          // Epoch during which the sector expires
-	DealWeight         abi.DealWeight          // Integral of active deals over sector lifetime
-	VerifiedDealWeight abi.DealWeight          // Integral of active verified deals over sector lifetime
-	InitialPledge      abi.TokenAmount         // Pledge collected to commit this sector
-	// Expiration Info
-	OnTime abi.ChainEpoch
-	// non-zero if sector is faulty, epoch at which it will be permanently
-	// removed if it doesn't recover
-	Early abi.ChainEpoch
 }
 
 type SealedRef struct {
 	SectorID abi.SectorNumber
-	Offset   abi.PaddedPieceSize
+	Offset   uint64
 	Size     abi.UnpaddedPieceSize
 }
 
@@ -280,54 +119,3 @@ func (st *SealSeed) Equals(ost *SealSeed) bool {
 }
 
 type SectorState string
-
-type AddrUse int
-
-const (
-	PreCommitAddr AddrUse = iota
-	CommitAddr
-	DealPublishAddr
-	PoStAddr
-
-	TerminateSectorsAddr
-)
-
-type AddressConfig struct {
-	PreCommitControl   []address.Address
-	CommitControl      []address.Address
-	TerminateControl   []address.Address
-	DealPublishControl []address.Address
-
-	DisableOwnerFallback  bool
-	DisableWorkerFallback bool
-}
-
-// PendingDealInfo has info about pending deals and when they are due to be
-// published
-type PendingDealInfo struct {
-	Deals              []market.ClientDealProposal
-	PublishPeriodStart time.Time
-	PublishPeriod      time.Duration
-}
-
-type SectorOffset struct {
-	Sector abi.SectorNumber
-	Offset abi.PaddedPieceSize
-}
-
-// DealInfo is a tuple of deal identity and its schedule
-type PieceDealInfo struct {
-	PublishCid   *cid.Cid
-	DealID       abi.DealID
-	DealProposal *market.DealProposal
-	DealSchedule DealSchedule
-	KeepUnsealed bool
-}
-
-// DealSchedule communicates the time interval of a storage deal. The deal must
-// appear in a sealed (proven) sector no later than StartEpoch, otherwise it
-// is invalid.
-type DealSchedule struct {
-	StartEpoch abi.ChainEpoch
-	EndEpoch   abi.ChainEpoch
-}

api/api_test.go

@@ -1,16 +1,12 @@
 package api
 
 import (
-	"encoding/json"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"reflect"
 	"runtime"
 	"strings"
 	"testing"
-
-	"github.com/stretchr/testify/require"
 )
 
 func goCmd() string {
@@ -36,86 +32,3 @@ func TestDoesntDependOnFFI(t *testing.T) {
 		}
 	}
 }
-
-func TestDoesntDependOnBuild(t *testing.T) {
-	deps, err := exec.Command(goCmd(), "list", "-deps", "github.com/filecoin-project/lotus/api").Output()
-	if err != nil {
-		t.Fatal(err)
-	}
-	for _, pkg := range strings.Fields(string(deps)) {
-		if pkg == "github.com/filecoin-project/build" {
-			t.Fatal("api depends on filecoin-ffi")
-		}
-	}
-}
-
-func TestReturnTypes(t *testing.T) {
-	errType := reflect.TypeOf(new(error)).Elem()
-	bareIface := reflect.TypeOf(new(interface{})).Elem()
-	jmarsh := reflect.TypeOf(new(json.Marshaler)).Elem()
-
-	tst := func(api interface{}) func(t *testing.T) {
-		return func(t *testing.T) {
-			ra := reflect.TypeOf(api).Elem()
-			for i := 0; i < ra.NumMethod(); i++ {
-				m := ra.Method(i)
-				switch m.Type.NumOut() {
-				case 1: // if 1 return value, it must be an error
-					require.Equal(t, errType, m.Type.Out(0), m.Name)
-
-				case 2: // if 2 return values, first cant be an interface/function, second must be an error
-					seen := map[reflect.Type]struct{}{}
-					todo := []reflect.Type{m.Type.Out(0)}
-					for len(todo) > 0 {
-						typ := todo[len(todo)-1]
-						todo = todo[:len(todo)-1]
-
-						if _, ok := seen[typ]; ok {
-							continue
-						}
-						seen[typ] = struct{}{}
-
-						if typ.Kind() == reflect.Interface && typ != bareIface && !typ.Implements(jmarsh) {
-							t.Error("methods can't return interfaces", m.Name)
-						}
-
-						switch typ.Kind() {
-						case reflect.Ptr:
-							fallthrough
-						case reflect.Array:
-							fallthrough
-						case reflect.Slice:
-							fallthrough
-						case reflect.Chan:
-							todo = append(todo, typ.Elem())
-						case reflect.Map:
-							todo = append(todo, typ.Elem())
-							todo = append(todo, typ.Key())
-						case reflect.Struct:
-							for i := 0; i < typ.NumField(); i++ {
-								todo = append(todo, typ.Field(i).Type)
-							}
-						}
-					}
-
-					require.NotEqual(t, reflect.Func.String(), m.Type.Out(0).Kind().String(), m.Name)
-					require.Equal(t, errType, m.Type.Out(1), m.Name)
-
-				default:
-					t.Error("methods can only have 1 or 2 return values", m.Name)
-				}
-			}
-		}
-	}
-
-	t.Run("common", tst(new(Common)))
-	t.Run("full", tst(new(FullNode)))
-	t.Run("miner", tst(new(StorageMiner)))
-	t.Run("worker", tst(new(Worker)))
-}
-
-func TestPermTags(t *testing.T) {
-	_ = PermissionedFullAPI(&FullNodeStruct{})
-	_ = PermissionedStorMinerAPI(&StorageMinerStruct{})
-	_ = PermissionedWorkerAPI(&WorkerStruct{})
-}

api/api_wallet.go

@@ -1,47 +0,0 @@
-package api
-
-import (
-	"context"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/crypto"
-
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-type MsgType string
-
-const (
-	MTUnknown = "unknown"
-
-	// Signing message CID. MsgMeta.Extra contains raw cbor message bytes
-	MTChainMsg = "message"
-
-	// Signing a blockheader. signing raw cbor block bytes (MsgMeta.Extra is empty)
-	MTBlock = "block"
-
-	// Signing a deal proposal. signing raw cbor proposal bytes (MsgMeta.Extra is empty)
-	MTDealProposal = "dealproposal"
-
-	// TODO: Deals, Vouchers, VRF
-)
-
-type MsgMeta struct {
-	Type MsgType
-
-	// Additional data related to what is signed. Should be verifiable with the
-	// signed bytes (e.g. CID(Extra).Bytes() == toSign)
-	Extra []byte
-}
-
-type Wallet interface {
-	WalletNew(context.Context, types.KeyType) (address.Address, error) //perm:admin
-	WalletHas(context.Context, address.Address) (bool, error)          //perm:admin
-	WalletList(context.Context) ([]address.Address, error)             //perm:admin
-
-	WalletSign(ctx context.Context, signer address.Address, toSign []byte, meta MsgMeta) (*crypto.Signature, error) //perm:admin
-
-	WalletExport(context.Context, address.Address) (*types.KeyInfo, error) //perm:admin
-	WalletImport(context.Context, *types.KeyInfo) (address.Address, error) //perm:admin
-	WalletDelete(context.Context, address.Address) error                   //perm:admin
-}

api/api_worker.go

@@ -2,71 +2,35 @@ package api
 
 import (
 	"context"
+	"io"
 
-	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
 
-	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
+	"github.com/filecoin-project/sector-storage/stores"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
+	"github.com/filecoin-project/sector-storage/storiface"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-storage/storage"
+
+	"github.com/filecoin-project/lotus/build"
 )
 
-//                       MODIFYING THE API INTERFACE
+type WorkerAPI interface {
-//
+	Version(context.Context) (build.Version, error)
-// When adding / changing methods in this file:
+	// TODO: Info() (name, ...) ?
-// * Do the change here
-// * Adjust implementation in `node/impl/`
-// * Run `make gen` - this will:
-//  * Generate proxy structs
-//  * Generate mocks
-//  * Generate markdown docs
-//  * Generate openrpc blobs
 
-type Worker interface {
+	TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) // TaskType -> Weight
-	Version(context.Context) (Version, error) //perm:admin
+	Paths(context.Context) ([]stores.StoragePath, error)
+	Info(context.Context) (storiface.WorkerInfo, error)
 
-	// TaskType -> Weight
+	storage.Sealer
-	TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) //perm:admin
-	Paths(context.Context) ([]stores.StoragePath, error)                //perm:admin
-	Info(context.Context) (storiface.WorkerInfo, error)                 //perm:admin
 
-	// storiface.WorkerCalls
+	MoveStorage(ctx context.Context, sector abi.SectorID) error
-	AddPiece(ctx context.Context, sector storage.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (storiface.CallID, error)                    //perm:admin
-	SealPreCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error)                                                           //perm:admin
-	SealPreCommit2(ctx context.Context, sector storage.SectorRef, pc1o storage.PreCommit1Out) (storiface.CallID, error)                                                                                  //perm:admin
-	SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error) //perm:admin
-	SealCommit2(ctx context.Context, sector storage.SectorRef, c1o storage.Commit1Out) (storiface.CallID, error)                                                                                         //perm:admin
-	FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                                //perm:admin
-	ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) (storiface.CallID, error)                                                                                 //perm:admin
-	MoveStorage(ctx context.Context, sector storage.SectorRef, types storiface.SectorFileType) (storiface.CallID, error)                                                                                 //perm:admin
-	UnsealPiece(context.Context, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error)                                           //perm:admin
-	Fetch(context.Context, storage.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error)                                                             //perm:admin
 
-	TaskDisable(ctx context.Context, tt sealtasks.TaskType) error //perm:admin
+	UnsealPiece(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error
-	TaskEnable(ctx context.Context, tt sealtasks.TaskType) error  //perm:admin
+	ReadPiece(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) error
 
-	// Storage / Other
+	Fetch(context.Context, abi.SectorID, stores.SectorFileType, stores.PathType, stores.AcquireMode) error
-	Remove(ctx context.Context, sector abi.SectorID) error //perm:admin
 
-	StorageAddLocal(ctx context.Context, path string) error //perm:admin
+	Closing(context.Context) (<-chan struct{}, error)
-
-	// SetEnabled marks the worker as enabled/disabled. Not that this setting
-	// may take a few seconds to propagate to task scheduler
-	SetEnabled(ctx context.Context, enabled bool) error //perm:admin
-
-	Enabled(ctx context.Context) (bool, error) //perm:admin
-
-	// WaitQuiet blocks until there are no tasks running
-	WaitQuiet(ctx context.Context) error //perm:admin
-
-	// returns a random UUID of worker session, generated randomly when worker
-	// process starts
-	ProcessSession(context.Context) (uuid.UUID, error) //perm:admin
-
-	// Like ProcessSession, but returns an error when worker is disabled
-	Session(context.Context) (uuid.UUID, error) //perm:admin
 }
-
-var _ storiface.WorkerCalls = *new(Worker)

api/apibstore/apibstore.go

@@ -0,0 +1,67 @@
+package apibstore
+
+import (
+	"context"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	"golang.org/x/xerrors"
+)
+
+type ChainIO interface {
+	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
+	ChainHasObj(context.Context, cid.Cid) (bool, error)
+}
+
+type apiBStore struct {
+	api ChainIO
+}
+
+func NewAPIBlockstore(cio ChainIO) blockstore.Blockstore {
+	return &apiBStore{
+		api: cio,
+	}
+}
+
+func (a *apiBStore) DeleteBlock(cid.Cid) error {
+	return xerrors.New("not supported")
+}
+
+func (a *apiBStore) Has(c cid.Cid) (bool, error) {
+	return a.api.ChainHasObj(context.TODO(), c)
+}
+
+func (a *apiBStore) Get(c cid.Cid) (blocks.Block, error) {
+	bb, err := a.api.ChainReadObj(context.TODO(), c)
+	if err != nil {
+		return nil, err
+	}
+	return blocks.NewBlockWithCid(bb, c)
+}
+
+func (a *apiBStore) GetSize(c cid.Cid) (int, error) {
+	bb, err := a.api.ChainReadObj(context.TODO(), c)
+	if err != nil {
+		return 0, err
+	}
+	return len(bb), nil
+}
+
+func (a *apiBStore) Put(blocks.Block) error {
+	return xerrors.New("not supported")
+}
+
+func (a *apiBStore) PutMany([]blocks.Block) error {
+	return xerrors.New("not supported")
+}
+
+func (a *apiBStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+	return nil, xerrors.New("not supported")
+}
+
+func (a *apiBStore) HashOnRead(enabled bool) {
+	return
+}
+
+var _ blockstore.Blockstore = &apiBStore{}

api/apistruct/permissioned.go

@@ -1,7 +1,8 @@
-package api
+package apistruct
 
 import (
 	"github.com/filecoin-project/go-jsonrpc/auth"
+	"github.com/filecoin-project/lotus/api"
 )
 
 const (
@@ -16,28 +17,22 @@ const (
 var AllPermissions = []auth.Permission{PermRead, PermWrite, PermSign, PermAdmin}
 var DefaultPerms = []auth.Permission{PermRead}
 
-func PermissionedStorMinerAPI(a StorageMiner) StorageMiner {
+func PermissionedStorMinerAPI(a api.StorageMiner) api.StorageMiner {
 	var out StorageMinerStruct
 	auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
 	auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.CommonStruct.Internal)
 	return &out
 }
 
-func PermissionedFullAPI(a FullNode) FullNode {
+func PermissionedFullAPI(a api.FullNode) api.FullNode {
 	var out FullNodeStruct
 	auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
 	auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.CommonStruct.Internal)
 	return &out
 }
 
-func PermissionedWorkerAPI(a Worker) Worker {
+func PermissionedWorkerAPI(a api.WorkerAPI) api.WorkerAPI {
 	var out WorkerStruct
 	auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
 	return &out
 }
-
-func PermissionedWalletAPI(a Wallet) Wallet {
-	var out WalletStruct
-	auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
-	return &out
-}

api/apistruct/struct.go

@@ -0,0 +1,915 @@
+package apistruct
+
+import (
+	"context"
+	"io"
+
+	"github.com/ipfs/go-cid"
+	"github.com/libp2p/go-libp2p-core/network"
+	"github.com/libp2p/go-libp2p-core/peer"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/filecoin-project/go-jsonrpc/auth"
+
+	"github.com/filecoin-project/sector-storage/sealtasks"
+	"github.com/filecoin-project/sector-storage/stores"
+	"github.com/filecoin-project/sector-storage/storiface"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
+	"github.com/filecoin-project/specs-actors/actors/builtin/paych"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
+	"github.com/filecoin-project/specs-storage/storage"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+)
+
+// All permissions are listed in permissioned.go
+var _ = AllPermissions
+
+type CommonStruct struct {
+	Internal struct {
+		AuthVerify func(ctx context.Context, token string) ([]auth.Permission, error) `perm:"read"`
+		AuthNew    func(ctx context.Context, perms []auth.Permission) ([]byte, error) `perm:"admin"`
+
+		NetConnectedness func(context.Context, peer.ID) (network.Connectedness, error) `perm:"read"`
+		NetPeers         func(context.Context) ([]peer.AddrInfo, error)                `perm:"read"`
+		NetConnect       func(context.Context, peer.AddrInfo) error                    `perm:"write"`
+		NetAddrsListen   func(context.Context) (peer.AddrInfo, error)                  `perm:"read"`
+		NetDisconnect    func(context.Context, peer.ID) error                          `perm:"write"`
+		NetFindPeer      func(context.Context, peer.ID) (peer.AddrInfo, error)         `perm:"read"`
+		NetPubsubScores  func(context.Context) ([]api.PubsubScore, error)              `perm:"read"`
+
+		ID      func(context.Context) (peer.ID, error)     `perm:"read"`
+		Version func(context.Context) (api.Version, error) `perm:"read"`
+
+		LogList     func(context.Context) ([]string, error)     `perm:"write"`
+		LogSetLevel func(context.Context, string, string) error `perm:"write"`
+
+		Shutdown func(context.Context) error `perm:"admin"`
+	}
+}
+
+// FullNodeStruct implements API passing calls to user-provided function values.
+type FullNodeStruct struct {
+	CommonStruct
+
+	Internal struct {
+		ChainNotify            func(context.Context) (<-chan []*api.HeadChange, error)                                                            `perm:"read"`
+		ChainHead              func(context.Context) (*types.TipSet, error)                                                                       `perm:"read"`
+		ChainGetRandomness     func(context.Context, types.TipSetKey, crypto.DomainSeparationTag, abi.ChainEpoch, []byte) (abi.Randomness, error) `perm:"read"`
+		ChainGetBlock          func(context.Context, cid.Cid) (*types.BlockHeader, error)                                                         `perm:"read"`
+		ChainGetTipSet         func(context.Context, types.TipSetKey) (*types.TipSet, error)                                                      `perm:"read"`
+		ChainGetBlockMessages  func(context.Context, cid.Cid) (*api.BlockMessages, error)                                                         `perm:"read"`
+		ChainGetParentReceipts func(context.Context, cid.Cid) ([]*types.MessageReceipt, error)                                                    `perm:"read"`
+		ChainGetParentMessages func(context.Context, cid.Cid) ([]api.Message, error)                                                              `perm:"read"`
+		ChainGetTipSetByHeight func(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)                                      `perm:"read"`
+		ChainReadObj           func(context.Context, cid.Cid) ([]byte, error)                                                                     `perm:"read"`
+		ChainHasObj            func(context.Context, cid.Cid) (bool, error)                                                                       `perm:"read"`
+		ChainStatObj           func(context.Context, cid.Cid, cid.Cid) (api.ObjStat, error)                                                       `perm:"read"`
+		ChainSetHead           func(context.Context, types.TipSetKey) error                                                                       `perm:"admin"`
+		ChainGetGenesis        func(context.Context) (*types.TipSet, error)                                                                       `perm:"read"`
+		ChainTipSetWeight      func(context.Context, types.TipSetKey) (types.BigInt, error)                                                       `perm:"read"`
+		ChainGetNode           func(ctx context.Context, p string) (*api.IpldObject, error)                                                       `perm:"read"`
+		ChainGetMessage        func(context.Context, cid.Cid) (*types.Message, error)                                                             `perm:"read"`
+		ChainGetPath           func(context.Context, types.TipSetKey, types.TipSetKey) ([]*api.HeadChange, error)                                 `perm:"read"`
+		ChainExport            func(context.Context, types.TipSetKey) (<-chan []byte, error)                                                      `perm:"read"`
+
+		SyncState          func(context.Context) (*api.SyncState, error)                `perm:"read"`
+		SyncSubmitBlock    func(ctx context.Context, blk *types.BlockMsg) error         `perm:"write"`
+		SyncIncomingBlocks func(ctx context.Context) (<-chan *types.BlockHeader, error) `perm:"read"`
+		SyncMarkBad        func(ctx context.Context, bcid cid.Cid) error                `perm:"admin"`
+		SyncCheckBad       func(ctx context.Context, bcid cid.Cid) (string, error)      `perm:"read"`
+
+		MpoolPending          func(context.Context, types.TipSetKey) ([]*types.SignedMessage, error)                       `perm:"read"`
+		MpoolPush             func(context.Context, *types.SignedMessage) (cid.Cid, error)                                 `perm:"write"`
+		MpoolPushMessage      func(context.Context, *types.Message) (*types.SignedMessage, error)                          `perm:"sign"`
+		MpoolGetNonce         func(context.Context, address.Address) (uint64, error)                                       `perm:"read"`
+		MpoolSub              func(context.Context) (<-chan api.MpoolUpdate, error)                                        `perm:"read"`
+		MpoolEstimateGasPrice func(context.Context, uint64, address.Address, int64, types.TipSetKey) (types.BigInt, error) `perm:"read"`
+
+		MinerGetBaseInfo func(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*api.MiningBaseInfo, error) `perm:"read"`
+		MinerCreateBlock func(context.Context, *api.BlockTemplate) (*types.BlockMsg, error)                                   `perm:"write"`
+
+		WalletNew            func(context.Context, crypto.SigType) (address.Address, error)                       `perm:"write"`
+		WalletHas            func(context.Context, address.Address) (bool, error)                                 `perm:"write"`
+		WalletList           func(context.Context) ([]address.Address, error)                                     `perm:"write"`
+		WalletBalance        func(context.Context, address.Address) (types.BigInt, error)                         `perm:"read"`
+		WalletSign           func(context.Context, address.Address, []byte) (*crypto.Signature, error)            `perm:"sign"`
+		WalletSignMessage    func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"`
+		WalletVerify         func(context.Context, address.Address, []byte, *crypto.Signature) bool               `perm:"read"`
+		WalletDefaultAddress func(context.Context) (address.Address, error)                                       `perm:"write"`
+		WalletSetDefault     func(context.Context, address.Address) error                                         `perm:"admin"`
+		WalletExport         func(context.Context, address.Address) (*types.KeyInfo, error)                       `perm:"admin"`
+		WalletImport         func(context.Context, *types.KeyInfo) (address.Address, error)                       `perm:"admin"`
+		WalletDelete         func(context.Context, address.Address) error                                         `perm:"write"`
+
+		ClientImport      func(ctx context.Context, ref api.FileRef) (cid.Cid, error)                                          `perm:"admin"`
+		ClientListImports func(ctx context.Context) ([]api.Import, error)                                                      `perm:"write"`
+		ClientHasLocal    func(ctx context.Context, root cid.Cid) (bool, error)                                                `perm:"write"`
+		ClientFindData    func(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error)                                    `perm:"read"`
+		ClientStartDeal   func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error)                             `perm:"admin"`
+		ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error)                                                `perm:"read"`
+		ClientListDeals   func(ctx context.Context) ([]api.DealInfo, error)                                                    `perm:"write"`
+		ClientRetrieve    func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error                          `perm:"admin"`
+		ClientQueryAsk    func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) `perm:"read"`
+		ClientCalcCommP   func(ctx context.Context, inpath string, miner address.Address) (*api.CommPRet, error)               `perm:"read"`
+		ClientGenCar      func(ctx context.Context, ref api.FileRef, outpath string) error                                     `perm:"write"`
+
+		StateNetworkName                  func(context.Context) (dtypes.NetworkName, error)                                                                   `perm:"read"`
+		StateMinerSectors                 func(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error)        `perm:"read"`
+		StateMinerProvingSet              func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error)                             `perm:"read"`
+		StateMinerProvingDeadline         func(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error)                                `perm:"read"`
+		StateMinerPower                   func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)                                    `perm:"read"`
+		StateMinerInfo                    func(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error)                                    `perm:"read"`
+		StateMinerDeadlines               func(context.Context, address.Address, types.TipSetKey) (*miner.Deadlines, error)                                   `perm:"read"`
+		StateMinerFaults                  func(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)                                      `perm:"read"`
+		StateAllMinerFaults               func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error)                                        `perm:"read"`
+		StateMinerRecoveries              func(context.Context, address.Address, types.TipSetKey) (*abi.BitField, error)                                      `perm:"read"`
+		StateMinerInitialPledgeCollateral func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (types.BigInt, error)                     `perm:"read"`
+		StateMinerAvailableBalance        func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)                                       `perm:"read"`
+		StateSectorPreCommitInfo          func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) `perm:"read"`
+		StateSectorGetInfo                func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error)         `perm:"read"`
+		StateCall                         func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error)                                    `perm:"read"`
+		StateReplay                       func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error)                                           `perm:"read"`
+		StateGetActor                     func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error)                                       `perm:"read"`
+		StateReadState                    func(context.Context, *types.Actor, types.TipSetKey) (*api.ActorState, error)                                       `perm:"read"`
+		StatePledgeCollateral             func(context.Context, types.TipSetKey) (types.BigInt, error)                                                        `perm:"read"`
+		StateWaitMsg                      func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error)                                   `perm:"read"`
+		StateSearchMsg                    func(context.Context, cid.Cid) (*api.MsgLookup, error)                                                              `perm:"read"`
+		StateListMiners                   func(context.Context, types.TipSetKey) ([]address.Address, error)                                                   `perm:"read"`
+		StateListActors                   func(context.Context, types.TipSetKey) ([]address.Address, error)                                                   `perm:"read"`
+		StateMarketBalance                func(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error)                                  `perm:"read"`
+		StateMarketParticipants           func(context.Context, types.TipSetKey) (map[string]api.MarketBalance, error)                                        `perm:"read"`
+		StateMarketDeals                  func(context.Context, types.TipSetKey) (map[string]api.MarketDeal, error)                                           `perm:"read"`
+		StateMarketStorageDeal            func(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error)                                         `perm:"read"`
+		StateLookupID                     func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)                       `perm:"read"`
+		StateAccountKey                   func(context.Context, address.Address, types.TipSetKey) (address.Address, error)                                    `perm:"read"`
+		StateChangedActors                func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error)                                             `perm:"read"`
+		StateGetReceipt                   func(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)                                      `perm:"read"`
+		StateMinerSectorCount             func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error)                                   `perm:"read"`
+		StateListMessages                 func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error)        `perm:"read"`
+		StateCompute                      func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error)           `perm:"read"`
+
+		MsigGetAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)                                                                    `perm:"read"`
+		MsigCreate              func(context.Context, int64, []address.Address, types.BigInt, address.Address, types.BigInt) (cid.Cid, error)                                    `perm:"sign"`
+		MsigPropose             func(context.Context, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error)                          `perm:"sign"`
+		MsigApprove             func(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"`
+		MsigCancel              func(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"`
+
+		MarketEnsureAvailable func(context.Context, address.Address, address.Address, types.BigInt) (cid.Cid, error) `perm:"sign"`
+
+		PaychGet                   func(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*api.ChannelInfo, error)   `perm:"sign"`
+		PaychList                  func(context.Context) ([]address.Address, error)                                                          `perm:"read"`
+		PaychStatus                func(context.Context, address.Address) (*api.PaychStatus, error)                                          `perm:"read"`
+		PaychClose                 func(context.Context, address.Address) (cid.Cid, error)                                                   `perm:"sign"`
+		PaychAllocateLane          func(context.Context, address.Address) (uint64, error)                                                    `perm:"sign"`
+		PaychNewPayment            func(ctx context.Context, from, to address.Address, vouchers []api.VoucherSpec) (*api.PaymentInfo, error) `perm:"sign"`
+		PaychVoucherCheck          func(context.Context, *paych.SignedVoucher) error                                                         `perm:"read"`
+		PaychVoucherCheckValid     func(context.Context, address.Address, *paych.SignedVoucher) error                                        `perm:"read"`
+		PaychVoucherCheckSpendable func(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (bool, error)                `perm:"read"`
+		PaychVoucherAdd            func(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error)  `perm:"write"`
+		PaychVoucherCreate         func(context.Context, address.Address, big.Int, uint64) (*paych.SignedVoucher, error)                     `perm:"sign"`
+		PaychVoucherList           func(context.Context, address.Address) ([]*paych.SignedVoucher, error)                                    `perm:"write"`
+		PaychVoucherSubmit         func(context.Context, address.Address, *paych.SignedVoucher) (cid.Cid, error)                             `perm:"sign"`
+	}
+}
+
+func (c *FullNodeStruct) StateMinerSectorCount(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MinerSectors, error) {
+	return c.Internal.StateMinerSectorCount(ctx, addr, tsk)
+}
+
+type StorageMinerStruct struct {
+	CommonStruct
+
+	Internal struct {
+		ActorAddress    func(context.Context) (address.Address, error)                 `perm:"read"`
+		ActorSectorSize func(context.Context, address.Address) (abi.SectorSize, error) `perm:"read"`
+
+		MiningBase func(context.Context) (*types.TipSet, error) `perm:"read"`
+
+		MarketImportDealData      func(context.Context, cid.Cid, string) error                   `perm:"write"`
+		MarketListDeals           func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"`
+		MarketListIncompleteDeals func(ctx context.Context) ([]storagemarket.MinerDeal, error)   `perm:"read"`
+		MarketSetPrice            func(context.Context, types.BigInt) error                      `perm:"admin"`
+
+		PledgeSector func(context.Context) error `perm:"write"`
+
+		SectorsStatus func(context.Context, abi.SectorNumber) (api.SectorInfo, error) `perm:"read"`
+		SectorsList   func(context.Context) ([]abi.SectorNumber, error)               `perm:"read"`
+		SectorsRefs   func(context.Context) (map[string][]api.SealedRef, error)       `perm:"read"`
+		SectorsUpdate func(context.Context, abi.SectorNumber, api.SectorState) error  `perm:"write"`
+
+		WorkerConnect func(context.Context, string) error                             `perm:"admin"` // TODO: worker perm
+		WorkerStats   func(context.Context) (map[uint64]storiface.WorkerStats, error) `perm:"admin"`
+
+		StorageList          func(context.Context) (map[stores.ID][]stores.Decl, error)                                                                                `perm:"admin"`
+		StorageLocal         func(context.Context) (map[stores.ID]string, error)                                                                                       `perm:"admin"`
+		StorageStat          func(context.Context, stores.ID) (stores.FsStat, error)                                                                                   `perm:"admin"`
+		StorageAttach        func(context.Context, stores.StorageInfo, stores.FsStat) error                                                                            `perm:"admin"`
+		StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType, bool) error                                                         `perm:"admin"`
+		StorageDropSector    func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType) error                                                               `perm:"admin"`
+		StorageFindSector    func(context.Context, abi.SectorID, stores.SectorFileType, bool) ([]stores.SectorStorageInfo, error)                                      `perm:"admin"`
+		StorageInfo          func(context.Context, stores.ID) (stores.StorageInfo, error)                                                                              `perm:"admin"`
+		StorageBestAlloc     func(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredProof, sealing stores.PathType) ([]stores.StorageInfo, error) `perm:"admin"`
+		StorageReportHealth  func(ctx context.Context, id stores.ID, report stores.HealthReport) error                                                                 `perm:"admin"`
+		StorageLock          func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error                             `perm:"admin"`
+
+		DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"`
+		DealsList       func(ctx context.Context) ([]storagemarket.StorageDeal, error)    `perm:"read"`
+
+		StorageAddLocal func(ctx context.Context, path string) error `perm:"admin"`
+	}
+}
+
+type WorkerStruct struct {
+	Internal struct {
+		// TODO: lower perms
+
+		Version func(context.Context) (build.Version, error) `perm:"admin"`
+
+		TaskTypes func(context.Context) (map[sealtasks.TaskType]struct{}, error) `perm:"admin"`
+		Paths     func(context.Context) ([]stores.StoragePath, error)            `perm:"admin"`
+		Info      func(context.Context) (storiface.WorkerInfo, error)            `perm:"admin"`
+
+		SealPreCommit1 func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error)                                                           `perm:"admin"`
+		SealPreCommit2 func(context.Context, abi.SectorID, storage.PreCommit1Out) (cids storage.SectorCids, err error)                                                                                            `perm:"admin"`
+		SealCommit1    func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) `perm:"admin"`
+		SealCommit2    func(context.Context, abi.SectorID, storage.Commit1Out) (storage.Proof, error)                                                                                                             `perm:"admin"`
+		FinalizeSector func(context.Context, abi.SectorID) error                                                                                                                                                  `perm:"admin"`
+		MoveStorage    func(ctx context.Context, sector abi.SectorID) error                                                                                                                                       `perm:"admin"`
+
+		UnsealPiece func(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error `perm:"admin"`
+		ReadPiece   func(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) error                   `perm:"admin"`
+
+		Fetch func(context.Context, abi.SectorID, stores.SectorFileType, stores.PathType, stores.AcquireMode) error `perm:"admin"`
+
+		Closing func(context.Context) (<-chan struct{}, error) `perm:"admin"`
+	}
+}
+
+// CommonStruct
+
+func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]auth.Permission, error) {
+	return c.Internal.AuthVerify(ctx, token)
+}
+
+func (c *CommonStruct) AuthNew(ctx context.Context, perms []auth.Permission) ([]byte, error) {
+	return c.Internal.AuthNew(ctx, perms)
+}
+
+func (c *CommonStruct) NetPubsubScores(ctx context.Context) ([]api.PubsubScore, error) {
+	return c.Internal.NetPubsubScores(ctx)
+}
+func (c *CommonStruct) NetConnectedness(ctx context.Context, pid peer.ID) (network.Connectedness, error) {
+	return c.Internal.NetConnectedness(ctx, pid)
+}
+
+func (c *CommonStruct) NetPeers(ctx context.Context) ([]peer.AddrInfo, error) {
+	return c.Internal.NetPeers(ctx)
+}
+
+func (c *CommonStruct) NetConnect(ctx context.Context, p peer.AddrInfo) error {
+	return c.Internal.NetConnect(ctx, p)
+}
+
+func (c *CommonStruct) NetAddrsListen(ctx context.Context) (peer.AddrInfo, error) {
+	return c.Internal.NetAddrsListen(ctx)
+}
+
+func (c *CommonStruct) NetDisconnect(ctx context.Context, p peer.ID) error {
+	return c.Internal.NetDisconnect(ctx, p)
+}
+
+func (c *CommonStruct) NetFindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo, error) {
+	return c.Internal.NetFindPeer(ctx, p)
+}
+
+// ID implements API.ID
+func (c *CommonStruct) ID(ctx context.Context) (peer.ID, error) {
+	return c.Internal.ID(ctx)
+}
+
+// Version implements API.Version
+func (c *CommonStruct) Version(ctx context.Context) (api.Version, error) {
+	return c.Internal.Version(ctx)
+}
+
+func (c *CommonStruct) LogList(ctx context.Context) ([]string, error) {
+	return c.Internal.LogList(ctx)
+}
+
+func (c *CommonStruct) LogSetLevel(ctx context.Context, group, level string) error {
+	return c.Internal.LogSetLevel(ctx, group, level)
+}
+
+func (c *CommonStruct) Shutdown(ctx context.Context) error {
+	return c.Internal.Shutdown(ctx)
+}
+
+// FullNodeStruct
+
+func (c *FullNodeStruct) ClientListImports(ctx context.Context) ([]api.Import, error) {
+	return c.Internal.ClientListImports(ctx)
+}
+
+func (c *FullNodeStruct) ClientImport(ctx context.Context, ref api.FileRef) (cid.Cid, error) {
+	return c.Internal.ClientImport(ctx, ref)
+}
+
+func (c *FullNodeStruct) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) {
+	return c.Internal.ClientHasLocal(ctx, root)
+}
+
+func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error) {
+	return c.Internal.ClientFindData(ctx, root)
+}
+
+func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) {
+	return c.Internal.ClientStartDeal(ctx, params)
+}
+func (c *FullNodeStruct) ClientGetDealInfo(ctx context.Context, deal cid.Cid) (*api.DealInfo, error) {
+	return c.Internal.ClientGetDealInfo(ctx, deal)
+}
+
+func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) {
+	return c.Internal.ClientListDeals(ctx)
+}
+
+func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error {
+	return c.Internal.ClientRetrieve(ctx, order, ref)
+}
+
+func (c *FullNodeStruct) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) {
+	return c.Internal.ClientQueryAsk(ctx, p, miner)
+}
+func (c *FullNodeStruct) ClientCalcCommP(ctx context.Context, inpath string, miner address.Address) (*api.CommPRet, error) {
+	return c.Internal.ClientCalcCommP(ctx, inpath, miner)
+}
+
+func (c *FullNodeStruct) ClientGenCar(ctx context.Context, ref api.FileRef, outpath string) error {
+	return c.Internal.ClientGenCar(ctx, ref, outpath)
+}
+
+func (c *FullNodeStruct) MpoolPending(ctx context.Context, tsk types.TipSetKey) ([]*types.SignedMessage, error) {
+	return c.Internal.MpoolPending(ctx, tsk)
+}
+
+func (c *FullNodeStruct) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) {
+	return c.Internal.MpoolPush(ctx, smsg)
+}
+
+func (c *FullNodeStruct) MpoolPushMessage(ctx context.Context, msg *types.Message) (*types.SignedMessage, error) {
+	return c.Internal.MpoolPushMessage(ctx, msg)
+}
+
+func (c *FullNodeStruct) MpoolSub(ctx context.Context) (<-chan api.MpoolUpdate, error) {
+	return c.Internal.MpoolSub(ctx)
+}
+
+func (c *FullNodeStruct) MpoolEstimateGasPrice(ctx context.Context, nblocksincl uint64, sender address.Address, limit int64, tsk types.TipSetKey) (types.BigInt, error) {
+	return c.Internal.MpoolEstimateGasPrice(ctx, nblocksincl, sender, limit, tsk)
+}
+
+func (c *FullNodeStruct) MinerGetBaseInfo(ctx context.Context, maddr address.Address, epoch abi.ChainEpoch, tsk types.TipSetKey) (*api.MiningBaseInfo, error) {
+	return c.Internal.MinerGetBaseInfo(ctx, maddr, epoch, tsk)
+}
+
+func (c *FullNodeStruct) MinerCreateBlock(ctx context.Context, bt *api.BlockTemplate) (*types.BlockMsg, error) {
+	return c.Internal.MinerCreateBlock(ctx, bt)
+}
+
+func (c *FullNodeStruct) ChainHead(ctx context.Context) (*types.TipSet, error) {
+	return c.Internal.ChainHead(ctx)
+}
+
+func (c *FullNodeStruct) ChainGetRandomness(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
+	return c.Internal.ChainGetRandomness(ctx, tsk, personalization, randEpoch, entropy)
+}
+
+func (c *FullNodeStruct) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) {
+	return c.Internal.ChainGetTipSetByHeight(ctx, h, tsk)
+}
+
+func (c *FullNodeStruct) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) {
+	return c.Internal.WalletNew(ctx, typ)
+}
+
+func (c *FullNodeStruct) WalletHas(ctx context.Context, addr address.Address) (bool, error) {
+	return c.Internal.WalletHas(ctx, addr)
+}
+
+func (c *FullNodeStruct) WalletList(ctx context.Context) ([]address.Address, error) {
+	return c.Internal.WalletList(ctx)
+}
+
+func (c *FullNodeStruct) WalletBalance(ctx context.Context, a address.Address) (types.BigInt, error) {
+	return c.Internal.WalletBalance(ctx, a)
+}
+
+func (c *FullNodeStruct) WalletSign(ctx context.Context, k address.Address, msg []byte) (*crypto.Signature, error) {
+	return c.Internal.WalletSign(ctx, k, msg)
+}
+
+func (c *FullNodeStruct) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) {
+	return c.Internal.WalletSignMessage(ctx, k, msg)
+}
+
+func (c *FullNodeStruct) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) bool {
+	return c.Internal.WalletVerify(ctx, k, msg, sig)
+}
+
+func (c *FullNodeStruct) WalletDefaultAddress(ctx context.Context) (address.Address, error) {
+	return c.Internal.WalletDefaultAddress(ctx)
+}
+
+func (c *FullNodeStruct) WalletSetDefault(ctx context.Context, a address.Address) error {
+	return c.Internal.WalletSetDefault(ctx, a)
+}
+
+func (c *FullNodeStruct) WalletExport(ctx context.Context, a address.Address) (*types.KeyInfo, error) {
+	return c.Internal.WalletExport(ctx, a)
+}
+
+func (c *FullNodeStruct) WalletImport(ctx context.Context, ki *types.KeyInfo) (address.Address, error) {
+	return c.Internal.WalletImport(ctx, ki)
+}
+
+func (c *FullNodeStruct) WalletDelete(ctx context.Context, addr address.Address) error {
+	return c.Internal.WalletDelete(ctx, addr)
+}
+
+func (c *FullNodeStruct) MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) {
+	return c.Internal.MpoolGetNonce(ctx, addr)
+}
+
+func (c *FullNodeStruct) ChainGetBlock(ctx context.Context, b cid.Cid) (*types.BlockHeader, error) {
+	return c.Internal.ChainGetBlock(ctx, b)
+}
+
+func (c *FullNodeStruct) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) {
+	return c.Internal.ChainGetTipSet(ctx, key)
+}
+
+func (c *FullNodeStruct) ChainGetBlockMessages(ctx context.Context, b cid.Cid) (*api.BlockMessages, error) {
+	return c.Internal.ChainGetBlockMessages(ctx, b)
+}
+
+func (c *FullNodeStruct) ChainGetParentReceipts(ctx context.Context, b cid.Cid) ([]*types.MessageReceipt, error) {
+	return c.Internal.ChainGetParentReceipts(ctx, b)
+}
+
+func (c *FullNodeStruct) ChainGetParentMessages(ctx context.Context, b cid.Cid) ([]api.Message, error) {
+	return c.Internal.ChainGetParentMessages(ctx, b)
+}
+
+func (c *FullNodeStruct) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, error) {
+	return c.Internal.ChainNotify(ctx)
+}
+
+func (c *FullNodeStruct) ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error) {
+	return c.Internal.ChainReadObj(ctx, obj)
+}
+
+func (c *FullNodeStruct) ChainHasObj(ctx context.Context, o cid.Cid) (bool, error) {
+	return c.Internal.ChainHasObj(ctx, o)
+}
+
+func (c *FullNodeStruct) ChainStatObj(ctx context.Context, obj, base cid.Cid) (api.ObjStat, error) {
+	return c.Internal.ChainStatObj(ctx, obj, base)
+}
+
+func (c *FullNodeStruct) ChainSetHead(ctx context.Context, tsk types.TipSetKey) error {
+	return c.Internal.ChainSetHead(ctx, tsk)
+}
+
+func (c *FullNodeStruct) ChainGetGenesis(ctx context.Context) (*types.TipSet, error) {
+	return c.Internal.ChainGetGenesis(ctx)
+}
+
+func (c *FullNodeStruct) ChainTipSetWeight(ctx context.Context, tsk types.TipSetKey) (types.BigInt, error) {
+	return c.Internal.ChainTipSetWeight(ctx, tsk)
+}
+
+func (c *FullNodeStruct) ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error) {
+	return c.Internal.ChainGetNode(ctx, p)
+}
+
+func (c *FullNodeStruct) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) {
+	return c.Internal.ChainGetMessage(ctx, mc)
+}
+
+func (c *FullNodeStruct) ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*api.HeadChange, error) {
+	return c.Internal.ChainGetPath(ctx, from, to)
+}
+
+func (c *FullNodeStruct) ChainExport(ctx context.Context, tsk types.TipSetKey) (<-chan []byte, error) {
+	return c.Internal.ChainExport(ctx, tsk)
+}
+
+func (c *FullNodeStruct) SyncState(ctx context.Context) (*api.SyncState, error) {
+	return c.Internal.SyncState(ctx)
+}
+
+func (c *FullNodeStruct) SyncSubmitBlock(ctx context.Context, blk *types.BlockMsg) error {
+	return c.Internal.SyncSubmitBlock(ctx, blk)
+}
+
+func (c *FullNodeStruct) SyncIncomingBlocks(ctx context.Context) (<-chan *types.BlockHeader, error) {
+	return c.Internal.SyncIncomingBlocks(ctx)
+}
+
+func (c *FullNodeStruct) SyncMarkBad(ctx context.Context, bcid cid.Cid) error {
+	return c.Internal.SyncMarkBad(ctx, bcid)
+}
+
+func (c *FullNodeStruct) SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error) {
+	return c.Internal.SyncCheckBad(ctx, bcid)
+}
+
+func (c *FullNodeStruct) StateNetworkName(ctx context.Context) (dtypes.NetworkName, error) {
+	return c.Internal.StateNetworkName(ctx)
+}
+
+func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, filter *abi.BitField, filterOut bool, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) {
+	return c.Internal.StateMinerSectors(ctx, addr, filter, filterOut, tsk)
+}
+
+func (c *FullNodeStruct) StateMinerProvingSet(ctx context.Context, addr address.Address, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) {
+	return c.Internal.StateMinerProvingSet(ctx, addr, tsk)
+}
+
+func (c *FullNodeStruct) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*miner.DeadlineInfo, error) {
+	return c.Internal.StateMinerProvingDeadline(ctx, addr, tsk)
+}
+
+func (c *FullNodeStruct) StateMinerPower(ctx context.Context, a address.Address, tsk types.TipSetKey) (*api.MinerPower, error) {
+	return c.Internal.StateMinerPower(ctx, a, tsk)
+}
+
+func (c *FullNodeStruct) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) {
+	return c.Internal.StateMinerInfo(ctx, actor, tsk)
+}
+
+func (c *FullNodeStruct) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) (*miner.Deadlines, error) {
+	return c.Internal.StateMinerDeadlines(ctx, m, tsk)
+}
+
+func (c *FullNodeStruct) StateMinerFaults(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*abi.BitField, error) {
+	return c.Internal.StateMinerFaults(ctx, actor, tsk)
+}
+
+func (c *FullNodeStruct) StateAllMinerFaults(ctx context.Context, cutoff abi.ChainEpoch, endTsk types.TipSetKey) ([]*api.Fault, error) {
+	return c.Internal.StateAllMinerFaults(ctx, cutoff, endTsk)
+}
+
+func (c *FullNodeStruct) StateMinerRecoveries(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*abi.BitField, error) {
+	return c.Internal.StateMinerRecoveries(ctx, actor, tsk)
+}
+
+func (c *FullNodeStruct) StateMinerInitialPledgeCollateral(ctx context.Context, maddr address.Address, snum abi.SectorNumber, tsk types.TipSetKey) (types.BigInt, error) {
+	return c.Internal.StateMinerInitialPledgeCollateral(ctx, maddr, snum, tsk)
+}
+
+func (c *FullNodeStruct) StateMinerAvailableBalance(ctx context.Context, maddr address.Address, tsk types.TipSetKey) (types.BigInt, error) {
+	return c.Internal.StateMinerAvailableBalance(ctx, maddr, tsk)
+}
+
+func (c *FullNodeStruct) StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) {
+	return c.Internal.StateSectorPreCommitInfo(ctx, maddr, n, tsk)
+}
+
+func (c *FullNodeStruct) StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error) {
+	return c.Internal.StateSectorGetInfo(ctx, maddr, n, tsk)
+}
+
+func (c *FullNodeStruct) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) {
+	return c.Internal.StateCall(ctx, msg, tsk)
+}
+
+func (c *FullNodeStruct) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) {
+	return c.Internal.StateReplay(ctx, tsk, mc)
+}
+
+func (c *FullNodeStruct) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) {
+	return c.Internal.StateGetActor(ctx, actor, tsk)
+}
+
+func (c *FullNodeStruct) StateReadState(ctx context.Context, act *types.Actor, tsk types.TipSetKey) (*api.ActorState, error) {
+	return c.Internal.StateReadState(ctx, act, tsk)
+}
+
+func (c *FullNodeStruct) StatePledgeCollateral(ctx context.Context, tsk types.TipSetKey) (types.BigInt, error) {
+	return c.Internal.StatePledgeCollateral(ctx, tsk)
+}
+
+func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confidence uint64) (*api.MsgLookup, error) {
+	return c.Internal.StateWaitMsg(ctx, msgc, confidence)
+}
+
+func (c *FullNodeStruct) StateSearchMsg(ctx context.Context, msgc cid.Cid) (*api.MsgLookup, error) {
+	return c.Internal.StateSearchMsg(ctx, msgc)
+}
+
+func (c *FullNodeStruct) StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) {
+	return c.Internal.StateListMiners(ctx, tsk)
+}
+
+func (c *FullNodeStruct) StateListActors(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) {
+	return c.Internal.StateListActors(ctx, tsk)
+}
+
+func (c *FullNodeStruct) StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error) {
+	return c.Internal.StateMarketBalance(ctx, addr, tsk)
+}
+
+func (c *FullNodeStruct) StateMarketParticipants(ctx context.Context, tsk types.TipSetKey) (map[string]api.MarketBalance, error) {
+	return c.Internal.StateMarketParticipants(ctx, tsk)
+}
+
+func (c *FullNodeStruct) StateMarketDeals(ctx context.Context, tsk types.TipSetKey) (map[string]api.MarketDeal, error) {
+	return c.Internal.StateMarketDeals(ctx, tsk)
+}
+
+func (c *FullNodeStruct) StateMarketStorageDeal(ctx context.Context, dealid abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) {
+	return c.Internal.StateMarketStorageDeal(ctx, dealid, tsk)
+}
+
+func (c *FullNodeStruct) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) {
+	return c.Internal.StateLookupID(ctx, addr, tsk)
+}
+
+func (c *FullNodeStruct) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) {
+	return c.Internal.StateAccountKey(ctx, addr, tsk)
+}
+
+func (c *FullNodeStruct) StateChangedActors(ctx context.Context, olnstate cid.Cid, newstate cid.Cid) (map[string]types.Actor, error) {
+	return c.Internal.StateChangedActors(ctx, olnstate, newstate)
+}
+
+func (c *FullNodeStruct) StateGetReceipt(ctx context.Context, msg cid.Cid, tsk types.TipSetKey) (*types.MessageReceipt, error) {
+	return c.Internal.StateGetReceipt(ctx, msg, tsk)
+}
+
+func (c *FullNodeStruct) StateListMessages(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) {
+	return c.Internal.StateListMessages(ctx, match, tsk, toht)
+}
+
+func (c *FullNodeStruct) StateCompute(ctx context.Context, height abi.ChainEpoch, msgs []*types.Message, tsk types.TipSetKey) (*api.ComputeStateOutput, error) {
+	return c.Internal.StateCompute(ctx, height, msgs, tsk)
+}
+
+func (c *FullNodeStruct) MsigGetAvailableBalance(ctx context.Context, a address.Address, tsk types.TipSetKey) (types.BigInt, error) {
+	return c.Internal.MsigGetAvailableBalance(ctx, a, tsk)
+}
+
+func (c *FullNodeStruct) MsigCreate(ctx context.Context, req int64, addrs []address.Address, val types.BigInt, src address.Address, gp types.BigInt) (cid.Cid, error) {
+	return c.Internal.MsigCreate(ctx, req, addrs, val, src, gp)
+}
+
+func (c *FullNodeStruct) MsigPropose(ctx context.Context, msig address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) {
+	return c.Internal.MsigPropose(ctx, msig, to, amt, src, method, params)
+}
+
+func (c *FullNodeStruct) MsigApprove(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) {
+	return c.Internal.MsigApprove(ctx, msig, txID, proposer, to, amt, src, method, params)
+}
+
+func (c *FullNodeStruct) MsigCancel(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) {
+	return c.Internal.MsigCancel(ctx, msig, txID, proposer, to, amt, src, method, params)
+}
+
+func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr, wallet address.Address, amt types.BigInt) (cid.Cid, error) {
+	return c.Internal.MarketEnsureAvailable(ctx, addr, wallet, amt)
+}
+
+func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*api.ChannelInfo, error) {
+	return c.Internal.PaychGet(ctx, from, to, ensureFunds)
+}
+
+func (c *FullNodeStruct) PaychList(ctx context.Context) ([]address.Address, error) {
+	return c.Internal.PaychList(ctx)
+}
+
+func (c *FullNodeStruct) PaychStatus(ctx context.Context, pch address.Address) (*api.PaychStatus, error) {
+	return c.Internal.PaychStatus(ctx, pch)
+}
+
+func (c *FullNodeStruct) PaychVoucherCheckValid(ctx context.Context, addr address.Address, sv *paych.SignedVoucher) error {
+	return c.Internal.PaychVoucherCheckValid(ctx, addr, sv)
+}
+
+func (c *FullNodeStruct) PaychVoucherCheckSpendable(ctx context.Context, addr address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (bool, error) {
+	return c.Internal.PaychVoucherCheckSpendable(ctx, addr, sv, secret, proof)
+}
+
+func (c *FullNodeStruct) PaychVoucherAdd(ctx context.Context, addr address.Address, sv *paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) {
+	return c.Internal.PaychVoucherAdd(ctx, addr, sv, proof, minDelta)
+}
+
+func (c *FullNodeStruct) PaychVoucherCreate(ctx context.Context, pch address.Address, amt types.BigInt, lane uint64) (*paych.SignedVoucher, error) {
+	return c.Internal.PaychVoucherCreate(ctx, pch, amt, lane)
+}
+
+func (c *FullNodeStruct) PaychVoucherList(ctx context.Context, pch address.Address) ([]*paych.SignedVoucher, error) {
+	return c.Internal.PaychVoucherList(ctx, pch)
+}
+
+func (c *FullNodeStruct) PaychClose(ctx context.Context, a address.Address) (cid.Cid, error) {
+	return c.Internal.PaychClose(ctx, a)
+}
+
+func (c *FullNodeStruct) PaychAllocateLane(ctx context.Context, ch address.Address) (uint64, error) {
+	return c.Internal.PaychAllocateLane(ctx, ch)
+}
+
+func (c *FullNodeStruct) PaychNewPayment(ctx context.Context, from, to address.Address, vouchers []api.VoucherSpec) (*api.PaymentInfo, error) {
+	return c.Internal.PaychNewPayment(ctx, from, to, vouchers)
+}
+
+func (c *FullNodeStruct) PaychVoucherSubmit(ctx context.Context, ch address.Address, sv *paych.SignedVoucher) (cid.Cid, error) {
+	return c.Internal.PaychVoucherSubmit(ctx, ch, sv)
+}
+
+// StorageMinerStruct
+
+func (c *StorageMinerStruct) ActorAddress(ctx context.Context) (address.Address, error) {
+	return c.Internal.ActorAddress(ctx)
+}
+
+func (c *StorageMinerStruct) MiningBase(ctx context.Context) (*types.TipSet, error) {
+	return c.Internal.MiningBase(ctx)
+}
+
+func (c *StorageMinerStruct) ActorSectorSize(ctx context.Context, addr address.Address) (abi.SectorSize, error) {
+	return c.Internal.ActorSectorSize(ctx, addr)
+}
+
+func (c *StorageMinerStruct) PledgeSector(ctx context.Context) error {
+	return c.Internal.PledgeSector(ctx)
+}
+
+// Get the status of a given sector by ID
+func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid abi.SectorNumber) (api.SectorInfo, error) {
+	return c.Internal.SectorsStatus(ctx, sid)
+}
+
+// List all staged sectors
+func (c *StorageMinerStruct) SectorsList(ctx context.Context) ([]abi.SectorNumber, error) {
+	return c.Internal.SectorsList(ctx)
+}
+
+func (c *StorageMinerStruct) SectorsRefs(ctx context.Context) (map[string][]api.SealedRef, error) {
+	return c.Internal.SectorsRefs(ctx)
+}
+
+func (c *StorageMinerStruct) SectorsUpdate(ctx context.Context, id abi.SectorNumber, state api.SectorState) error {
+	return c.Internal.SectorsUpdate(ctx, id, state)
+}
+
+func (c *StorageMinerStruct) WorkerConnect(ctx context.Context, url string) error {
+	return c.Internal.WorkerConnect(ctx, url)
+}
+
+func (c *StorageMinerStruct) WorkerStats(ctx context.Context) (map[uint64]storiface.WorkerStats, error) {
+	return c.Internal.WorkerStats(ctx)
+}
+
+func (c *StorageMinerStruct) StorageAttach(ctx context.Context, si stores.StorageInfo, st stores.FsStat) error {
+	return c.Internal.StorageAttach(ctx, si, st)
+}
+
+func (c *StorageMinerStruct) StorageDeclareSector(ctx context.Context, storageId stores.ID, s abi.SectorID, ft stores.SectorFileType, primary bool) error {
+	return c.Internal.StorageDeclareSector(ctx, storageId, s, ft, primary)
+}
+
+func (c *StorageMinerStruct) StorageDropSector(ctx context.Context, storageId stores.ID, s abi.SectorID, ft stores.SectorFileType) error {
+	return c.Internal.StorageDropSector(ctx, storageId, s, ft)
+}
+
+func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types stores.SectorFileType, allowFetch bool) ([]stores.SectorStorageInfo, error) {
+	return c.Internal.StorageFindSector(ctx, si, types, allowFetch)
+}
+
+func (c *StorageMinerStruct) StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) {
+	return c.Internal.StorageList(ctx)
+}
+
+func (c *StorageMinerStruct) StorageLocal(ctx context.Context) (map[stores.ID]string, error) {
+	return c.Internal.StorageLocal(ctx)
+}
+
+func (c *StorageMinerStruct) StorageStat(ctx context.Context, id stores.ID) (stores.FsStat, error) {
+	return c.Internal.StorageStat(ctx, id)
+}
+
+func (c *StorageMinerStruct) StorageInfo(ctx context.Context, id stores.ID) (stores.StorageInfo, error) {
+	return c.Internal.StorageInfo(ctx, id)
+}
+
+func (c *StorageMinerStruct) StorageBestAlloc(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredProof, pt stores.PathType) ([]stores.StorageInfo, error) {
+	return c.Internal.StorageBestAlloc(ctx, allocate, spt, pt)
+}
+
+func (c *StorageMinerStruct) StorageReportHealth(ctx context.Context, id stores.ID, report stores.HealthReport) error {
+	return c.Internal.StorageReportHealth(ctx, id, report)
+}
+
+func (c *StorageMinerStruct) StorageLock(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error {
+	return c.Internal.StorageLock(ctx, sector, read, write)
+}
+
+func (c *StorageMinerStruct) MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error {
+	return c.Internal.MarketImportDealData(ctx, propcid, path)
+}
+
+func (c *StorageMinerStruct) MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error) {
+	return c.Internal.MarketListDeals(ctx)
+}
+
+func (c *StorageMinerStruct) MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error) {
+	return c.Internal.MarketListIncompleteDeals(ctx)
+}
+
+func (c *StorageMinerStruct) MarketSetPrice(ctx context.Context, p types.BigInt) error {
+	return c.Internal.MarketSetPrice(ctx, p)
+}
+
+func (c *StorageMinerStruct) DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error {
+	return c.Internal.DealsImportData(ctx, dealPropCid, file)
+}
+
+func (c *StorageMinerStruct) DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) {
+	return c.Internal.DealsList(ctx)
+}
+
+func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) error {
+	return c.Internal.StorageAddLocal(ctx, path)
+}
+
+// WorkerStruct
+
+func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) {
+	return w.Internal.Version(ctx)
+}
+
+func (w *WorkerStruct) TaskTypes(ctx context.Context) (map[sealtasks.TaskType]struct{}, error) {
+	return w.Internal.TaskTypes(ctx)
+}
+
+func (w *WorkerStruct) Paths(ctx context.Context) ([]stores.StoragePath, error) {
+	return w.Internal.Paths(ctx)
+}
+
+func (w *WorkerStruct) Info(ctx context.Context) (storiface.WorkerInfo, error) {
+	return w.Internal.Info(ctx)
+}
+
+func (w *WorkerStruct) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) {
+	return w.Internal.SealPreCommit1(ctx, sector, ticket, pieces)
+}
+
+func (w *WorkerStruct) SealPreCommit2(ctx context.Context, sector abi.SectorID, p1o storage.PreCommit1Out) (storage.SectorCids, error) {
+	return w.Internal.SealPreCommit2(ctx, sector, p1o)
+}
+
+func (w *WorkerStruct) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) {
+	return w.Internal.SealCommit1(ctx, sector, ticket, seed, pieces, cids)
+}
+
+func (w *WorkerStruct) SealCommit2(ctx context.Context, sector abi.SectorID, c1o storage.Commit1Out) (storage.Proof, error) {
+	return w.Internal.SealCommit2(ctx, sector, c1o)
+}
+
+func (w *WorkerStruct) FinalizeSector(ctx context.Context, sector abi.SectorID) error {
+	return w.Internal.FinalizeSector(ctx, sector)
+}
+
+func (w *WorkerStruct) MoveStorage(ctx context.Context, sector abi.SectorID) error {
+	return w.Internal.MoveStorage(ctx, sector)
+}
+
+func (w *WorkerStruct) UnsealPiece(ctx context.Context, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, c cid.Cid) error {
+	return w.Internal.UnsealPiece(ctx, id, index, size, randomness, c)
+}
+
+func (w *WorkerStruct) ReadPiece(ctx context.Context, writer io.Writer, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) error {
+	return w.Internal.ReadPiece(ctx, writer, id, index, size)
+}
+
+func (w *WorkerStruct) Fetch(ctx context.Context, id abi.SectorID, fileType stores.SectorFileType, ptype stores.PathType, am stores.AcquireMode) error {
+	return w.Internal.Fetch(ctx, id, fileType, ptype, am)
+}
+
+func (w *WorkerStruct) Closing(ctx context.Context) (<-chan struct{}, error) {
+	return w.Internal.Closing(ctx)
+}
+
+var _ api.Common = &CommonStruct{}
+var _ api.FullNode = &FullNodeStruct{}
+var _ api.StorageMiner = &StorageMinerStruct{}
+var _ api.WorkerAPI = &WorkerStruct{}

api/apistruct/struct_test.go

@@ -0,0 +1,9 @@
+package apistruct
+
+import "testing"
+
+func TestPermTags(t *testing.T) {
+	_ = PermissionedFullAPI(&FullNodeStruct{})
+	_ = PermissionedStorMinerAPI(&StorageMinerStruct{})
+	_ = PermissionedWorkerAPI(&WorkerStruct{})
+}

api/cbor_gen.go

@@ -5,19 +5,14 @@ package api
 import (
 	"fmt"
 	"io"
-	"sort"
 
-	abi "github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi"
-	market "github.com/filecoin-project/specs-actors/actors/builtin/market"
+	"github.com/filecoin-project/specs-actors/actors/builtin/paych"
-	paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
-	cid "github.com/ipfs/go-cid"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	xerrors "golang.org/x/xerrors"
 )
 
 var _ = xerrors.Errorf
-var _ = cid.Undef
-var _ = sort.Sort
 
 func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	if t == nil {
@@ -38,7 +33,7 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Channel"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Channel")); err != nil {
+	if _, err := io.WriteString(w, "Channel"); err != nil {
 		return err
 	}
 
@@ -46,20 +41,26 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	// t.WaitSentinel (cid.Cid) (struct)
+	// t.ChannelMessage (cid.Cid) (struct)
-	if len("WaitSentinel") > cbg.MaxLength {
+	if len("ChannelMessage") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"WaitSentinel\" was too long")
+		return xerrors.Errorf("Value in field \"ChannelMessage\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("WaitSentinel"))); err != nil {
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ChannelMessage"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("WaitSentinel")); err != nil {
+	if _, err := io.WriteString(w, "ChannelMessage"); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteCidBuf(scratch, w, t.WaitSentinel); err != nil {
+	if t.ChannelMessage == nil {
-		return xerrors.Errorf("failed to write cid field t.WaitSentinel: %w", err)
+		if _, err := w.Write(cbg.CborNull); err != nil {
+			return err
+		}
+	} else {
+		if err := cbg.WriteCidBuf(scratch, w, *t.ChannelMessage); err != nil {
+			return xerrors.Errorf("failed to write cid field t.ChannelMessage: %w", err)
+		}
 	}
 
 	// t.Vouchers ([]*paych.SignedVoucher) (slice)
@@ -70,7 +71,7 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Vouchers"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Vouchers")); err != nil {
+	if _, err := io.WriteString(w, "Vouchers"); err != nil {
 		return err
 	}
 
@@ -90,8 +91,6 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 }
 
 func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
-	*t = PaymentInfo{}
-
 	br := cbg.GetPeeker(r)
 	scratch := make([]byte, 8)
 
@@ -132,17 +131,29 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 				}
 
 			}
-			// t.WaitSentinel (cid.Cid) (struct)
+			// t.ChannelMessage (cid.Cid) (struct)
-		case "WaitSentinel":
+		case "ChannelMessage":
 
 			{
 
-				c, err := cbg.ReadCid(br)
+				pb, err := br.PeekByte()
 				if err != nil {
-					return xerrors.Errorf("failed to read cid field t.WaitSentinel: %w", err)
+					return err
 				}
+				if pb == cbg.CborNull[0] {
+					var nbuf [1]byte
+					if _, err := br.Read(nbuf[:]); err != nil {
+						return err
+					}
+				} else {
 
-				t.WaitSentinel = c
+					c, err := cbg.ReadCid(br)
+					if err != nil {
+						return xerrors.Errorf("failed to read cid field t.ChannelMessage: %w", err)
+					}
+
+					t.ChannelMessage = &c
+				}
 
 			}
 			// t.Vouchers ([]*paych.SignedVoucher) (slice)
@@ -176,8 +187,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			// Field doesn't exist on this type, so ignore it
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
-			cbg.ScanForLinks(r, func(cid.Cid) {})
 		}
 	}
 
@@ -202,7 +212,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SectorID"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("SectorID")); err != nil {
+	if _, err := io.WriteString(w, "SectorID"); err != nil {
 		return err
 	}
 
@@ -210,7 +220,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	// t.Offset (abi.PaddedPieceSize) (uint64)
+	// t.Offset (uint64) (uint64)
 	if len("Offset") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"Offset\" was too long")
 	}
@@ -218,7 +228,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Offset"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Offset")); err != nil {
+	if _, err := io.WriteString(w, "Offset"); err != nil {
 		return err
 	}
 
@@ -234,7 +244,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Size"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Size")); err != nil {
+	if _, err := io.WriteString(w, "Size"); err != nil {
 		return err
 	}
 
@@ -246,8 +256,6 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 }
 
 func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
-	*t = SealedRef{}
-
 	br := cbg.GetPeeker(r)
 	scratch := make([]byte, 8)
 
@@ -293,7 +301,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 				t.SectorID = abi.SectorNumber(extra)
 
 			}
-			// t.Offset (abi.PaddedPieceSize) (uint64)
+			// t.Offset (uint64) (uint64)
 		case "Offset":
 
 			{
@@ -305,7 +313,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 				if maj != cbg.MajUnsignedInt {
 					return fmt.Errorf("wrong type for uint64 field")
 				}
-				t.Offset = abi.PaddedPieceSize(extra)
+				t.Offset = uint64(extra)
 
 			}
 			// t.Size (abi.UnpaddedPieceSize) (uint64)
@@ -325,8 +333,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			// Field doesn't exist on this type, so ignore it
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
-			cbg.ScanForLinks(r, func(cid.Cid) {})
 		}
 	}
 
@@ -351,7 +358,7 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Refs"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Refs")); err != nil {
+	if _, err := io.WriteString(w, "Refs"); err != nil {
 		return err
 	}
 
@@ -371,8 +378,6 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
 }
 
 func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
-	*t = SealedRefs{}
-
 	br := cbg.GetPeeker(r)
 	scratch := make([]byte, 8)
 
@@ -434,8 +439,7 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			// Field doesn't exist on this type, so ignore it
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
-			cbg.ScanForLinks(r, func(cid.Cid) {})
 		}
 	}
 
@@ -460,7 +464,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Value")); err != nil {
+	if _, err := io.WriteString(w, "Value"); err != nil {
 		return err
 	}
 
@@ -472,7 +476,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	if _, err := w.Write(t.Value[:]); err != nil {
+	if _, err := w.Write(t.Value); err != nil {
 		return err
 	}
 
@@ -484,7 +488,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Epoch")); err != nil {
+	if _, err := io.WriteString(w, "Epoch"); err != nil {
 		return err
 	}
 
@@ -501,8 +505,6 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 }
 
 func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
-	*t = SealTicket{}
-
 	br := cbg.GetPeeker(r)
 	scratch := make([]byte, 8)
 
@@ -547,12 +549,8 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
 			if maj != cbg.MajByteString {
 				return fmt.Errorf("expected byte array")
 			}
-
+			t.Value = make([]byte, extra)
-			if extra > 0 {
+			if _, err := io.ReadFull(br, t.Value); err != nil {
-				t.Value = make([]uint8, extra)
-			}
-
-			if _, err := io.ReadFull(br, t.Value[:]); err != nil {
 				return err
 			}
 			// t.Epoch (abi.ChainEpoch) (int64)
@@ -583,8 +581,7 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			// Field doesn't exist on this type, so ignore it
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
-			cbg.ScanForLinks(r, func(cid.Cid) {})
 		}
 	}
 
@@ -609,7 +606,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Value")); err != nil {
+	if _, err := io.WriteString(w, "Value"); err != nil {
 		return err
 	}
 
@@ -621,7 +618,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	if _, err := w.Write(t.Value[:]); err != nil {
+	if _, err := w.Write(t.Value); err != nil {
 		return err
 	}
 
@@ -633,7 +630,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Epoch")); err != nil {
+	if _, err := io.WriteString(w, "Epoch"); err != nil {
 		return err
 	}
 
@@ -650,8 +647,6 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 }
 
 func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
-	*t = SealSeed{}
-
 	br := cbg.GetPeeker(r)
 	scratch := make([]byte, 8)
 
@@ -696,12 +691,8 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
 			if maj != cbg.MajByteString {
 				return fmt.Errorf("expected byte array")
 			}
-
+			t.Value = make([]byte, extra)
-			if extra > 0 {
+			if _, err := io.ReadFull(br, t.Value); err != nil {
-				t.Value = make([]uint8, extra)
-			}
-
-			if _, err := io.ReadFull(br, t.Value[:]); err != nil {
 				return err
 			}
 			// t.Epoch (abi.ChainEpoch) (int64)
@@ -732,386 +723,7 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			// Field doesn't exist on this type, so ignore it
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
-			cbg.ScanForLinks(r, func(cid.Cid) {})
-		}
-	}
-
-	return nil
-}
-func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{165}); err != nil {
-		return err
-	}
-
-	scratch := make([]byte, 9)
-
-	// t.PublishCid (cid.Cid) (struct)
-	if len("PublishCid") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"PublishCid\" was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PublishCid"))); err != nil {
-		return err
-	}
-	if _, err := io.WriteString(w, string("PublishCid")); err != nil {
-		return err
-	}
-
-	if t.PublishCid == nil {
-		if _, err := w.Write(cbg.CborNull); err != nil {
-			return err
-		}
-	} else {
-		if err := cbg.WriteCidBuf(scratch, w, *t.PublishCid); err != nil {
-			return xerrors.Errorf("failed to write cid field t.PublishCid: %w", err)
-		}
-	}
-
-	// t.DealID (abi.DealID) (uint64)
-	if len("DealID") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"DealID\" was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealID"))); err != nil {
-		return err
-	}
-	if _, err := io.WriteString(w, string("DealID")); err != nil {
-		return err
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.DealID)); err != nil {
-		return err
-	}
-
-	// t.DealProposal (market.DealProposal) (struct)
-	if len("DealProposal") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"DealProposal\" was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealProposal"))); err != nil {
-		return err
-	}
-	if _, err := io.WriteString(w, string("DealProposal")); err != nil {
-		return err
-	}
-
-	if err := t.DealProposal.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.DealSchedule (api.DealSchedule) (struct)
-	if len("DealSchedule") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"DealSchedule\" was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealSchedule"))); err != nil {
-		return err
-	}
-	if _, err := io.WriteString(w, string("DealSchedule")); err != nil {
-		return err
-	}
-
-	if err := t.DealSchedule.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.KeepUnsealed (bool) (bool)
-	if len("KeepUnsealed") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"KeepUnsealed\" was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("KeepUnsealed"))); err != nil {
-		return err
-	}
-	if _, err := io.WriteString(w, string("KeepUnsealed")); err != nil {
-		return err
-	}
-
-	if err := cbg.WriteBool(w, t.KeepUnsealed); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
-	*t = PieceDealInfo{}
-
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
-
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajMap {
-		return fmt.Errorf("cbor input should be of type map")
-	}
-
-	if extra > cbg.MaxLength {
-		return fmt.Errorf("PieceDealInfo: map struct too large (%d)", extra)
-	}
-
-	var name string
-	n := extra
-
-	for i := uint64(0); i < n; i++ {
-
-		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
-			if err != nil {
-				return err
-			}
-
-			name = string(sval)
-		}
-
-		switch name {
-		// t.PublishCid (cid.Cid) (struct)
-		case "PublishCid":
-
-			{
-
-				b, err := br.ReadByte()
-				if err != nil {
-					return err
-				}
-				if b != cbg.CborNull[0] {
-					if err := br.UnreadByte(); err != nil {
-						return err
-					}
-
-					c, err := cbg.ReadCid(br)
-					if err != nil {
-						return xerrors.Errorf("failed to read cid field t.PublishCid: %w", err)
-					}
-
-					t.PublishCid = &c
-				}
-
-			}
-			// t.DealID (abi.DealID) (uint64)
-		case "DealID":
-
-			{
-
-				maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
-				if err != nil {
-					return err
-				}
-				if maj != cbg.MajUnsignedInt {
-					return fmt.Errorf("wrong type for uint64 field")
-				}
-				t.DealID = abi.DealID(extra)
-
-			}
-			// t.DealProposal (market.DealProposal) (struct)
-		case "DealProposal":
-
-			{
-
-				b, err := br.ReadByte()
-				if err != nil {
-					return err
-				}
-				if b != cbg.CborNull[0] {
-					if err := br.UnreadByte(); err != nil {
-						return err
-					}
-					t.DealProposal = new(market.DealProposal)
-					if err := t.DealProposal.UnmarshalCBOR(br); err != nil {
-						return xerrors.Errorf("unmarshaling t.DealProposal pointer: %w", err)
-					}
-				}
-
-			}
-			// t.DealSchedule (api.DealSchedule) (struct)
-		case "DealSchedule":
-
-			{
-
-				if err := t.DealSchedule.UnmarshalCBOR(br); err != nil {
-					return xerrors.Errorf("unmarshaling t.DealSchedule: %w", err)
-				}
-
-			}
-			// t.KeepUnsealed (bool) (bool)
-		case "KeepUnsealed":
-
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
-			if err != nil {
-				return err
-			}
-			if maj != cbg.MajOther {
-				return fmt.Errorf("booleans must be major type 7")
-			}
-			switch extra {
-			case 20:
-				t.KeepUnsealed = false
-			case 21:
-				t.KeepUnsealed = true
-			default:
-				return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra)
-			}
-
-		default:
-			// Field doesn't exist on this type, so ignore it
-			cbg.ScanForLinks(r, func(cid.Cid) {})
-		}
-	}
-
-	return nil
-}
-func (t *DealSchedule) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{162}); err != nil {
-		return err
-	}
-
-	scratch := make([]byte, 9)
-
-	// t.StartEpoch (abi.ChainEpoch) (int64)
-	if len("StartEpoch") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"StartEpoch\" was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("StartEpoch"))); err != nil {
-		return err
-	}
-	if _, err := io.WriteString(w, string("StartEpoch")); err != nil {
-		return err
-	}
-
-	if t.StartEpoch >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.StartEpoch)); err != nil {
-			return err
-		}
-	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.StartEpoch-1)); err != nil {
-			return err
-		}
-	}
-
-	// t.EndEpoch (abi.ChainEpoch) (int64)
-	if len("EndEpoch") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"EndEpoch\" was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("EndEpoch"))); err != nil {
-		return err
-	}
-	if _, err := io.WriteString(w, string("EndEpoch")); err != nil {
-		return err
-	}
-
-	if t.EndEpoch >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.EndEpoch)); err != nil {
-			return err
-		}
-	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.EndEpoch-1)); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error {
-	*t = DealSchedule{}
-
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
-
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajMap {
-		return fmt.Errorf("cbor input should be of type map")
-	}
-
-	if extra > cbg.MaxLength {
-		return fmt.Errorf("DealSchedule: map struct too large (%d)", extra)
-	}
-
-	var name string
-	n := extra
-
-	for i := uint64(0); i < n; i++ {
-
-		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
-			if err != nil {
-				return err
-			}
-
-			name = string(sval)
-		}
-
-		switch name {
-		// t.StartEpoch (abi.ChainEpoch) (int64)
-		case "StartEpoch":
-			{
-				maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
-				var extraI int64
-				if err != nil {
-					return err
-				}
-				switch maj {
-				case cbg.MajUnsignedInt:
-					extraI = int64(extra)
-					if extraI < 0 {
-						return fmt.Errorf("int64 positive overflow")
-					}
-				case cbg.MajNegativeInt:
-					extraI = int64(extra)
-					if extraI < 0 {
-						return fmt.Errorf("int64 negative oveflow")
-					}
-					extraI = -1 - extraI
-				default:
-					return fmt.Errorf("wrong type for int64 field: %d", maj)
-				}
-
-				t.StartEpoch = abi.ChainEpoch(extraI)
-			}
-			// t.EndEpoch (abi.ChainEpoch) (int64)
-		case "EndEpoch":
-			{
-				maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
-				var extraI int64
-				if err != nil {
-					return err
-				}
-				switch maj {
-				case cbg.MajUnsignedInt:
-					extraI = int64(extra)
-					if extraI < 0 {
-						return fmt.Errorf("int64 positive overflow")
-					}
-				case cbg.MajNegativeInt:
-					extraI = int64(extra)
-					if extraI < 0 {
-						return fmt.Errorf("int64 negative oveflow")
-					}
-					extraI = -1 - extraI
-				default:
-					return fmt.Errorf("wrong type for int64 field: %d", maj)
-				}
-
-				t.EndEpoch = abi.ChainEpoch(extraI)
-			}
-
-		default:
-			// Field doesn't exist on this type, so ignore it
-			cbg.ScanForLinks(r, func(cid.Cid) {})
 		}
 	}
 

api/checkstatuscode_string.go

@@ -1,35 +0,0 @@
-// Code generated by "stringer -type=CheckStatusCode -trimprefix=CheckStatus"; DO NOT EDIT.
-
-package api
-
-import "strconv"
-
-func _() {
-	// An "invalid array index" compiler error signifies that the constant values have changed.
-	// Re-run the stringer command to generate them again.
-	var x [1]struct{}
-	_ = x[CheckStatusMessageSerialize-1]
-	_ = x[CheckStatusMessageSize-2]
-	_ = x[CheckStatusMessageValidity-3]
-	_ = x[CheckStatusMessageMinGas-4]
-	_ = x[CheckStatusMessageMinBaseFee-5]
-	_ = x[CheckStatusMessageBaseFee-6]
-	_ = x[CheckStatusMessageBaseFeeLowerBound-7]
-	_ = x[CheckStatusMessageBaseFeeUpperBound-8]
-	_ = x[CheckStatusMessageGetStateNonce-9]
-	_ = x[CheckStatusMessageNonce-10]
-	_ = x[CheckStatusMessageGetStateBalance-11]
-	_ = x[CheckStatusMessageBalance-12]
-}
-
-const _CheckStatusCode_name = "MessageSerializeMessageSizeMessageValidityMessageMinGasMessageMinBaseFeeMessageBaseFeeMessageBaseFeeLowerBoundMessageBaseFeeUpperBoundMessageGetStateNonceMessageNonceMessageGetStateBalanceMessageBalance"
-
-var _CheckStatusCode_index = [...]uint8{0, 16, 27, 42, 55, 72, 86, 110, 134, 154, 166, 188, 202}
-
-func (i CheckStatusCode) String() string {
-	i -= 1
-	if i < 0 || i >= CheckStatusCode(len(_CheckStatusCode_index)-1) {
-		return "CheckStatusCode(" + strconv.FormatInt(int64(i+1), 10) + ")"
-	}
-	return _CheckStatusCode_name[_CheckStatusCode_index[i]:_CheckStatusCode_index[i+1]]
-}

api/client/client.go

@@ -1,24 +1,18 @@
 package client
 
 import (
-	"context"
 	"net/http"
-	"net/url"
-	"path"
-	"time"
 
 	"github.com/filecoin-project/go-jsonrpc"
 
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/api/v0api"
+	"github.com/filecoin-project/lotus/api/apistruct"
-	"github.com/filecoin-project/lotus/api/v1api"
-	"github.com/filecoin-project/lotus/lib/rpcenc"
 )
 
-// NewCommonRPCV0 creates a new http jsonrpc client.
+// NewCommonRPC creates a new http jsonrpc client.
-func NewCommonRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.Common, jsonrpc.ClientCloser, error) {
+func NewCommonRPC(addr string, requestHeader http.Header) (api.Common, jsonrpc.ClientCloser, error) {
-	var res v0api.CommonStruct
+	var res apistruct.CommonStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
 		[]interface{}{
 			&res.Internal,
 		},
@@ -28,10 +22,10 @@ func NewCommonRPCV0(ctx context.Context, addr string, requestHeader http.Header)
 	return &res, closer, err
 }
 
-// NewFullNodeRPCV0 creates a new http jsonrpc client.
+// NewFullNodeRPC creates a new http jsonrpc client.
-func NewFullNodeRPCV0(ctx context.Context, addr string, requestHeader http.Header) (v0api.FullNode, jsonrpc.ClientCloser, error) {
+func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
-	var res v0api.FullNodeStruct
+	var res apistruct.FullNodeStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
 		[]interface{}{
 			&res.CommonStruct.Internal,
 			&res.Internal,
@@ -40,108 +34,23 @@ func NewFullNodeRPCV0(ctx context.Context, addr string, requestHeader http.Heade
 	return &res, closer, err
 }
 
-// NewFullNodeRPCV1 creates a new http jsonrpc client.
+// NewStorageMinerRPC creates a new http jsonrpc client for storage miner
-func NewFullNodeRPCV1(ctx context.Context, addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
+func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMiner, jsonrpc.ClientCloser, error) {
-	var res v1api.FullNodeStruct
+	var res apistruct.StorageMinerStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
-		[]interface{}{
-			&res.CommonStruct.Internal,
-			&res.Internal,
-		}, requestHeader)
-
-	return &res, closer, err
-}
-
-func getPushUrl(addr string) (string, error) {
-	pushUrl, err := url.Parse(addr)
-	if err != nil {
-		return "", err
-	}
-	switch pushUrl.Scheme {
-	case "ws":
-		pushUrl.Scheme = "http"
-	case "wss":
-		pushUrl.Scheme = "https"
-	}
-	///rpc/v0 -> /rpc/streams/v0/push
-
-	pushUrl.Path = path.Join(pushUrl.Path, "../streams/v0/push")
-	return pushUrl.String(), nil
-}
-
-// NewStorageMinerRPCV0 creates a new http jsonrpc client for miner
-func NewStorageMinerRPCV0(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (v0api.StorageMiner, jsonrpc.ClientCloser, error) {
-	pushUrl, err := getPushUrl(addr)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	var res v0api.StorageMinerStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
 		[]interface{}{
 			&res.CommonStruct.Internal,
 			&res.Internal,
 		},
 		requestHeader,
-		append([]jsonrpc.Option{
-			rpcenc.ReaderParamEncoder(pushUrl),
-		}, opts...)...,
 	)
 
 	return &res, closer, err
 }
 
-func NewWorkerRPCV0(ctx context.Context, addr string, requestHeader http.Header) (v0api.Worker, jsonrpc.ClientCloser, error) {
+func NewWorkerRPC(addr string, requestHeader http.Header) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
-	pushUrl, err := getPushUrl(addr)
+	var res apistruct.WorkerStruct
-	if err != nil {
+	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
-		return nil, nil, err
-	}
-
-	var res api.WorkerStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
-		[]interface{}{
-			&res.Internal,
-		},
-		requestHeader,
-		rpcenc.ReaderParamEncoder(pushUrl),
-		jsonrpc.WithNoReconnect(),
-		jsonrpc.WithTimeout(30*time.Second),
-	)
-
-	return &res, closer, err
-}
-
-// NewGatewayRPCV1 creates a new http jsonrpc client for a gateway node.
-func NewGatewayRPCV1(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.Gateway, jsonrpc.ClientCloser, error) {
-	var res api.GatewayStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
-		[]interface{}{
-			&res.Internal,
-		},
-		requestHeader,
-		opts...,
-	)
-
-	return &res, closer, err
-}
-
-// NewGatewayRPCV0 creates a new http jsonrpc client for a gateway node.
-func NewGatewayRPCV0(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (v0api.Gateway, jsonrpc.ClientCloser, error) {
-	var res v0api.GatewayStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
-		[]interface{}{
-			&res.Internal,
-		},
-		requestHeader,
-		opts...,
-	)
-
-	return &res, closer, err
-}
-
-func NewWalletRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.Wallet, jsonrpc.ClientCloser, error) {
-	var res api.WalletStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
 		[]interface{}{
 			&res.Internal,
 		},

api/docgen-openrpc/cmd/docgen_openrpc.go

@@ -1,74 +0,0 @@
-package main
-
-import (
-	"compress/gzip"
-	"encoding/json"
-	"io"
-	"log"
-	"os"
-
-	"github.com/filecoin-project/lotus/api/docgen"
-
-	docgen_openrpc "github.com/filecoin-project/lotus/api/docgen-openrpc"
-)
-
-/*
-main defines a small program that writes an OpenRPC document describing
-a Lotus API to stdout.
-
-If the first argument is "miner", the document will describe the StorageMiner API.
-If not (no, or any other args), the document will describe the Full API.
-
-Use:
-
-		go run ./api/openrpc/cmd ["api/api_full.go"|"api/api_storage.go"|"api/api_worker.go"] ["FullNode"|"StorageMiner"|"Worker"]
-
-	With gzip compression: a '-gzip' flag is made available as an optional third argument. Note that position matters.
-
-		go run ./api/openrpc/cmd ["api/api_full.go"|"api/api_storage.go"|"api/api_worker.go"] ["FullNode"|"StorageMiner"|"Worker"] -gzip
-
-*/
-
-func main() {
-	Comments, GroupDocs := docgen.ParseApiASTInfo(os.Args[1], os.Args[2], os.Args[3], os.Args[4])
-
-	doc := docgen_openrpc.NewLotusOpenRPCDocument(Comments, GroupDocs)
-
-	i, _, _, _ := docgen.GetAPIType(os.Args[2], os.Args[3])
-	doc.RegisterReceiverName("Filecoin", i)
-
-	out, err := doc.Discover()
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	var jsonOut []byte
-	var writer io.WriteCloser
-
-	// Use os.Args to handle a somewhat hacky flag for the gzip option.
-	// Could use flags package to handle this more cleanly, but that requires changes elsewhere
-	// the scope of which just isn't warranted by this one use case which will usually be run
-	// programmatically anyways.
-	if len(os.Args) > 5 && os.Args[5] == "-gzip" {
-		jsonOut, err = json.Marshal(out)
-		if err != nil {
-			log.Fatalln(err)
-		}
-		writer = gzip.NewWriter(os.Stdout)
-	} else {
-		jsonOut, err = json.MarshalIndent(out, "", "    ")
-		if err != nil {
-			log.Fatalln(err)
-		}
-		writer = os.Stdout
-	}
-
-	_, err = writer.Write(jsonOut)
-	if err != nil {
-		log.Fatalln(err)
-	}
-	err = writer.Close()
-	if err != nil {
-		log.Fatalln(err)
-	}
-}

api/docgen-openrpc/openrpc.go

@@ -1,161 +0,0 @@
-package docgenopenrpc
-
-import (
-	"encoding/json"
-	"go/ast"
-	"net"
-	"reflect"
-
-	"github.com/alecthomas/jsonschema"
-	go_openrpc_reflect "github.com/etclabscore/go-openrpc-reflect"
-	"github.com/filecoin-project/lotus/api/docgen"
-	"github.com/filecoin-project/lotus/build"
-	"github.com/ipfs/go-cid"
-	meta_schema "github.com/open-rpc/meta-schema"
-)
-
-// schemaDictEntry represents a type association passed to the jsonschema reflector.
-type schemaDictEntry struct {
-	example interface{}
-	rawJson string
-}
-
-const integerD = `{
-          "title": "number",
-          "type": "number",
-          "description": "Number is a number"
-        }`
-
-const cidCidD = `{"title": "Content Identifier", "type": "string", "description": "Cid represents a self-describing content addressed identifier. It is formed by a Version, a Codec (which indicates a multicodec-packed content type) and a Multihash."}`
-
-func OpenRPCSchemaTypeMapper(ty reflect.Type) *jsonschema.Type {
-	unmarshalJSONToJSONSchemaType := func(input string) *jsonschema.Type {
-		var js jsonschema.Type
-		err := json.Unmarshal([]byte(input), &js)
-		if err != nil {
-			panic(err)
-		}
-		return &js
-	}
-
-	if ty.Kind() == reflect.Ptr {
-		ty = ty.Elem()
-	}
-
-	if ty == reflect.TypeOf((*interface{})(nil)).Elem() {
-		return &jsonschema.Type{Type: "object", AdditionalProperties: []byte("true")}
-	}
-
-	// Second, handle other types.
-	// Use a slice instead of a map because it preserves order, as a logic safeguard/fallback.
-	dict := []schemaDictEntry{
-		{cid.Cid{}, cidCidD},
-	}
-
-	for _, d := range dict {
-		if reflect.TypeOf(d.example) == ty {
-			tt := unmarshalJSONToJSONSchemaType(d.rawJson)
-
-			return tt
-		}
-	}
-
-	// Handle primitive types in case there are generic cases
-	// specific to our services.
-	switch ty.Kind() {
-	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
-		// Return all integer types as the hex representation integer schemea.
-		ret := unmarshalJSONToJSONSchemaType(integerD)
-		return ret
-	case reflect.Uintptr:
-		return &jsonschema.Type{Type: "number", Title: "uintptr-title"}
-	case reflect.Struct:
-	case reflect.Map:
-	case reflect.Slice, reflect.Array:
-	case reflect.Float32, reflect.Float64:
-	case reflect.Bool:
-	case reflect.String:
-	case reflect.Ptr, reflect.Interface:
-	default:
-	}
-
-	return nil
-}
-
-// NewLotusOpenRPCDocument defines application-specific documentation and configuration for its OpenRPC document.
-func NewLotusOpenRPCDocument(Comments, GroupDocs map[string]string) *go_openrpc_reflect.Document {
-	d := &go_openrpc_reflect.Document{}
-
-	// Register "Meta" document fields.
-	// These include getters for
-	// - Servers object
-	// - Info object
-	// - ExternalDocs object
-	//
-	// These objects represent server-specific data that cannot be
-	// reflected.
-	d.WithMeta(&go_openrpc_reflect.MetaT{
-		GetServersFn: func() func(listeners []net.Listener) (*meta_schema.Servers, error) {
-			return func(listeners []net.Listener) (*meta_schema.Servers, error) {
-				return nil, nil
-			}
-		},
-		GetInfoFn: func() (info *meta_schema.InfoObject) {
-			info = &meta_schema.InfoObject{}
-			title := "Lotus RPC API"
-			info.Title = (*meta_schema.InfoObjectProperties)(&title)
-
-			version := build.BuildVersion
-			info.Version = (*meta_schema.InfoObjectVersion)(&version)
-			return info
-		},
-		GetExternalDocsFn: func() (exdocs *meta_schema.ExternalDocumentationObject) {
-			return nil // FIXME
-		},
-	})
-
-	// Use a provided Ethereum default configuration as a base.
-	appReflector := &go_openrpc_reflect.EthereumReflectorT{}
-
-	// Install overrides for the json schema->type map fn used by the jsonschema reflect package.
-	appReflector.FnSchemaTypeMap = func() func(ty reflect.Type) *jsonschema.Type {
-		return OpenRPCSchemaTypeMapper
-	}
-
-	appReflector.FnIsMethodEligible = func(m reflect.Method) bool {
-		for i := 0; i < m.Func.Type().NumOut(); i++ {
-			if m.Func.Type().Out(i).Kind() == reflect.Chan {
-				return false
-			}
-		}
-		return go_openrpc_reflect.EthereumReflector.IsMethodEligible(m)
-	}
-	appReflector.FnGetMethodName = func(moduleName string, r reflect.Value, m reflect.Method, funcDecl *ast.FuncDecl) (string, error) {
-		if m.Name == "ID" {
-			return moduleName + "_ID", nil
-		}
-		if moduleName == "rpc" && m.Name == "Discover" {
-			return "rpc.discover", nil
-		}
-
-		return moduleName + "." + m.Name, nil
-	}
-
-	appReflector.FnGetMethodSummary = func(r reflect.Value, m reflect.Method, funcDecl *ast.FuncDecl) (string, error) {
-		if v, ok := Comments[m.Name]; ok {
-			return v, nil
-		}
-		return "", nil // noComment
-	}
-
-	appReflector.FnSchemaExamples = func(ty reflect.Type) (examples *meta_schema.Examples, err error) {
-		v := docgen.ExampleValue("unknown", ty, ty) // This isn't ideal, but seems to work well enough.
-		return &meta_schema.Examples{
-			meta_schema.AlwaysTrue(v),
-		}, nil
-	}
-
-	// Finally, register the configured reflector to the document.
-	d.WithReflector(appReflector)
-	return d
-}

api/docgen/cmd/docgen.go

@@ -1,116 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-	"sort"
-	"strings"
-
-	"github.com/filecoin-project/lotus/api/docgen"
-)
-
-func main() {
-	comments, groupComments := docgen.ParseApiASTInfo(os.Args[1], os.Args[2], os.Args[3], os.Args[4])
-
-	groups := make(map[string]*docgen.MethodGroup)
-
-	_, t, permStruct, commonPermStruct := docgen.GetAPIType(os.Args[2], os.Args[3])
-
-	for i := 0; i < t.NumMethod(); i++ {
-		m := t.Method(i)
-
-		groupName := docgen.MethodGroupFromName(m.Name)
-
-		g, ok := groups[groupName]
-		if !ok {
-			g = new(docgen.MethodGroup)
-			g.Header = groupComments[groupName]
-			g.GroupName = groupName
-			groups[groupName] = g
-		}
-
-		var args []interface{}
-		ft := m.Func.Type()
-		for j := 2; j < ft.NumIn(); j++ {
-			inp := ft.In(j)
-			args = append(args, docgen.ExampleValue(m.Name, inp, nil))
-		}
-
-		v, err := json.MarshalIndent(args, "", "  ")
-		if err != nil {
-			panic(err)
-		}
-
-		outv := docgen.ExampleValue(m.Name, ft.Out(0), nil)
-
-		ov, err := json.MarshalIndent(outv, "", "  ")
-		if err != nil {
-			panic(err)
-		}
-
-		g.Methods = append(g.Methods, &docgen.Method{
-			Name:            m.Name,
-			Comment:         comments[m.Name],
-			InputExample:    string(v),
-			ResponseExample: string(ov),
-		})
-	}
-
-	var groupslice []*docgen.MethodGroup
-	for _, g := range groups {
-		groupslice = append(groupslice, g)
-	}
-
-	sort.Slice(groupslice, func(i, j int) bool {
-		return groupslice[i].GroupName < groupslice[j].GroupName
-	})
-
-	fmt.Printf("# Groups\n")
-
-	for _, g := range groupslice {
-		fmt.Printf("* [%s](#%s)\n", g.GroupName, g.GroupName)
-		for _, method := range g.Methods {
-			fmt.Printf("  * [%s](#%s)\n", method.Name, method.Name)
-		}
-	}
-
-	for _, g := range groupslice {
-		g := g
-		fmt.Printf("## %s\n", g.GroupName)
-		fmt.Printf("%s\n\n", g.Header)
-
-		sort.Slice(g.Methods, func(i, j int) bool {
-			return g.Methods[i].Name < g.Methods[j].Name
-		})
-
-		for _, m := range g.Methods {
-			fmt.Printf("### %s\n", m.Name)
-			fmt.Printf("%s\n\n", m.Comment)
-
-			meth, ok := permStruct.FieldByName(m.Name)
-			if !ok {
-				meth, ok = commonPermStruct.FieldByName(m.Name)
-				if !ok {
-					panic("no perms for method: " + m.Name)
-				}
-			}
-
-			perms := meth.Tag.Get("perm")
-
-			fmt.Printf("Perms: %s\n\n", perms)
-
-			if strings.Count(m.InputExample, "\n") > 0 {
-				fmt.Printf("Inputs:\n```json\n%s\n```\n\n", m.InputExample)
-			} else {
-				fmt.Printf("Inputs: `%s`\n\n", m.InputExample)
-			}
-
-			if strings.Count(m.ResponseExample, "\n") > 0 {
-				fmt.Printf("Response:\n```json\n%s\n```\n\n", m.ResponseExample)
-			} else {
-				fmt.Printf("Response: `%s`\n\n", m.ResponseExample)
-			}
-		}
-	}
-}

api/docgen/docgen.go

@@ -1,49 +1,32 @@
-package docgen
+package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"go/ast"
 	"go/parser"
 	"go/token"
-	"path/filepath"
 	"reflect"
+	"sort"
 	"strings"
 	"time"
 	"unicode"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
-	"github.com/filecoin-project/go-multistore"
-	"github.com/filecoin-project/lotus/node/repo/importmgr"
-	"github.com/google/uuid"
-	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-filestore"
-	metrics "github.com/libp2p/go-libp2p-core/metrics"
-	"github.com/libp2p/go-libp2p-core/network"
-	"github.com/libp2p/go-libp2p-core/peer"
-	protocol "github.com/libp2p/go-libp2p-core/protocol"
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
-	"github.com/multiformats/go-multiaddr"
-
-	datatransfer "github.com/filecoin-project/go-data-transfer"
-	filestore2 "github.com/filecoin-project/go-fil-markets/filestore"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-jsonrpc/auth"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/crypto"
-	"github.com/filecoin-project/go-state-types/exitcode"
-
 	"github.com/filecoin-project/lotus/api"
-	apitypes "github.com/filecoin-project/lotus/api/types"
-	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
+	"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-filestore"
+	"github.com/libp2p/go-libp2p-core/network"
+	peer "github.com/libp2p/go-libp2p-peer"
+	"github.com/multiformats/go-multiaddr"
 )
 
 var ExampleValues = map[reflect.Type]interface{}{
@@ -82,26 +65,17 @@ func init() {
 
 	ExampleValues[reflect.TypeOf(addr)] = addr
 
-	pid, err := peer.Decode("12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf")
+	pid, err := peer.IDB58Decode("12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf")
 	if err != nil {
 		panic(err)
 	}
 	addExample(pid)
-	addExample(&pid)
-
-	multistoreIDExample := multistore.StoreID(50)
-	storeIDExample := importmgr.ImportID(50)
 
 	addExample(bitfield.NewFromSet([]uint64{5}))
-	addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1)
+	addExample(abi.RegisteredProof_StackedDRG32GiBPoSt)
-	addExample(abi.RegisteredPoStProof_StackedDrgWindow32GiBV1)
 	addExample(abi.ChainEpoch(10101))
 	addExample(crypto.SigTypeBLS)
-	addExample(types.KTBLS)
 	addExample(int64(9))
-	addExample(12.3)
-	addExample(123)
-	addExample(uintptr(0))
 	addExample(abi.MethodNum(1))
 	addExample(exitcode.ExitCode(0))
 	addExample(crypto.DomainSeparationTag_ElectionProofProduction)
@@ -116,57 +90,21 @@ func init() {
 	addExample(network.Connected)
 	addExample(dtypes.NetworkName("lotus"))
 	addExample(api.SyncStateStage(1))
-	addExample(api.FullAPIVersion1)
+	addExample(build.APIVersion)
 	addExample(api.PCHInbound)
 	addExample(time.Minute)
-	addExample(datatransfer.TransferID(3))
+	addExample(&types.ExecutionResult{
-	addExample(datatransfer.Ongoing)
+		Msg:    exampleValue(reflect.TypeOf(&types.Message{})).(*types.Message),
-	addExample(storeIDExample)
+		MsgRct: exampleValue(reflect.TypeOf(&types.MessageReceipt{})).(*types.MessageReceipt),
-	addExample(&storeIDExample)
-	addExample(multistoreIDExample)
-	addExample(&multistoreIDExample)
-	addExample(retrievalmarket.ClientEventDealAccepted)
-	addExample(retrievalmarket.DealStatusNew)
-	addExample(network.ReachabilityPublic)
-	addExample(build.NewestNetworkVersion)
-	addExample(map[string]int{"name": 42})
-	addExample(map[string]time.Time{"name": time.Unix(1615243938, 0).UTC()})
-	addExample(&types.ExecutionTrace{
-		Msg:    ExampleValue("init", reflect.TypeOf(&types.Message{}), nil).(*types.Message),
-		MsgRct: ExampleValue("init", reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt),
 	})
 	addExample(map[string]types.Actor{
-		"t01236": ExampleValue("init", reflect.TypeOf(types.Actor{}), nil).(types.Actor),
+		"t01236": exampleValue(reflect.TypeOf(types.Actor{})).(types.Actor),
 	})
 	addExample(map[string]api.MarketDeal{
-		"t026363": ExampleValue("init", reflect.TypeOf(api.MarketDeal{}), nil).(api.MarketDeal),
+		"t026363": exampleValue(reflect.TypeOf(api.MarketDeal{})).(api.MarketDeal),
 	})
 	addExample(map[string]api.MarketBalance{
-		"t026363": ExampleValue("init", reflect.TypeOf(api.MarketBalance{}), nil).(api.MarketBalance),
+		"t026363": exampleValue(reflect.TypeOf(api.MarketBalance{})).(api.MarketBalance),
-	})
-	addExample(map[string]*pubsub.TopicScoreSnapshot{
-		"/blocks": {
-			TimeInMesh:               time.Minute,
-			FirstMessageDeliveries:   122,
-			MeshMessageDeliveries:    1234,
-			InvalidMessageDeliveries: 3,
-		},
-	})
-	addExample(map[string]metrics.Stats{
-		"12D3KooWSXmXLJmBR1M7i9RW9GQPNUhZSzXKzxDHWtAgNuJAbyEJ": {
-			RateIn:   100,
-			RateOut:  50,
-			TotalIn:  174000,
-			TotalOut: 12500,
-		},
-	})
-	addExample(map[protocol.ID]metrics.Stats{
-		"/fil/hello/1.0.0": {
-			RateIn:   100,
-			RateOut:  50,
-			TotalIn:  174000,
-			TotalOut: 12500,
-		},
 	})
 
 	maddr, err := multiaddr.NewMultiaddr("/ip4/52.36.61.156/tcp/1347/p2p/12D3KooWFETiESTf1v4PGUvtnxMAcEFMzLZbJGg4tjWfGEimYior")
@@ -177,136 +115,9 @@ func init() {
 	// because reflect.TypeOf(maddr) returns the concrete type...
 	ExampleValues[reflect.TypeOf(struct{ A multiaddr.Multiaddr }{}).Field(0).Type] = maddr
 
-	// miner specific
-	addExample(filestore2.Path(".lotusminer/fstmp123"))
-	si := uint64(12)
-	addExample(&si)
-	addExample(retrievalmarket.DealID(5))
-	addExample(abi.ActorID(1000))
-	addExample(map[string][]api.SealedRef{
-		"98000": {
-			api.SealedRef{
-				SectorID: 100,
-				Offset:   10 << 20,
-				Size:     1 << 20,
-			},
-		},
-	})
-	addExample(api.SectorState(sealing.Proving))
-	addExample(stores.ID("76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8"))
-	addExample(storiface.FTUnsealed)
-	addExample(storiface.PathSealing)
-	addExample(map[stores.ID][]stores.Decl{
-		"76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8": {
-			{
-				SectorID:       abi.SectorID{Miner: 1000, Number: 100},
-				SectorFileType: storiface.FTSealed,
-			},
-		},
-	})
-	addExample(map[stores.ID]string{
-		"76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8": "/data/path",
-	})
-	addExample(map[uuid.UUID][]storiface.WorkerJob{
-		uuid.MustParse("ef8d99a2-6865-4189-8ffa-9fef0f806eee"): {
-			{
-				ID: storiface.CallID{
-					Sector: abi.SectorID{Miner: 1000, Number: 100},
-					ID:     uuid.MustParse("76081ba0-61bd-45a5-bc08-af05f1c26e5d"),
-				},
-				Sector:   abi.SectorID{Miner: 1000, Number: 100},
-				Task:     sealtasks.TTPreCommit2,
-				RunWait:  0,
-				Start:    time.Unix(1605172927, 0).UTC(),
-				Hostname: "host",
-			},
-		},
-	})
-	addExample(map[uuid.UUID]storiface.WorkerStats{
-		uuid.MustParse("ef8d99a2-6865-4189-8ffa-9fef0f806eee"): {
-			Info: storiface.WorkerInfo{
-				Hostname: "host",
-				Resources: storiface.WorkerResources{
-					MemPhysical: 256 << 30,
-					MemSwap:     120 << 30,
-					MemReserved: 2 << 30,
-					CPUs:        64,
-					GPUs:        []string{"aGPU 1337"},
-				},
-			},
-			Enabled:    true,
-			MemUsedMin: 0,
-			MemUsedMax: 0,
-			GpuUsed:    false,
-			CpuUse:     0,
-		},
-	})
-	addExample(storiface.ErrorCode(0))
-	addExample(map[abi.SectorNumber]string{
-		123: "can't acquire read lock",
-	})
-	addExample(map[api.SectorState]int{
-		api.SectorState(sealing.Proving): 120,
-	})
-	addExample([]abi.SectorNumber{123, 124})
-
-	// worker specific
-	addExample(storiface.AcquireMove)
-	addExample(storiface.UnpaddedByteIndex(abi.PaddedPieceSize(1 << 20).Unpadded()))
-	addExample(map[sealtasks.TaskType]struct{}{
-		sealtasks.TTPreCommit2: {},
-	})
-	addExample(sealtasks.TTCommit2)
-	addExample(apitypes.OpenRPCDocument{
-		"openrpc": "1.2.6",
-		"info": map[string]interface{}{
-			"title":   "Lotus RPC API",
-			"version": "1.2.1/generated=2020-11-22T08:22:42-06:00",
-		},
-		"methods": []interface{}{}},
-	)
-
-	addExample(api.CheckStatusCode(0))
-	addExample(map[string]interface{}{"abc": 123})
 }
 
-func GetAPIType(name, pkg string) (i interface{}, t, permStruct, commonPermStruct reflect.Type) {
+func exampleValue(t reflect.Type) interface{} {
-	switch pkg {
-	case "api": // latest
-		switch name {
-		case "FullNode":
-			i = &api.FullNodeStruct{}
-			t = reflect.TypeOf(new(struct{ api.FullNode })).Elem()
-			permStruct = reflect.TypeOf(api.FullNodeStruct{}.Internal)
-			commonPermStruct = reflect.TypeOf(api.CommonStruct{}.Internal)
-		case "StorageMiner":
-			i = &api.StorageMinerStruct{}
-			t = reflect.TypeOf(new(struct{ api.StorageMiner })).Elem()
-			permStruct = reflect.TypeOf(api.StorageMinerStruct{}.Internal)
-			commonPermStruct = reflect.TypeOf(api.CommonStruct{}.Internal)
-		case "Worker":
-			i = &api.WorkerStruct{}
-			t = reflect.TypeOf(new(struct{ api.Worker })).Elem()
-			permStruct = reflect.TypeOf(api.WorkerStruct{}.Internal)
-			commonPermStruct = reflect.TypeOf(api.WorkerStruct{}.Internal)
-		default:
-			panic("unknown type")
-		}
-	case "v0api":
-		switch name {
-		case "FullNode":
-			i = v0api.FullNodeStruct{}
-			t = reflect.TypeOf(new(struct{ v0api.FullNode })).Elem()
-			permStruct = reflect.TypeOf(v0api.FullNodeStruct{}.Internal)
-			commonPermStruct = reflect.TypeOf(v0api.CommonStruct{}.Internal)
-		default:
-			panic("unknown type")
-		}
-	}
-	return
-}
-
-func ExampleValue(method string, t, parent reflect.Type) interface{} {
 	v, ok := ExampleValues[t]
 	if ok {
 		return v
@@ -315,25 +126,25 @@ func ExampleValue(method string, t, parent reflect.Type) interface{} {
 	switch t.Kind() {
 	case reflect.Slice:
 		out := reflect.New(t).Elem()
-		reflect.Append(out, reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
+		reflect.Append(out, reflect.ValueOf(exampleValue(t.Elem())))
 		return out.Interface()
 	case reflect.Chan:
-		return ExampleValue(method, t.Elem(), nil)
+		return exampleValue(t.Elem())
 	case reflect.Struct:
-		es := exampleStruct(method, t, parent)
+		es := exampleStruct(t)
 		v := reflect.ValueOf(es).Elem().Interface()
 		ExampleValues[t] = v
 		return v
 	case reflect.Array:
 		out := reflect.New(t).Elem()
 		for i := 0; i < t.Len(); i++ {
-			out.Index(i).Set(reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
+			out.Index(i).Set(reflect.ValueOf(exampleValue(t.Elem())))
 		}
 		return out.Interface()
 
 	case reflect.Ptr:
 		if t.Elem().Kind() == reflect.Struct {
-			es := exampleStruct(method, t.Elem(), t)
+			es := exampleStruct(t.Elem())
 			//ExampleValues[t] = es
 			return es
 		}
@@ -341,18 +152,15 @@ func ExampleValue(method string, t, parent reflect.Type) interface{} {
 		return struct{}{}
 	}
 
-	panic(fmt.Sprintf("No example value for type: %s (method '%s')", t, method))
+	panic(fmt.Sprintf("No example value for type: %s", t))
 }
 
-func exampleStruct(method string, t, parent reflect.Type) interface{} {
+func exampleStruct(t reflect.Type) interface{} {
 	ns := reflect.New(t)
 	for i := 0; i < t.NumField(); i++ {
 		f := t.Field(i)
-		if f.Type == parent {
-			continue
-		}
 		if strings.Title(f.Name) == f.Name {
-			ns.Elem().Field(i).Set(reflect.ValueOf(ExampleValue(method, f.Type, t)))
+			ns.Elem().Field(i).Set(reflect.ValueOf(exampleValue(f.Type)))
 		}
 	}
 
@@ -360,7 +168,6 @@ func exampleStruct(method string, t, parent reflect.Type) interface{} {
 }
 
 type Visitor struct {
-	Root    string
 	Methods map[string]ast.Node
 }
 
@@ -370,7 +177,7 @@ func (v *Visitor) Visit(node ast.Node) ast.Visitor {
 		return v
 	}
 
-	if st.Name.Name != v.Root {
+	if st.Name.Name != "FullNode" {
 		return nil
 	}
 
@@ -384,43 +191,32 @@ func (v *Visitor) Visit(node ast.Node) ast.Visitor {
 	return v
 }
 
-const NoComment = "There are not yet any comments for this method."
+const noComment = "There are not yet any comments for this method."
 
-func ParseApiASTInfo(apiFile, iface, pkg, dir string) (comments map[string]string, groupDocs map[string]string) { //nolint:golint
+func parseApiASTInfo() (map[string]string, map[string]string) { //nolint:golint
 	fset := token.NewFileSet()
-	apiDir, err := filepath.Abs(dir)
+	pkgs, err := parser.ParseDir(fset, "./api", nil, parser.AllErrors|parser.ParseComments)
-	if err != nil {
-		fmt.Println("./api filepath absolute error: ", err)
-		return
-	}
-	apiFile, err = filepath.Abs(apiFile)
-	if err != nil {
-		fmt.Println("filepath absolute error: ", err, "file:", apiFile)
-		return
-	}
-	pkgs, err := parser.ParseDir(fset, apiDir, nil, parser.AllErrors|parser.ParseComments)
 	if err != nil {
 		fmt.Println("parse error: ", err)
-		return
 	}
 
-	ap := pkgs[pkg]
+	ap := pkgs["api"]
 
-	f := ap.Files[apiFile]
+	f := ap.Files["api/api_full.go"]
 
 	cmap := ast.NewCommentMap(fset, f, f.Comments)
 
-	v := &Visitor{iface, make(map[string]ast.Node)}
+	v := &Visitor{make(map[string]ast.Node)}
-	ast.Walk(v, ap)
+	ast.Walk(v, pkgs["api"])
 
-	comments = make(map[string]string)
+	groupDocs := make(map[string]string)
-	groupDocs = make(map[string]string)
+	out := make(map[string]string)
 	for mn, node := range v.Methods {
-		filteredComments := cmap.Filter(node).Comments()
+		cs := cmap.Filter(node).Comments()
-		if len(filteredComments) == 0 {
+		if len(cs) == 0 {
-			comments[mn] = NoComment
+			out[mn] = noComment
 		} else {
-			for _, c := range filteredComments {
+			for _, c := range cs {
 				if strings.HasPrefix(c.Text(), "MethodGroup:") {
 					parts := strings.Split(c.Text(), "\n")
 					groupName := strings.TrimSpace(parts[0][12:])
@@ -431,19 +227,15 @@ func ParseApiASTInfo(apiFile, iface, pkg, dir string) (comments map[string]strin
 				}
 			}
 
-			l := len(filteredComments) - 1
+			last := cs[len(cs)-1].Text()
-			if len(filteredComments) > 1 {
-				l = len(filteredComments) - 2
-			}
-			last := filteredComments[l].Text()
 			if !strings.HasPrefix(last, "MethodGroup:") {
-				comments[mn] = last
+				out[mn] = last
 			} else {
-				comments[mn] = NoComment
+				out[mn] = noComment
 			}
 		}
 	}
-	return comments, groupDocs
+	return out, groupDocs
 }
 
 type MethodGroup struct {
@@ -459,7 +251,7 @@ type Method struct {
 	ResponseExample string
 }
 
-func MethodGroupFromName(mn string) string {
+func methodGroupFromName(mn string) string {
 	i := strings.IndexFunc(mn[1:], func(r rune) bool {
 		return unicode.IsUpper(r)
 	})
@@ -468,3 +260,79 @@ func MethodGroupFromName(mn string) string {
 	}
 	return mn[:i+1]
 }
+
+func main() {
+
+	comments, groupComments := parseApiASTInfo()
+
+	groups := make(map[string]*MethodGroup)
+
+	var api struct{ api.FullNode }
+	t := reflect.TypeOf(api)
+	for i := 0; i < t.NumMethod(); i++ {
+		m := t.Method(i)
+
+		groupName := methodGroupFromName(m.Name)
+
+		g, ok := groups[groupName]
+		if !ok {
+			g = new(MethodGroup)
+			g.Header = groupComments[groupName]
+			g.GroupName = groupName
+			groups[groupName] = g
+		}
+
+		var args []interface{}
+		ft := m.Func.Type()
+		for j := 2; j < ft.NumIn(); j++ {
+			inp := ft.In(j)
+			args = append(args, exampleValue(inp))
+		}
+
+		v, err := json.Marshal(args)
+		if err != nil {
+			panic(err)
+		}
+
+		outv := exampleValue(ft.Out(0))
+
+		ov, err := json.Marshal(outv)
+		if err != nil {
+			panic(err)
+		}
+
+		g.Methods = append(g.Methods, &Method{
+			Name:            m.Name,
+			Comment:         comments[m.Name],
+			InputExample:    string(v),
+			ResponseExample: string(ov),
+		})
+	}
+
+	var groupslice []*MethodGroup
+	for _, g := range groups {
+		groupslice = append(groupslice, g)
+	}
+
+	sort.Slice(groupslice, func(i, j int) bool {
+		return groupslice[i].GroupName < groupslice[j].GroupName
+	})
+
+	for _, g := range groupslice {
+		g := g
+		fmt.Printf("## %s\n", g.GroupName)
+		fmt.Printf("%s\n\n", g.Header)
+
+		sort.Slice(g.Methods, func(i, j int) bool {
+			return g.Methods[i].Name < g.Methods[j].Name
+		})
+
+		for _, m := range g.Methods {
+			fmt.Printf("### %s\n", m.Name)
+			fmt.Printf("%s\n\n", m.Comment)
+
+			fmt.Printf("Inputs: `%s`\n\n", m.InputExample)
+			fmt.Printf("Response: `%s`\n\n", m.ResponseExample)
+		}
+	}
+}

api/test/deals.go

@@ -0,0 +1,254 @@
+package test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/ipfs/go-cid"
+
+	files "github.com/ipfs/go-ipfs-files"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/ipld/go-car"
+
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
+	dag "github.com/ipfs/go-merkledag"
+	dstest "github.com/ipfs/go-merkledag/test"
+	unixfile "github.com/ipfs/go-unixfs/file"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/node/impl"
+	ipld "github.com/ipfs/go-ipld-format"
+)
+
+func init() {
+	logging.SetAllLoggers(logging.LevelInfo)
+	build.InsecurePoStValidation = true
+}
+
+func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport bool) {
+	_ = os.Setenv("BELLMAN_NO_GPU", "1")
+
+	ctx := context.Background()
+	n, sn := b(t, 1, oneMiner)
+	client := n[0].FullNode.(*impl.FullNodeAPI)
+	miner := sn[0]
+
+	addrinfo, err := client.NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := miner.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Second)
+
+	mine := true
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for mine {
+			time.Sleep(blocktime)
+			if err := sn[0].MineOne(ctx, func(bool) {}); err != nil {
+				t.Error(err)
+			}
+		}
+	}()
+
+	makeDeal(t, ctx, 6, client, miner, carExport)
+
+	mine = false
+	fmt.Println("shutting down mining")
+	<-done
+}
+
+func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
+	_ = os.Setenv("BELLMAN_NO_GPU", "1")
+
+	ctx := context.Background()
+	n, sn := b(t, 1, oneMiner)
+	client := n[0].FullNode.(*impl.FullNodeAPI)
+	miner := sn[0]
+
+	addrinfo, err := client.NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := miner.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Second)
+
+	mine := true
+	done := make(chan struct{})
+
+	go func() {
+		defer close(done)
+		for mine {
+			time.Sleep(blocktime)
+			if err := sn[0].MineOne(ctx, func(bool) {}); err != nil {
+				t.Error(err)
+			}
+		}
+	}()
+
+	makeDeal(t, ctx, 6, client, miner, false)
+	makeDeal(t, ctx, 7, client, miner, false)
+
+	mine = false
+	fmt.Println("shutting down mining")
+	<-done
+}
+
+func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNodeAPI, miner TestStorageNode, carExport bool) {
+	data := make([]byte, 1600)
+	rand.New(rand.NewSource(int64(rseed))).Read(data)
+
+	r := bytes.NewReader(data)
+	fcid, err := client.ClientImportLocal(ctx, r)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Println("FILE CID: ", fcid)
+
+	deal := startDeal(t, ctx, miner, client, fcid)
+
+	// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
+	time.Sleep(time.Second)
+	waitDealSealed(t, ctx, client, deal)
+
+	// Retrieval
+
+	testRetrieval(t, ctx, err, client, fcid, carExport, data)
+}
+
+func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, fcid cid.Cid) *cid.Cid {
+	maddr, err := miner.ActorAddress(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	addr, err := client.WalletDefaultAddress(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	deal, err := client.ClientStartDeal(ctx, &api.StartDealParams{
+		Data:              &storagemarket.DataRef{Root: fcid},
+		Wallet:            addr,
+		Miner:             maddr,
+		EpochPrice:        types.NewInt(1000000),
+		MinBlocksDuration: 100,
+	})
+	if err != nil {
+		t.Fatalf("%+v", err)
+	}
+	return deal
+}
+
+func waitDealSealed(t *testing.T, ctx context.Context, client *impl.FullNodeAPI, deal *cid.Cid) {
+loop:
+	for {
+		di, err := client.ClientGetDealInfo(ctx, *deal)
+		if err != nil {
+			t.Fatal(err)
+		}
+		switch di.State {
+		case storagemarket.StorageDealProposalRejected:
+			t.Fatal("deal rejected")
+		case storagemarket.StorageDealFailing:
+			t.Fatal("deal failed")
+		case storagemarket.StorageDealError:
+			t.Fatal("deal errored", di.Message)
+		case storagemarket.StorageDealActive:
+			fmt.Println("COMPLETE", di)
+			break loop
+		}
+		fmt.Println("Deal state: ", storagemarket.DealStates[di.State])
+		time.Sleep(time.Second / 2)
+	}
+}
+
+func testRetrieval(t *testing.T, ctx context.Context, err error, client *impl.FullNodeAPI, fcid cid.Cid, carExport bool, data []byte) {
+	offers, err := client.ClientFindData(ctx, fcid)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(offers) < 1 {
+		t.Fatal("no offers")
+	}
+
+	rpath, err := ioutil.TempDir("", "lotus-retrieve-test-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(rpath) //nolint:errcheck
+
+	caddr, err := client.WalletDefaultAddress(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ref := &api.FileRef{
+		Path:  filepath.Join(rpath, "ret"),
+		IsCAR: carExport,
+	}
+	err = client.ClientRetrieve(ctx, offers[0].Order(caddr), ref)
+	if err != nil {
+		t.Fatalf("%+v", err)
+	}
+
+	rdata, err := ioutil.ReadFile(filepath.Join(rpath, "ret"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if carExport {
+		rdata = extractCarData(t, ctx, rdata, rpath)
+	}
+
+	if !bytes.Equal(rdata, data) {
+		t.Fatal("wrong data retrieved")
+	}
+}
+
+func extractCarData(t *testing.T, ctx context.Context, rdata []byte, rpath string) []byte {
+	bserv := dstest.Bserv()
+	ch, err := car.LoadCar(bserv.Blockstore(), bytes.NewReader(rdata))
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, err := bserv.GetBlock(ctx, ch.Roots[0])
+	if err != nil {
+		t.Fatal(err)
+	}
+	nd, err := ipld.Decode(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dserv := dag.NewDAGService(bserv)
+	fil, err := unixfile.NewUnixfsFile(ctx, dserv, nd)
+	if err != nil {
+		t.Fatal(err)
+	}
+	outPath := filepath.Join(rpath, "retLoadedCAR")
+	if err := files.WriteTo(fil, outPath); err != nil {
+		t.Fatal(err)
+	}
+	rdata, err = ioutil.ReadFile(outPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return rdata
+}

api/test/mining.go

@@ -0,0 +1,199 @@
+package test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"math/rand"
+	"os"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	logging "github.com/ipfs/go-log/v2"
+
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/node/impl"
+)
+
+var log = logging.Logger("apitest")
+
+func (ts *testSuite) testMining(t *testing.T) {
+	ctx := context.Background()
+	apis, sn := ts.makeNodes(t, 1, oneMiner)
+	api := apis[0]
+
+	h1, err := api.ChainHead(ctx)
+	require.NoError(t, err)
+	require.Equal(t, abi.ChainEpoch(0), h1.Height())
+
+	newHeads, err := api.ChainNotify(ctx)
+	require.NoError(t, err)
+	<-newHeads
+
+	err = sn[0].MineOne(ctx, func(bool) {})
+	require.NoError(t, err)
+
+	<-newHeads
+
+	h2, err := api.ChainHead(ctx)
+	require.NoError(t, err)
+	require.Equal(t, abi.ChainEpoch(1), h2.Height())
+}
+
+func (ts *testSuite) testMiningReal(t *testing.T) {
+	build.InsecurePoStValidation = false
+	defer func() {
+		build.InsecurePoStValidation = true
+	}()
+
+	ctx := context.Background()
+	apis, sn := ts.makeNodes(t, 1, oneMiner)
+	api := apis[0]
+
+	h1, err := api.ChainHead(ctx)
+	require.NoError(t, err)
+	require.Equal(t, abi.ChainEpoch(0), h1.Height())
+
+	newHeads, err := api.ChainNotify(ctx)
+	require.NoError(t, err)
+	<-newHeads
+
+	err = sn[0].MineOne(ctx, func(bool) {})
+	require.NoError(t, err)
+
+	<-newHeads
+
+	h2, err := api.ChainHead(ctx)
+	require.NoError(t, err)
+	require.Equal(t, abi.ChainEpoch(1), h2.Height())
+
+	err = sn[0].MineOne(ctx, func(bool) {})
+	require.NoError(t, err)
+
+	<-newHeads
+
+	h2, err = api.ChainHead(ctx)
+	require.NoError(t, err)
+	require.Equal(t, abi.ChainEpoch(2), h2.Height())
+}
+
+func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExport bool) {
+	_ = os.Setenv("BELLMAN_NO_GPU", "1")
+
+	// test making a deal with a fresh miner, and see if it starts to mine
+
+	ctx := context.Background()
+	n, sn := b(t, 1, []StorageMiner{
+		{Full: 0, Preseal: PresealGenesis},
+		{Full: 0, Preseal: 0}, // TODO: Add support for storage miners on non-first full node
+	})
+	client := n[0].FullNode.(*impl.FullNodeAPI)
+	provider := sn[1]
+	genesisMiner := sn[0]
+
+	addrinfo, err := client.NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := provider.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := genesisMiner.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
+
+	time.Sleep(time.Second)
+
+	data := make([]byte, 600)
+	rand.New(rand.NewSource(5)).Read(data)
+
+	r := bytes.NewReader(data)
+	fcid, err := client.ClientImportLocal(ctx, r)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Println("FILE CID: ", fcid)
+
+	var mine int32 = 1
+	done := make(chan struct{})
+	minedTwo := make(chan struct{})
+
+	go func() {
+		defer close(done)
+
+		prevExpect := 0
+		for atomic.LoadInt32(&mine) != 0 {
+			wait := make(chan int, 2)
+			mdone := func(mined bool) {
+				go func() {
+					n := 0
+					if mined {
+						n = 1
+					}
+					wait <- n
+				}()
+			}
+
+			if err := sn[0].MineOne(ctx, mdone); err != nil {
+				t.Error(err)
+			}
+
+			if err := sn[1].MineOne(ctx, mdone); err != nil {
+				t.Error(err)
+			}
+
+			expect := <-wait
+			expect += <-wait
+
+			time.Sleep(blocktime)
+
+			for {
+				n := 0
+				for i, node := range sn {
+					mb, err := node.MiningBase(ctx)
+					if err != nil {
+						t.Error(err)
+						return
+					}
+
+					if len(mb.Cids()) != expect {
+						log.Warnf("node %d mining base not complete (%d, want %d)", i, len(mb.Cids()), expect)
+						continue
+					}
+					n++
+				}
+				if n == len(sn) {
+					break
+				}
+				time.Sleep(blocktime)
+			}
+
+			if prevExpect == 2 && expect == 2 && minedTwo != nil {
+				close(minedTwo)
+				minedTwo = nil
+			}
+
+			prevExpect = expect
+		}
+	}()
+
+	deal := startDeal(t, ctx, provider, client, fcid)
+
+	// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
+	time.Sleep(time.Second)
+
+	waitDealSealed(t, ctx, client, deal)
+
+	<-minedTwo
+
+	atomic.StoreInt32(&mine, 0)
+	fmt.Println("shutting down mining")
+	<-done
+}

api/test/test.go

@@ -0,0 +1,126 @@
+package test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
+)
+
+type TestNode struct {
+	api.FullNode
+}
+
+type TestStorageNode struct {
+	api.StorageMiner
+
+	MineOne func(context.Context, func(bool)) error
+}
+
+var PresealGenesis = -1
+
+const GenesisPreseals = 2
+
+type StorageMiner struct {
+	Full    int
+	Preseal int
+}
+
+// APIBuilder is a function which is invoked in test suite to provide
+// test nodes and networks
+//
+// storage array defines storage nodes, numbers in the array specify full node
+// index the storage node 'belongs' to
+type APIBuilder func(t *testing.T, nFull int, storage []StorageMiner) ([]TestNode, []TestStorageNode)
+type testSuite struct {
+	makeNodes APIBuilder
+}
+
+// TestApis is the entry point to API test suite
+func TestApis(t *testing.T, b APIBuilder) {
+	ts := testSuite{
+		makeNodes: b,
+	}
+
+	t.Run("version", ts.testVersion)
+	t.Run("id", ts.testID)
+	t.Run("testConnectTwo", ts.testConnectTwo)
+	t.Run("testMining", ts.testMining)
+	t.Run("testMiningReal", ts.testMiningReal)
+}
+
+var oneMiner = []StorageMiner{{Full: 0, Preseal: PresealGenesis}}
+
+func (ts *testSuite) testVersion(t *testing.T) {
+	ctx := context.Background()
+	apis, _ := ts.makeNodes(t, 1, oneMiner)
+	api := apis[0]
+
+	v, err := api.Version(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	require.Equal(t, v.Version, build.BuildVersion)
+}
+
+func (ts *testSuite) testID(t *testing.T) {
+	ctx := context.Background()
+	apis, _ := ts.makeNodes(t, 1, oneMiner)
+	api := apis[0]
+
+	id, err := api.ID(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Regexp(t, "^12", id.Pretty())
+}
+
+func (ts *testSuite) testConnectTwo(t *testing.T) {
+	ctx := context.Background()
+	apis, _ := ts.makeNodes(t, 2, oneMiner)
+
+	p, err := apis[0].NetPeers(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(p) != 0 {
+		t.Error("Node 0 has a peer")
+	}
+
+	p, err = apis[1].NetPeers(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(p) != 0 {
+		t.Error("Node 1 has a peer")
+	}
+
+	addrs, err := apis[1].NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := apis[0].NetConnect(ctx, addrs); err != nil {
+		t.Fatal(err)
+	}
+
+	p, err = apis[0].NetPeers(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(p) != 1 {
+		t.Error("Node 0 doesn't have 1 peer")
+	}
+
+	p, err = apis[1].NetPeers(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(p) != 1 {
+		t.Error("Node 0 doesn't have 1 peer")
+	}
+}

api/test/window_post.go

@@ -0,0 +1,169 @@
+package test
+
+import (
+	"context"
+	"fmt"
+	"github.com/filecoin-project/lotus/api"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	miner2 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
+	sealing "github.com/filecoin-project/storage-fsm"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/node/impl"
+)
+
+func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) {
+	os.Setenv("BELLMAN_NO_GPU", "1")
+
+	ctx := context.Background()
+	n, sn := b(t, 1, oneMiner)
+	client := n[0].FullNode.(*impl.FullNodeAPI)
+	miner := sn[0]
+
+	addrinfo, err := client.NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := miner.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Second)
+
+	mine := true
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for mine {
+			time.Sleep(blocktime)
+			if err := sn[0].MineOne(ctx, func(bool) {}); err != nil {
+				t.Error(err)
+			}
+		}
+	}()
+
+	pledgeSectors(t, ctx, miner, nSectors)
+
+	mine = false
+	<-done
+}
+
+func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n int) {
+	for i := 0; i < n; i++ {
+		err := miner.PledgeSector(ctx)
+		require.NoError(t, err)
+	}
+
+	for {
+		s, err := miner.SectorsList(ctx) // Note - the test builder doesn't import genesis sectors into FSM
+		require.NoError(t, err)
+		fmt.Printf("Sectors: %d\n", len(s))
+		if len(s) >= n {
+			break
+		}
+
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	fmt.Printf("All sectors is fsm\n")
+
+	s, err := miner.SectorsList(ctx)
+	require.NoError(t, err)
+
+	toCheck := map[abi.SectorNumber]struct{}{}
+	for _, number := range s {
+		toCheck[number] = struct{}{}
+	}
+
+	for len(toCheck) > 0 {
+		for n := range toCheck {
+			st, err := miner.SectorsStatus(ctx, n)
+			require.NoError(t, err)
+			if st.State == api.SectorState(sealing.Proving) {
+				delete(toCheck, n)
+			}
+			if strings.Contains(string(st.State), "Fail") {
+				t.Fatal("sector in a failed state", st.State)
+			}
+		}
+
+		time.Sleep(100 * time.Millisecond)
+		fmt.Printf("WaitSeal: %d\n", len(s))
+	}
+}
+
+func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) {
+	os.Setenv("BELLMAN_NO_GPU", "1")
+
+	ctx := context.Background()
+	n, sn := b(t, 1, oneMiner)
+	client := n[0].FullNode.(*impl.FullNodeAPI)
+	miner := sn[0]
+
+	addrinfo, err := client.NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := miner.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Second)
+
+	mine := true
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for mine {
+			time.Sleep(blocktime)
+			if err := sn[0].MineOne(ctx, func(bool) {}); err != nil {
+				t.Error(err)
+			}
+		}
+	}()
+
+	pledgeSectors(t, ctx, miner, nSectors)
+
+	maddr, err := miner.ActorAddress(ctx)
+	require.NoError(t, err)
+
+	di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+
+	fmt.Printf("Running one proving periods\n")
+
+	for {
+		head, err := client.ChainHead(ctx)
+		require.NoError(t, err)
+
+		if head.Height() > di.PeriodStart+(miner2.WPoStProvingPeriod)+2 {
+			break
+		}
+
+		if head.Height()%100 == 0 {
+			fmt.Printf("@%d\n", head.Height())
+		}
+		time.Sleep(blocktime)
+	}
+
+	p, err := client.StateMinerPower(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+
+	ssz, err := miner.ActorSectorSize(ctx, maddr)
+	require.NoError(t, err)
+
+	require.Equal(t, p.MinerPower, p.TotalPower)
+	require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*uint64(nSectors+GenesisPreseals)))
+
+	// TODO: Inject faults here
+
+	mine = false
+	<-done
+}

api/types.go

@@ -2,18 +2,8 @@ package api
 
 import (
 	"encoding/json"
-	"fmt"
-	"time"
-
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/lotus/chain/types"
-
-	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/ipfs/go-cid"
 
 	"github.com/libp2p/go-libp2p-core/peer"
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	ma "github.com/multiformats/go-multiaddr"
 )
 
@@ -47,151 +37,5 @@ type ObjStat struct {
 
 type PubsubScore struct {
 	ID    peer.ID
-	Score *pubsub.PeerScoreSnapshot
+	Score float64
-}
-
-type MessageSendSpec struct {
-	MaxFee abi.TokenAmount
-}
-
-type DataTransferChannel struct {
-	TransferID  datatransfer.TransferID
-	Status      datatransfer.Status
-	BaseCID     cid.Cid
-	IsInitiator bool
-	IsSender    bool
-	Voucher     string
-	Message     string
-	OtherPeer   peer.ID
-	Transferred uint64
-	Stages      *datatransfer.ChannelStages
-}
-
-// NewDataTransferChannel constructs an API DataTransferChannel type from full channel state snapshot and a host id
-func NewDataTransferChannel(hostID peer.ID, channelState datatransfer.ChannelState) DataTransferChannel {
-	channel := DataTransferChannel{
-		TransferID: channelState.TransferID(),
-		Status:     channelState.Status(),
-		BaseCID:    channelState.BaseCID(),
-		IsSender:   channelState.Sender() == hostID,
-		Message:    channelState.Message(),
-	}
-	stringer, ok := channelState.Voucher().(fmt.Stringer)
-	if ok {
-		channel.Voucher = stringer.String()
-	} else {
-		voucherJSON, err := json.Marshal(channelState.Voucher())
-		if err != nil {
-			channel.Voucher = fmt.Errorf("Voucher Serialization: %w", err).Error()
-		} else {
-			channel.Voucher = string(voucherJSON)
-		}
-	}
-	if channel.IsSender {
-		channel.IsInitiator = !channelState.IsPull()
-		channel.Transferred = channelState.Sent()
-		channel.OtherPeer = channelState.Recipient()
-	} else {
-		channel.IsInitiator = channelState.IsPull()
-		channel.Transferred = channelState.Received()
-		channel.OtherPeer = channelState.Sender()
-	}
-	return channel
-}
-
-type NetBlockList struct {
-	Peers     []peer.ID
-	IPAddrs   []string
-	IPSubnets []string
-}
-
-type ExtendedPeerInfo struct {
-	ID          peer.ID
-	Agent       string
-	Addrs       []string
-	Protocols   []string
-	ConnMgrMeta *ConnMgrInfo
-}
-
-type ConnMgrInfo struct {
-	FirstSeen time.Time
-	Value     int
-	Tags      map[string]int
-	Conns     map[string]time.Time
-}
-
-type NodeStatus struct {
-	SyncStatus  NodeSyncStatus
-	PeerStatus  NodePeerStatus
-	ChainStatus NodeChainStatus
-}
-
-type NodeSyncStatus struct {
-	Epoch  uint64
-	Behind uint64
-}
-
-type NodePeerStatus struct {
-	PeersToPublishMsgs   int
-	PeersToPublishBlocks int
-}
-
-type NodeChainStatus struct {
-	BlocksPerTipsetLast100      float64
-	BlocksPerTipsetLastFinality float64
-}
-
-type CheckStatusCode int
-
-//go:generate go run golang.org/x/tools/cmd/stringer -type=CheckStatusCode -trimprefix=CheckStatus
-const (
-	_ CheckStatusCode = iota
-	// Message Checks
-	CheckStatusMessageSerialize
-	CheckStatusMessageSize
-	CheckStatusMessageValidity
-	CheckStatusMessageMinGas
-	CheckStatusMessageMinBaseFee
-	CheckStatusMessageBaseFee
-	CheckStatusMessageBaseFeeLowerBound
-	CheckStatusMessageBaseFeeUpperBound
-	CheckStatusMessageGetStateNonce
-	CheckStatusMessageNonce
-	CheckStatusMessageGetStateBalance
-	CheckStatusMessageBalance
-)
-
-type CheckStatus struct {
-	Code CheckStatusCode
-	OK   bool
-	Err  string
-	Hint map[string]interface{}
-}
-
-type MessageCheckStatus struct {
-	Cid cid.Cid
-	CheckStatus
-}
-
-type MessagePrototype struct {
-	Message    types.Message
-	ValidNonce bool
-}
-
-type RetrievalInfo struct {
-	PayloadCID   cid.Cid
-	ID           retrievalmarket.DealID
-	PieceCID     *cid.Cid
-	PricePerByte abi.TokenAmount
-	UnsealPrice  abi.TokenAmount
-
-	Status        retrievalmarket.DealStatus
-	Message       string // more information about deal state, particularly errors
-	Provider      peer.ID
-	BytesReceived uint64
-	BytesPaidFor  uint64
-	TotalPaid     abi.TokenAmount
-
-	TransferChannelID *datatransfer.ChannelID
-	DataTransfer      *DataTransferChannel
 }

api/types/actors.go

@@ -1,5 +0,0 @@
-package apitypes
-
-import "github.com/filecoin-project/go-state-types/network"
-
-type NetworkVersion = network.Version

api/types/openrpc.go

@@ -1,3 +0,0 @@
-package apitypes
-
-type OpenRPCDocument map[string]interface{}

api/utils.go

@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
 )
 
 type SignFunc = func(context.Context, []byte) (*crypto.Signature, error)

api/v0api/full.go

@@ -1,710 +0,0 @@
-package v0api
-
-import (
-	"context"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-bitfield"
-	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/go-fil-markets/storagemarket"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/crypto"
-	"github.com/filecoin-project/go-state-types/dline"
-	"github.com/filecoin-project/lotus/node/repo/importmgr"
-	"github.com/ipfs/go-cid"
-	"github.com/libp2p/go-libp2p-core/peer"
-
-	"github.com/filecoin-project/lotus/api"
-	apitypes "github.com/filecoin-project/lotus/api/types"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
-	"github.com/filecoin-project/lotus/chain/types"
-	marketevents "github.com/filecoin-project/lotus/markets/loggers"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-)
-
-//go:generate go run github.com/golang/mock/mockgen -destination=v0mocks/mock_full.go -package=v0mocks . FullNode
-
-//                       MODIFYING THE API INTERFACE
-//
-// NOTE: This is the V0 (Stable) API - when adding methods to this interface,
-// you'll need to make sure they are also present on the V1 (Unstable) API
-//
-// This API is implemented in `v1_wrapper.go` as a compatibility layer backed
-// by the V1 api
-//
-// When adding / changing methods in this file:
-// * Do the change here
-// * Adjust implementation in `node/impl/`
-// * Run `make gen` - this will:
-//  * Generate proxy structs
-//  * Generate mocks
-//  * Generate markdown docs
-//  * Generate openrpc blobs
-
-// FullNode API is a low-level interface to the Filecoin network full node
-type FullNode interface {
-	Common
-
-	// MethodGroup: Chain
-	// The Chain method group contains methods for interacting with the
-	// blockchain, but that do not require any form of state computation.
-
-	// ChainNotify returns channel with chain head updates.
-	// First message is guaranteed to be of len == 1, and type == 'current'.
-	ChainNotify(context.Context) (<-chan []*api.HeadChange, error) //perm:read
-
-	// ChainHead returns the current head of the chain.
-	ChainHead(context.Context) (*types.TipSet, error) //perm:read
-
-	// ChainGetRandomnessFromTickets is used to sample the chain for randomness.
-	ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) //perm:read
-
-	// ChainGetRandomnessFromBeacon is used to sample the beacon for randomness.
-	ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) //perm:read
-
-	// ChainGetBlock returns the block specified by the given CID.
-	ChainGetBlock(context.Context, cid.Cid) (*types.BlockHeader, error) //perm:read
-	// ChainGetTipSet returns the tipset specified by the given TipSetKey.
-	ChainGetTipSet(context.Context, types.TipSetKey) (*types.TipSet, error) //perm:read
-
-	// ChainGetBlockMessages returns messages stored in the specified block.
-	//
-	// Note: If there are multiple blocks in a tipset, it's likely that some
-	// messages will be duplicated. It's also possible for blocks in a tipset to have
-	// different messages from the same sender at the same nonce. When that happens,
-	// only the first message (in a block with lowest ticket) will be considered
-	// for execution
-	//
-	// NOTE: THIS METHOD SHOULD ONLY BE USED FOR GETTING MESSAGES IN A SPECIFIC BLOCK
-	//
-	// DO NOT USE THIS METHOD TO GET MESSAGES INCLUDED IN A TIPSET
-	// Use ChainGetParentMessages, which will perform correct message deduplication
-	ChainGetBlockMessages(ctx context.Context, blockCid cid.Cid) (*api.BlockMessages, error) //perm:read
-
-	// ChainGetParentReceipts returns receipts for messages in parent tipset of
-	// the specified block. The receipts in the list returned is one-to-one with the
-	// messages returned by a call to ChainGetParentMessages with the same blockCid.
-	ChainGetParentReceipts(ctx context.Context, blockCid cid.Cid) ([]*types.MessageReceipt, error) //perm:read
-
-	// ChainGetParentMessages returns messages stored in parent tipset of the
-	// specified block.
-	ChainGetParentMessages(ctx context.Context, blockCid cid.Cid) ([]api.Message, error) //perm:read
-
-	// ChainGetMessagesInTipset returns message stores in current tipset
-	ChainGetMessagesInTipset(ctx context.Context, tsk types.TipSetKey) ([]api.Message, error) //perm:read
-
-	// ChainGetTipSetByHeight looks back for a tipset at the specified epoch.
-	// If there are no blocks at the specified epoch, a tipset at an earlier epoch
-	// will be returned.
-	ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) //perm:read
-
-	// ChainReadObj reads ipld nodes referenced by the specified CID from chain
-	// blockstore and returns raw bytes.
-	ChainReadObj(context.Context, cid.Cid) ([]byte, error) //perm:read
-
-	// ChainDeleteObj deletes node referenced by the given CID
-	ChainDeleteObj(context.Context, cid.Cid) error //perm:admin
-
-	// ChainHasObj checks if a given CID exists in the chain blockstore.
-	ChainHasObj(context.Context, cid.Cid) (bool, error) //perm:read
-
-	// ChainStatObj returns statistics about the graph referenced by 'obj'.
-	// If 'base' is also specified, then the returned stat will be a diff
-	// between the two objects.
-	ChainStatObj(ctx context.Context, obj cid.Cid, base cid.Cid) (api.ObjStat, error) //perm:read
-
-	// ChainSetHead forcefully sets current chain head. Use with caution.
-	ChainSetHead(context.Context, types.TipSetKey) error //perm:admin
-
-	// ChainGetGenesis returns the genesis tipset.
-	ChainGetGenesis(context.Context) (*types.TipSet, error) //perm:read
-
-	// ChainTipSetWeight computes weight for the specified tipset.
-	ChainTipSetWeight(context.Context, types.TipSetKey) (types.BigInt, error) //perm:read
-	ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error)      //perm:read
-
-	// ChainGetMessage reads a message referenced by the specified CID from the
-	// chain blockstore.
-	ChainGetMessage(context.Context, cid.Cid) (*types.Message, error) //perm:read
-
-	// ChainGetPath returns a set of revert/apply operations needed to get from
-	// one tipset to another, for example:
-	//```
-	//        to
-	//         ^
-	// from   tAA
-	//   ^     ^
-	// tBA    tAB
-	//  ^---*--^
-	//      ^
-	//     tRR
-	//```
-	// Would return `[revert(tBA), apply(tAB), apply(tAA)]`
-	ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*api.HeadChange, error) //perm:read
-
-	// ChainExport returns a stream of bytes with CAR dump of chain data.
-	// The exported chain data includes the header chain from the given tipset
-	// back to genesis, the entire genesis state, and the most recent 'nroots'
-	// state trees.
-	// If oldmsgskip is set, messages from before the requested roots are also not included.
-	ChainExport(ctx context.Context, nroots abi.ChainEpoch, oldmsgskip bool, tsk types.TipSetKey) (<-chan []byte, error) //perm:read
-
-	// MethodGroup: Beacon
-	// The Beacon method group contains methods for interacting with the random beacon (DRAND)
-
-	// BeaconGetEntry returns the beacon entry for the given filecoin epoch. If
-	// the entry has not yet been produced, the call will block until the entry
-	// becomes available
-	BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) //perm:read
-
-	// GasEstimateFeeCap estimates gas fee cap
-	GasEstimateFeeCap(context.Context, *types.Message, int64, types.TipSetKey) (types.BigInt, error) //perm:read
-
-	// GasEstimateGasLimit estimates gas used by the message and returns it.
-	// It fails if message fails to execute.
-	GasEstimateGasLimit(context.Context, *types.Message, types.TipSetKey) (int64, error) //perm:read
-
-	// GasEstimateGasPremium estimates what gas price should be used for a
-	// message to have high likelihood of inclusion in `nblocksincl` epochs.
-
-	GasEstimateGasPremium(_ context.Context, nblocksincl uint64,
-		sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) //perm:read
-
-	// GasEstimateMessageGas estimates gas values for unset message gas fields
-	GasEstimateMessageGas(context.Context, *types.Message, *api.MessageSendSpec, types.TipSetKey) (*types.Message, error) //perm:read
-
-	// MethodGroup: Sync
-	// The Sync method group contains methods for interacting with and
-	// observing the lotus sync service.
-
-	// SyncState returns the current status of the lotus sync system.
-	SyncState(context.Context) (*api.SyncState, error) //perm:read
-
-	// SyncSubmitBlock can be used to submit a newly created block to the.
-	// network through this node
-	SyncSubmitBlock(ctx context.Context, blk *types.BlockMsg) error //perm:write
-
-	// SyncIncomingBlocks returns a channel streaming incoming, potentially not
-	// yet synced block headers.
-	SyncIncomingBlocks(ctx context.Context) (<-chan *types.BlockHeader, error) //perm:read
-
-	// SyncCheckpoint marks a blocks as checkpointed, meaning that it won't ever fork away from it.
-	SyncCheckpoint(ctx context.Context, tsk types.TipSetKey) error //perm:admin
-
-	// SyncMarkBad marks a blocks as bad, meaning that it won't ever by synced.
-	// Use with extreme caution.
-	SyncMarkBad(ctx context.Context, bcid cid.Cid) error //perm:admin
-
-	// SyncUnmarkBad unmarks a blocks as bad, making it possible to be validated and synced again.
-	SyncUnmarkBad(ctx context.Context, bcid cid.Cid) error //perm:admin
-
-	// SyncUnmarkAllBad purges bad block cache, making it possible to sync to chains previously marked as bad
-	SyncUnmarkAllBad(ctx context.Context) error //perm:admin
-
-	// SyncCheckBad checks if a block was marked as bad, and if it was, returns
-	// the reason.
-	SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error) //perm:read
-
-	// SyncValidateTipset indicates whether the provided tipset is valid or not
-	SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) (bool, error) //perm:read
-
-	// MethodGroup: Mpool
-	// The Mpool methods are for interacting with the message pool. The message pool
-	// manages all incoming and outgoing 'messages' going over the network.
-
-	// MpoolPending returns pending mempool messages.
-	MpoolPending(context.Context, types.TipSetKey) ([]*types.SignedMessage, error) //perm:read
-
-	// MpoolSelect returns a list of pending messages for inclusion in the next block
-	MpoolSelect(context.Context, types.TipSetKey, float64) ([]*types.SignedMessage, error) //perm:read
-
-	// MpoolPush pushes a signed message to mempool.
-	MpoolPush(context.Context, *types.SignedMessage) (cid.Cid, error) //perm:write
-
-	// MpoolPushUntrusted pushes a signed message to mempool from untrusted sources.
-	MpoolPushUntrusted(context.Context, *types.SignedMessage) (cid.Cid, error) //perm:write
-
-	// MpoolPushMessage atomically assigns a nonce, signs, and pushes a message
-	// to mempool.
-	// maxFee is only used when GasFeeCap/GasPremium fields aren't specified
-	//
-	// When maxFee is set to 0, MpoolPushMessage will guess appropriate fee
-	// based on current chain conditions
-	MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) //perm:sign
-
-	// MpoolBatchPush batch pushes a signed message to mempool.
-	MpoolBatchPush(context.Context, []*types.SignedMessage) ([]cid.Cid, error) //perm:write
-
-	// MpoolBatchPushUntrusted batch pushes a signed message to mempool from untrusted sources.
-	MpoolBatchPushUntrusted(context.Context, []*types.SignedMessage) ([]cid.Cid, error) //perm:write
-
-	// MpoolBatchPushMessage batch pushes a unsigned message to mempool.
-	MpoolBatchPushMessage(context.Context, []*types.Message, *api.MessageSendSpec) ([]*types.SignedMessage, error) //perm:sign
-
-	// MpoolGetNonce gets next nonce for the specified sender.
-	// Note that this method may not be atomic. Use MpoolPushMessage instead.
-	MpoolGetNonce(context.Context, address.Address) (uint64, error) //perm:read
-	MpoolSub(context.Context) (<-chan api.MpoolUpdate, error)       //perm:read
-
-	// MpoolClear clears pending messages from the mpool
-	MpoolClear(context.Context, bool) error //perm:write
-
-	// MpoolGetConfig returns (a copy of) the current mpool config
-	MpoolGetConfig(context.Context) (*types.MpoolConfig, error) //perm:read
-	// MpoolSetConfig sets the mpool config to (a copy of) the supplied config
-	MpoolSetConfig(context.Context, *types.MpoolConfig) error //perm:admin
-
-	// MethodGroup: Miner
-
-	MinerGetBaseInfo(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*api.MiningBaseInfo, error) //perm:read
-	MinerCreateBlock(context.Context, *api.BlockTemplate) (*types.BlockMsg, error)                                   //perm:write
-
-	// // UX ?
-
-	// MethodGroup: Wallet
-
-	// WalletNew creates a new address in the wallet with the given sigType.
-	// Available key types: bls, secp256k1, secp256k1-ledger
-	// Support for numerical types: 1 - secp256k1, 2 - BLS is deprecated
-	WalletNew(context.Context, types.KeyType) (address.Address, error) //perm:write
-	// WalletHas indicates whether the given address is in the wallet.
-	WalletHas(context.Context, address.Address) (bool, error) //perm:write
-	// WalletList lists all the addresses in the wallet.
-	WalletList(context.Context) ([]address.Address, error) //perm:write
-	// WalletBalance returns the balance of the given address at the current head of the chain.
-	WalletBalance(context.Context, address.Address) (types.BigInt, error) //perm:read
-	// WalletSign signs the given bytes using the given address.
-	WalletSign(context.Context, address.Address, []byte) (*crypto.Signature, error) //perm:sign
-	// WalletSignMessage signs the given message using the given address.
-	WalletSignMessage(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) //perm:sign
-	// WalletVerify takes an address, a signature, and some bytes, and indicates whether the signature is valid.
-	// The address does not have to be in the wallet.
-	WalletVerify(context.Context, address.Address, []byte, *crypto.Signature) (bool, error) //perm:read
-	// WalletDefaultAddress returns the address marked as default in the wallet.
-	WalletDefaultAddress(context.Context) (address.Address, error) //perm:write
-	// WalletSetDefault marks the given address as as the default one.
-	WalletSetDefault(context.Context, address.Address) error //perm:write
-	// WalletExport returns the private key of an address in the wallet.
-	WalletExport(context.Context, address.Address) (*types.KeyInfo, error) //perm:admin
-	// WalletImport receives a KeyInfo, which includes a private key, and imports it into the wallet.
-	WalletImport(context.Context, *types.KeyInfo) (address.Address, error) //perm:admin
-	// WalletDelete deletes an address from the wallet.
-	WalletDelete(context.Context, address.Address) error //perm:admin
-	// WalletValidateAddress validates whether a given string can be decoded as a well-formed address
-	WalletValidateAddress(context.Context, string) (address.Address, error) //perm:read
-
-	// Other
-
-	// MethodGroup: Client
-	// The Client methods all have to do with interacting with the storage and
-	// retrieval markets as a client
-
-	// ClientImport imports file under the specified path into filestore.
-	ClientImport(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) //perm:admin
-	// ClientRemoveImport removes file import
-	ClientRemoveImport(ctx context.Context, importID importmgr.ImportID) error //perm:admin
-	// ClientStartDeal proposes a deal with a miner.
-	ClientStartDeal(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) //perm:admin
-	// ClientStatelessDeal fire-and-forget-proposes an offline deal to a miner without subsequent tracking.
-	ClientStatelessDeal(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) //perm:write
-	// ClientGetDealInfo returns the latest information about a given deal.
-	ClientGetDealInfo(context.Context, cid.Cid) (*api.DealInfo, error) //perm:read
-	// ClientListDeals returns information about the deals made by the local client.
-	ClientListDeals(ctx context.Context) ([]api.DealInfo, error) //perm:write
-	// ClientGetDealUpdates returns the status of updated deals
-	ClientGetDealUpdates(ctx context.Context) (<-chan api.DealInfo, error) //perm:write
-	// ClientGetDealStatus returns status given a code
-	ClientGetDealStatus(ctx context.Context, statusCode uint64) (string, error) //perm:read
-	// ClientHasLocal indicates whether a certain CID is locally stored.
-	ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) //perm:write
-	// ClientFindData identifies peers that have a certain file, and returns QueryOffers (one per peer).
-	ClientFindData(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]api.QueryOffer, error) //perm:read
-	// ClientMinerQueryOffer returns a QueryOffer for the specific miner and file.
-	ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error) //perm:read
-	// ClientRetrieve initiates the retrieval of a file, as specified in the order.
-	ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error //perm:admin
-	// ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel
-	// of status updates.
-	ClientRetrieveWithEvents(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin
-	// ClientQueryAsk returns a signed StorageAsk from the specified miner.
-	// ClientListRetrievals returns information about retrievals made by the local client
-	ClientListRetrievals(ctx context.Context) ([]api.RetrievalInfo, error) //perm:write
-	// ClientGetRetrievalUpdates returns status of updated retrieval deals
-	ClientGetRetrievalUpdates(ctx context.Context) (<-chan api.RetrievalInfo, error)                         //perm:write
-	ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error) //perm:read
-	// ClientCalcCommP calculates the CommP and data size of the specified CID
-	ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCIDSize, error) //perm:read
-	// ClientCalcCommP calculates the CommP for a specified file
-	ClientCalcCommP(ctx context.Context, inpath string) (*api.CommPRet, error) //perm:write
-	// ClientGenCar generates a CAR file for the specified file.
-	ClientGenCar(ctx context.Context, ref api.FileRef, outpath string) error //perm:write
-	// ClientDealSize calculates real deal data size
-	ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, error) //perm:read
-	// ClientListTransfers returns the status of all ongoing transfers of data
-	ClientListDataTransfers(ctx context.Context) ([]api.DataTransferChannel, error)        //perm:write
-	ClientDataTransferUpdates(ctx context.Context) (<-chan api.DataTransferChannel, error) //perm:write
-	// ClientRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer
-	ClientRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
-	// ClientCancelDataTransfer cancels a data transfer with the given transfer ID and other peer
-	ClientCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
-	// ClientRetrieveTryRestartInsufficientFunds attempts to restart stalled retrievals on a given payment channel
-	// which are stuck due to insufficient funds
-	ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error //perm:write
-
-	// ClientCancelRetrievalDeal cancels an ongoing retrieval deal based on DealID
-	ClientCancelRetrievalDeal(ctx context.Context, dealid retrievalmarket.DealID) error //perm:write
-
-	// ClientUnimport removes references to the specified file from filestore
-	//ClientUnimport(path string)
-
-	// ClientListImports lists imported files and their root CIDs
-	ClientListImports(ctx context.Context) ([]api.Import, error) //perm:write
-
-	//ClientListAsks() []Ask
-
-	// MethodGroup: State
-	// The State methods are used to query, inspect, and interact with chain state.
-	// Most methods take a TipSetKey as a parameter. The state looked up is the parent state of the tipset.
-	// A nil TipSetKey can be provided as a param, this will cause the heaviest tipset in the chain to be used.
-
-	// StateCall runs the given message and returns its result without any persisted changes.
-	//
-	// StateCall applies the message to the tipset's parent state. The
-	// message is not applied on-top-of the messages in the passed-in
-	// tipset.
-	StateCall(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) //perm:read
-	// StateReplay replays a given message, assuming it was included in a block in the specified tipset.
-	//
-	// If a tipset key is provided, and a replacing message is found on chain,
-	// the method will return an error saying that the message wasn't found
-	//
-	// If no tipset key is provided, the appropriate tipset is looked up, and if
-	// the message was gas-repriced, the on-chain message will be replayed - in
-	// that case the returned InvocResult.MsgCid will not match the Cid param
-	//
-	// If the caller wants to ensure that exactly the requested message was executed,
-	// they MUST check that InvocResult.MsgCid is equal to the provided Cid.
-	// Without this check both the requested and original message may appear as
-	// successfully executed on-chain, which may look like a double-spend.
-	//
-	// A replacing message is a message with a different CID, any of Gas values, and
-	// different signature, but with all other parameters matching (source/destination,
-	// nonce, params, etc.)
-	StateReplay(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) //perm:read
-	// StateGetActor returns the indicated actor's nonce and balance.
-	StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) //perm:read
-	// StateReadState returns the indicated actor's state.
-	StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) //perm:read
-	// StateListMessages looks back and returns all messages with a matching to or from address, stopping at the given height.
-	StateListMessages(ctx context.Context, match *api.MessageMatch, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) //perm:read
-	// StateDecodeParams attempts to decode the provided params, based on the recipient actor address and method number.
-	StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error) //perm:read
-
-	// StateNetworkName returns the name of the network the node is synced to
-	StateNetworkName(context.Context) (dtypes.NetworkName, error) //perm:read
-	// StateMinerSectors returns info about the given miner's sectors. If the filter bitfield is nil, all sectors are included.
-	StateMinerSectors(context.Context, address.Address, *bitfield.BitField, types.TipSetKey) ([]*miner.SectorOnChainInfo, error) //perm:read
-	// StateMinerActiveSectors returns info about sectors that a given miner is actively proving.
-	StateMinerActiveSectors(context.Context, address.Address, types.TipSetKey) ([]*miner.SectorOnChainInfo, error) //perm:read
-	// StateMinerProvingDeadline calculates the deadline at some epoch for a proving period
-	// and returns the deadline-related calculations.
-	StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) //perm:read
-	// StateMinerPower returns the power of the indicated miner
-	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) //perm:read
-	// StateMinerInfo returns info about the indicated miner
-	StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) //perm:read
-	// StateMinerDeadlines returns all the proving deadlines for the given miner
-	StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]api.Deadline, error) //perm:read
-	// StateMinerPartitions returns all partitions in the specified deadline
-	StateMinerPartitions(ctx context.Context, m address.Address, dlIdx uint64, tsk types.TipSetKey) ([]api.Partition, error) //perm:read
-	// StateMinerFaults returns a bitfield indicating the faulty sectors of the given miner
-	StateMinerFaults(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) //perm:read
-	// StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset
-	StateAllMinerFaults(ctx context.Context, lookback abi.ChainEpoch, ts types.TipSetKey) ([]*api.Fault, error) //perm:read
-	// StateMinerRecoveries returns a bitfield indicating the recovering sectors of the given miner
-	StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) //perm:read
-	// StateMinerInitialPledgeCollateral returns the precommit deposit for the specified miner's sector
-	StateMinerPreCommitDepositForPower(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) //perm:read
-	// StateMinerInitialPledgeCollateral returns the initial pledge collateral for the specified miner's sector
-	StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) //perm:read
-	// StateMinerAvailableBalance returns the portion of a miner's balance that can be withdrawn or spent
-	StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) //perm:read
-	// StateMinerSectorAllocated checks if a sector is allocated
-	StateMinerSectorAllocated(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (bool, error) //perm:read
-	// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector
-	StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) //perm:read
-	// StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found
-	// NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate
-	// expiration epoch
-	StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) //perm:read
-	// StateSectorExpiration returns epoch at which given sector will expire
-	StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error) //perm:read
-	// StateSectorPartition finds deadline/partition with the specified sector
-	StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) //perm:read
-	// StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed
-	//
-	// NOTE: If a replacing message is found on chain, this method will return
-	// a MsgLookup for the replacing message - the MsgLookup.Message will be a different
-	// CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
-	// result of the execution of the replacing message.
-	//
-	// If the caller wants to ensure that exactly the requested message was executed,
-	// they MUST check that MsgLookup.Message is equal to the provided 'cid'.
-	// Without this check both the requested and original message may appear as
-	// successfully executed on-chain, which may look like a double-spend.
-	//
-	// A replacing message is a message with a different CID, any of Gas values, and
-	// different signature, but with all other parameters matching (source/destination,
-	// nonce, params, etc.)
-	StateSearchMsg(context.Context, cid.Cid) (*api.MsgLookup, error) //perm:read
-	// StateSearchMsgLimited looks back up to limit epochs in the chain for a message, and returns its receipt and the tipset where it was executed
-	//
-	// NOTE: If a replacing message is found on chain, this method will return
-	// a MsgLookup for the replacing message - the MsgLookup.Message will be a different
-	// CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
-	// result of the execution of the replacing message.
-	//
-	// If the caller wants to ensure that exactly the requested message was executed,
-	// they MUST check that MsgLookup.Message is equal to the provided 'cid'.
-	// Without this check both the requested and original message may appear as
-	// successfully executed on-chain, which may look like a double-spend.
-	//
-	// A replacing message is a message with a different CID, any of Gas values, and
-	// different signature, but with all other parameters matching (source/destination,
-	// nonce, params, etc.)
-	StateSearchMsgLimited(ctx context.Context, msg cid.Cid, limit abi.ChainEpoch) (*api.MsgLookup, error) //perm:read
-	// StateWaitMsg looks back in the chain for a message. If not found, it blocks until the
-	// message arrives on chain, and gets to the indicated confidence depth.
-	//
-	// NOTE: If a replacing message is found on chain, this method will return
-	// a MsgLookup for the replacing message - the MsgLookup.Message will be a different
-	// CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
-	// result of the execution of the replacing message.
-	//
-	// If the caller wants to ensure that exactly the requested message was executed,
-	// they MUST check that MsgLookup.Message is equal to the provided 'cid'.
-	// Without this check both the requested and original message may appear as
-	// successfully executed on-chain, which may look like a double-spend.
-	//
-	// A replacing message is a message with a different CID, any of Gas values, and
-	// different signature, but with all other parameters matching (source/destination,
-	// nonce, params, etc.)
-	StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) //perm:read
-	// StateWaitMsgLimited looks back up to limit epochs in the chain for a message.
-	// If not found, it blocks until the message arrives on chain, and gets to the
-	// indicated confidence depth.
-	//
-	// NOTE: If a replacing message is found on chain, this method will return
-	// a MsgLookup for the replacing message - the MsgLookup.Message will be a different
-	// CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
-	// result of the execution of the replacing message.
-	//
-	// If the caller wants to ensure that exactly the requested message was executed,
-	// they MUST check that MsgLookup.Message is equal to the provided 'cid'.
-	// Without this check both the requested and original message may appear as
-	// successfully executed on-chain, which may look like a double-spend.
-	//
-	// A replacing message is a message with a different CID, any of Gas values, and
-	// different signature, but with all other parameters matching (source/destination,
-	// nonce, params, etc.)
-	StateWaitMsgLimited(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch) (*api.MsgLookup, error) //perm:read
-	// StateListMiners returns the addresses of every miner that has claimed power in the Power Actor
-	StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error) //perm:read
-	// StateListActors returns the addresses of every actor in the state
-	StateListActors(context.Context, types.TipSetKey) ([]address.Address, error) //perm:read
-	// StateMarketBalance looks up the Escrow and Locked balances of the given address in the Storage Market
-	StateMarketBalance(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error) //perm:read
-	// StateMarketParticipants returns the Escrow and Locked balances of every participant in the Storage Market
-	StateMarketParticipants(context.Context, types.TipSetKey) (map[string]api.MarketBalance, error) //perm:read
-	// StateMarketDeals returns information about every deal in the Storage Market
-	StateMarketDeals(context.Context, types.TipSetKey) (map[string]api.MarketDeal, error) //perm:read
-	// StateMarketStorageDeal returns information about the indicated deal
-	StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error) //perm:read
-	// StateLookupID retrieves the ID address of the given address
-	StateLookupID(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
-	// StateAccountKey returns the public key address of the given ID address
-	StateAccountKey(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
-	// StateChangedActors returns all the actors whose states change between the two given state CIDs
-	// TODO: Should this take tipset keys instead?
-	StateChangedActors(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) //perm:read
-	// StateGetReceipt returns the message receipt for the given message or for a
-	// matching gas-repriced replacing message
-	//
-	// NOTE: If the requested message was replaced, this method will return the receipt
-	// for the replacing message - if the caller needs the receipt for exactly the
-	// requested message, use StateSearchMsg().Receipt, and check that MsgLookup.Message
-	// is matching the requested CID
-	//
-	// DEPRECATED: Use StateSearchMsg, this method won't be supported in v1 API
-	StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) //perm:read
-	// StateMinerSectorCount returns the number of sectors in a miner's sector set and proving set
-	StateMinerSectorCount(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error) //perm:read
-	// StateCompute is a flexible command that applies the given messages on the given tipset.
-	// The messages are run as though the VM were at the provided height.
-	//
-	// When called, StateCompute will:
-	// - Load the provided tipset, or use the current chain head if not provided
-	// - Compute the tipset state of the provided tipset on top of the parent state
-	//   - (note that this step runs before vmheight is applied to the execution)
-	//   - Execute state upgrade if any were scheduled at the epoch, or in null
-	//     blocks preceding the tipset
-	//   - Call the cron actor on null blocks preceding the tipset
-	//   - For each block in the tipset
-	//     - Apply messages in blocks in the specified
-	//     - Award block reward by calling the reward actor
-	//   - Call the cron actor for the current epoch
-	// - If the specified vmheight is higher than the current epoch, apply any
-	//   needed state upgrades to the state
-	// - Apply the specified messages to the state
-	//
-	// The vmheight parameter sets VM execution epoch, and can be used to simulate
-	// message execution in different network versions. If the specified vmheight
-	// epoch is higher than the epoch of the specified tipset, any state upgrades
-	// until the vmheight will be executed on the state before applying messages
-	// specified by the user.
-	//
-	// Note that the initial tipset state computation is not affected by the
-	// vmheight parameter - only the messages in the `apply` set are
-	//
-	// If the caller wants to simply compute the state, vmheight should be set to
-	// the epoch of the specified tipset.
-	//
-	// Messages in the `apply` parameter must have the correct nonces, and gas
-	// values set.
-	StateCompute(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error) //perm:read
-	// StateVerifierStatus returns the data cap for the given address.
-	// Returns nil if there is no entry in the data cap table for the
-	// address.
-	StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) //perm:read
-	// StateVerifiedClientStatus returns the data cap for the given address.
-	// Returns nil if there is no entry in the data cap table for the
-	// address.
-	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) //perm:read
-	// StateVerifiedClientStatus returns the address of the Verified Registry's root key
-	StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) //perm:read
-	// StateDealProviderCollateralBounds returns the min and max collateral a storage provider
-	// can issue. It takes the deal size and verified status as parameters.
-	StateDealProviderCollateralBounds(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error) //perm:read
-
-	// StateCirculatingSupply returns the exact circulating supply of Filecoin at the given tipset.
-	// This is not used anywhere in the protocol itself, and is only for external consumption.
-	StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error) //perm:read
-	// StateVMCirculatingSupplyInternal returns an approximation of the circulating supply of Filecoin at the given tipset.
-	// This is the value reported by the runtime interface to actors code.
-	StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error) //perm:read
-	// StateNetworkVersion returns the network version at the given tipset
-	StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) //perm:read
-
-	// MethodGroup: Msig
-	// The Msig methods are used to interact with multisig wallets on the
-	// filecoin network
-
-	// MsigGetAvailableBalance returns the portion of a multisig's balance that can be withdrawn or spent
-	MsigGetAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) //perm:read
-	// MsigGetVestingSchedule returns the vesting details of a given multisig.
-	MsigGetVestingSchedule(context.Context, address.Address, types.TipSetKey) (api.MsigVesting, error) //perm:read
-	// MsigGetVested returns the amount of FIL that vested in a multisig in a certain period.
-	// It takes the following params: <multisig address>, <start epoch>, <end epoch>
-	MsigGetVested(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) //perm:read
-
-	//MsigGetPending returns pending transactions for the given multisig
-	//wallet. Once pending transactions are fully approved, they will no longer
-	//appear here.
-	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*api.MsigTransaction, error) //perm:read
-
-	// MsigCreate creates a multisig wallet
-	// It takes the following params: <required number of senders>, <approving addresses>, <unlock duration>
-	//<initial balance>, <sender address of the create msg>, <gas price>
-	MsigCreate(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error) //perm:sign
-	// MsigPropose proposes a multisig message
-	// It takes the following params: <multisig address>, <recipient address>, <value to transfer>,
-	// <sender address of the propose msg>, <method to call in the proposed message>, <params to include in the proposed message>
-	MsigPropose(context.Context, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) //perm:sign
-
-	// MsigApprove approves a previously-proposed multisig message by transaction ID
-	// It takes the following params: <multisig address>, <proposed transaction ID> <signer address>
-	MsigApprove(context.Context, address.Address, uint64, address.Address) (cid.Cid, error) //perm:sign
-
-	// MsigApproveTxnHash approves a previously-proposed multisig message, specified
-	// using both transaction ID and a hash of the parameters used in the
-	// proposal. This method of approval can be used to ensure you only approve
-	// exactly the transaction you think you are.
-	// It takes the following params: <multisig address>, <proposed message ID>, <proposer address>, <recipient address>, <value to transfer>,
-	// <sender address of the approve msg>, <method to call in the approved message>, <params to include in the proposed message>
-	MsigApproveTxnHash(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) //perm:sign
-
-	// MsigCancel cancels a previously-proposed multisig message
-	// It takes the following params: <multisig address>, <proposed transaction ID>, <recipient address>, <value to transfer>,
-	// <sender address of the cancel msg>, <method to call in the proposed message>, <params to include in the proposed message>
-	MsigCancel(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) //perm:sign
-	// MsigAddPropose proposes adding a signer in the multisig
-	// It takes the following params: <multisig address>, <sender address of the propose msg>,
-	// <new signer>, <whether the number of required signers should be increased>
-	MsigAddPropose(context.Context, address.Address, address.Address, address.Address, bool) (cid.Cid, error) //perm:sign
-	// MsigAddApprove approves a previously proposed AddSigner message
-	// It takes the following params: <multisig address>, <sender address of the approve msg>, <proposed message ID>,
-	// <proposer address>, <new signer>, <whether the number of required signers should be increased>
-	MsigAddApprove(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, bool) (cid.Cid, error) //perm:sign
-	// MsigAddCancel cancels a previously proposed AddSigner message
-	// It takes the following params: <multisig address>, <sender address of the cancel msg>, <proposed message ID>,
-	// <new signer>, <whether the number of required signers should be increased>
-	MsigAddCancel(context.Context, address.Address, address.Address, uint64, address.Address, bool) (cid.Cid, error) //perm:sign
-	// MsigSwapPropose proposes swapping 2 signers in the multisig
-	// It takes the following params: <multisig address>, <sender address of the propose msg>,
-	// <old signer>, <new signer>
-	MsigSwapPropose(context.Context, address.Address, address.Address, address.Address, address.Address) (cid.Cid, error) //perm:sign
-	// MsigSwapApprove approves a previously proposed SwapSigner
-	// It takes the following params: <multisig address>, <sender address of the approve msg>, <proposed message ID>,
-	// <proposer address>, <old signer>, <new signer>
-	MsigSwapApprove(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, address.Address) (cid.Cid, error) //perm:sign
-	// MsigSwapCancel cancels a previously proposed SwapSigner message
-	// It takes the following params: <multisig address>, <sender address of the cancel msg>, <proposed message ID>,
-	// <old signer>, <new signer>
-	MsigSwapCancel(context.Context, address.Address, address.Address, uint64, address.Address, address.Address) (cid.Cid, error) //perm:sign
-
-	// MsigRemoveSigner proposes the removal of a signer from the multisig.
-	// It accepts the multisig to make the change on, the proposer address to
-	// send the message from, the address to be removed, and a boolean
-	// indicating whether or not the signing threshold should be lowered by one
-	// along with the address removal.
-	MsigRemoveSigner(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error) //perm:sign
-
-	// MarketAddBalance adds funds to the market actor
-	MarketAddBalance(ctx context.Context, wallet, addr address.Address, amt types.BigInt) (cid.Cid, error) //perm:sign
-	// MarketGetReserved gets the amount of funds that are currently reserved for the address
-	MarketGetReserved(ctx context.Context, addr address.Address) (types.BigInt, error) //perm:sign
-	// MarketReserveFunds reserves funds for a deal
-	MarketReserveFunds(ctx context.Context, wallet address.Address, addr address.Address, amt types.BigInt) (cid.Cid, error) //perm:sign
-	// MarketReleaseFunds releases funds reserved by MarketReserveFunds
-	MarketReleaseFunds(ctx context.Context, addr address.Address, amt types.BigInt) error //perm:sign
-	// MarketWithdraw withdraws unlocked funds from the market actor
-	MarketWithdraw(ctx context.Context, wallet, addr address.Address, amt types.BigInt) (cid.Cid, error) //perm:sign
-
-	// MethodGroup: Paych
-	// The Paych methods are for interacting with and managing payment channels
-
-	PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*api.ChannelInfo, error)                  //perm:sign
-	PaychGetWaitReady(context.Context, cid.Cid) (address.Address, error)                                                 //perm:sign
-	PaychAvailableFunds(ctx context.Context, ch address.Address) (*api.ChannelAvailableFunds, error)                     //perm:sign
-	PaychAvailableFundsByFromTo(ctx context.Context, from, to address.Address) (*api.ChannelAvailableFunds, error)       //perm:sign
-	PaychList(context.Context) ([]address.Address, error)                                                                //perm:read
-	PaychStatus(context.Context, address.Address) (*api.PaychStatus, error)                                              //perm:read
-	PaychSettle(context.Context, address.Address) (cid.Cid, error)                                                       //perm:sign
-	PaychCollect(context.Context, address.Address) (cid.Cid, error)                                                      //perm:sign
-	PaychAllocateLane(ctx context.Context, ch address.Address) (uint64, error)                                           //perm:sign
-	PaychNewPayment(ctx context.Context, from, to address.Address, vouchers []api.VoucherSpec) (*api.PaymentInfo, error) //perm:sign
-	PaychVoucherCheckValid(context.Context, address.Address, *paych.SignedVoucher) error                                 //perm:read
-	PaychVoucherCheckSpendable(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (bool, error)     //perm:read
-	PaychVoucherCreate(context.Context, address.Address, types.BigInt, uint64) (*api.VoucherCreateResult, error)         //perm:sign
-	PaychVoucherAdd(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error)  //perm:write
-	PaychVoucherList(context.Context, address.Address) ([]*paych.SignedVoucher, error)                                   //perm:write
-	PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error)          //perm:sign
-
-	// CreateBackup creates node backup onder the specified file name. The
-	// method requires that the lotus daemon is running with the
-	// LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that
-	// the path specified when calling CreateBackup is within the base path
-	CreateBackup(ctx context.Context, fpath string) error //perm:admin
-}

api/v0api/gateway.go

@@ -1,69 +0,0 @@
-package v0api
-
-import (
-	"context"
-
-	"github.com/ipfs/go-cid"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/dline"
-	"github.com/filecoin-project/go-state-types/network"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-//                       MODIFYING THE API INTERFACE
-//
-// NOTE: This is the V0 (Stable) API - when adding methods to this interface,
-// you'll need to make sure they are also present on the V1 (Unstable) API
-//
-// This API is implemented in `v1_wrapper.go` as a compatibility layer backed
-// by the V1 api
-//
-// When adding / changing methods in this file:
-// * Do the change here
-// * Adjust implementation in `node/impl/`
-// * Run `make gen` - this will:
-//  * Generate proxy structs
-//  * Generate mocks
-//  * Generate markdown docs
-//  * Generate openrpc blobs
-
-type Gateway interface {
-	ChainHasObj(context.Context, cid.Cid) (bool, error)
-	ChainHead(ctx context.Context) (*types.TipSet, error)
-	ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error)
-	ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
-	ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error)
-	ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error)
-	ChainNotify(context.Context) (<-chan []*api.HeadChange, error)
-	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
-	GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
-	MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
-	MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
-	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
-	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*api.MsigTransaction, error)
-	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
-	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error)
-	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
-	StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)
-	StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
-	StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
-	StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error)
-	StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error)
-	StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error)
-	StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
-	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)
-	StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error)
-	StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error)
-	StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
-	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
-	StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error)
-	WalletBalance(context.Context, address.Address) (types.BigInt, error)
-	Version(context.Context) (api.APIVersion, error)
-}
-
-var _ Gateway = *new(FullNode)

api/v0api/latest.go

@@ -1,25 +0,0 @@
-package v0api
-
-import (
-	"github.com/filecoin-project/lotus/api"
-)
-
-type Common = api.Common
-type CommonStruct = api.CommonStruct
-type CommonStub = api.CommonStub
-
-type StorageMiner = api.StorageMiner
-type StorageMinerStruct = api.StorageMinerStruct
-
-type Worker = api.Worker
-type WorkerStruct = api.WorkerStruct
-
-type Wallet = api.Wallet
-
-func PermissionedStorMinerAPI(a StorageMiner) StorageMiner {
-	return api.PermissionedStorMinerAPI(a)
-}
-
-func PermissionedWorkerAPI(a Worker) Worker {
-	return api.PermissionedWorkerAPI(a)
-}

api/v0api/permissioned.go

@@ -1,13 +0,0 @@
-package v0api
-
-import (
-	"github.com/filecoin-project/go-jsonrpc/auth"
-	"github.com/filecoin-project/lotus/api"
-)
-
-func PermissionedFullAPI(a FullNode) FullNode {
-	var out FullNodeStruct
-	auth.PermissionedProxy(api.AllPermissions, api.DefaultPerms, a, &out.Internal)
-	auth.PermissionedProxy(api.AllPermissions, api.DefaultPerms, a, &out.CommonStruct.Internal)
-	return &out
-}

api/v0api/v1_wrapper.go

@@ -1,187 +0,0 @@
-package v0api
-
-import (
-	"context"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/lotus/chain/types"
-	"golang.org/x/xerrors"
-
-	"github.com/ipfs/go-cid"
-
-	"github.com/filecoin-project/go-state-types/abi"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/api/v1api"
-)
-
-type WrapperV1Full struct {
-	v1api.FullNode
-}
-
-func (w *WrapperV1Full) StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error) {
-	return w.FullNode.StateSearchMsg(ctx, types.EmptyTSK, msg, api.LookbackNoLimit, true)
-}
-
-func (w *WrapperV1Full) StateSearchMsgLimited(ctx context.Context, msg cid.Cid, limit abi.ChainEpoch) (*api.MsgLookup, error) {
-	return w.FullNode.StateSearchMsg(ctx, types.EmptyTSK, msg, limit, true)
-}
-
-func (w *WrapperV1Full) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) {
-	return w.FullNode.StateWaitMsg(ctx, msg, confidence, api.LookbackNoLimit, true)
-}
-
-func (w *WrapperV1Full) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, limit abi.ChainEpoch) (*api.MsgLookup, error) {
-	return w.FullNode.StateWaitMsg(ctx, msg, confidence, limit, true)
-}
-
-func (w *WrapperV1Full) StateGetReceipt(ctx context.Context, msg cid.Cid, from types.TipSetKey) (*types.MessageReceipt, error) {
-	ml, err := w.FullNode.StateSearchMsg(ctx, from, msg, api.LookbackNoLimit, true)
-	if err != nil {
-		return nil, err
-	}
-
-	if ml == nil {
-		return nil, nil
-	}
-
-	return &ml.Receipt, nil
-}
-
-func (w *WrapperV1Full) Version(ctx context.Context) (api.APIVersion, error) {
-	ver, err := w.FullNode.Version(ctx)
-	if err != nil {
-		return api.APIVersion{}, err
-	}
-
-	ver.APIVersion = api.FullAPIVersion0
-
-	return ver, nil
-}
-
-func (w *WrapperV1Full) executePrototype(ctx context.Context, p *api.MessagePrototype) (cid.Cid, error) {
-	sm, err := w.FullNode.MpoolPushMessage(ctx, &p.Message, nil)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("pushing message: %w", err)
-	}
-
-	return sm.Cid(), nil
-}
-func (w *WrapperV1Full) MsigCreate(ctx context.Context, req uint64, addrs []address.Address, duration abi.ChainEpoch, val types.BigInt, src address.Address, gp types.BigInt) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigCreate(ctx, req, addrs, duration, val, src, gp)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigPropose(ctx context.Context, msig address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigPropose(ctx, msig, to, amt, src, method, params)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-func (w *WrapperV1Full) MsigApprove(ctx context.Context, msig address.Address, txID uint64, src address.Address) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigApprove(ctx, msig, txID, src)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigApproveTxnHash(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) {
-	p, err := w.FullNode.MsigApproveTxnHash(ctx, msig, txID, proposer, to, amt, src, method, params)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigCancel(ctx context.Context, msig address.Address, txID uint64, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) {
-	p, err := w.FullNode.MsigCancel(ctx, msig, txID, to, amt, src, method, params)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigAddPropose(ctx context.Context, msig address.Address, src address.Address, newAdd address.Address, inc bool) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigAddPropose(ctx, msig, src, newAdd, inc)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigAddApprove(ctx context.Context, msig address.Address, src address.Address, txID uint64, proposer address.Address, newAdd address.Address, inc bool) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigAddApprove(ctx, msig, src, txID, proposer, newAdd, inc)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigAddCancel(ctx context.Context, msig address.Address, src address.Address, txID uint64, newAdd address.Address, inc bool) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigAddCancel(ctx, msig, src, txID, newAdd, inc)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigSwapPropose(ctx context.Context, msig address.Address, src address.Address, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigSwapPropose(ctx, msig, src, oldAdd, newAdd)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigSwapApprove(ctx context.Context, msig address.Address, src address.Address, txID uint64, proposer address.Address, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigSwapApprove(ctx, msig, src, txID, proposer, oldAdd, newAdd)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigSwapCancel(ctx context.Context, msig address.Address, src address.Address, txID uint64, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigSwapCancel(ctx, msig, src, txID, oldAdd, newAdd)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-func (w *WrapperV1Full) MsigRemoveSigner(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error) {
-
-	p, err := w.FullNode.MsigRemoveSigner(ctx, msig, proposer, toRemove, decrease)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
-	}
-
-	return w.executePrototype(ctx, p)
-}
-
-var _ FullNode = &WrapperV1Full{}

api/v1api/latest.go

@@ -1,12 +0,0 @@
-package v1api
-
-import (
-	"github.com/filecoin-project/lotus/api"
-)
-
-type FullNode = api.FullNode
-type FullNodeStruct = api.FullNodeStruct
-
-func PermissionedFullAPI(a FullNode) FullNode {
-	return api.PermissionedFullAPI(a)
-}

api/version.go

@@ -1,73 +0,0 @@
-package api
-
-import (
-	"fmt"
-
-	xerrors "golang.org/x/xerrors"
-)
-
-type Version uint32
-
-func newVer(major, minor, patch uint8) Version {
-	return Version(uint32(major)<<16 | uint32(minor)<<8 | uint32(patch))
-}
-
-// Ints returns (major, minor, patch) versions
-func (ve Version) Ints() (uint32, uint32, uint32) {
-	v := uint32(ve)
-	return (v & majorOnlyMask) >> 16, (v & minorOnlyMask) >> 8, v & patchOnlyMask
-}
-
-func (ve Version) String() string {
-	vmj, vmi, vp := ve.Ints()
-	return fmt.Sprintf("%d.%d.%d", vmj, vmi, vp)
-}
-
-func (ve Version) EqMajorMinor(v2 Version) bool {
-	return ve&minorMask == v2&minorMask
-}
-
-type NodeType int
-
-const (
-	NodeUnknown NodeType = iota
-
-	NodeFull
-	NodeMiner
-	NodeWorker
-)
-
-var RunningNodeType NodeType
-
-func VersionForType(nodeType NodeType) (Version, error) {
-	switch nodeType {
-	case NodeFull:
-		return FullAPIVersion1, nil
-	case NodeMiner:
-		return MinerAPIVersion0, nil
-	case NodeWorker:
-		return WorkerAPIVersion0, nil
-	default:
-		return Version(0), xerrors.Errorf("unknown node type %d", nodeType)
-	}
-}
-
-// semver versions of the rpc api exposed
-var (
-	FullAPIVersion0 = newVer(1, 3, 0)
-	FullAPIVersion1 = newVer(2, 1, 0)
-
-	MinerAPIVersion0  = newVer(1, 2, 0)
-	WorkerAPIVersion0 = newVer(1, 1, 0)
-)
-
-//nolint:varcheck,deadcode
-const (
-	majorMask = 0xff0000
-	minorMask = 0xffff00
-	patchMask = 0xffffff
-
-	majorOnlyMask = 0xff0000
-	minorOnlyMask = 0x00ff00
-	patchOnlyMask = 0x0000ff
-)

api/wrap.go

@@ -1,53 +0,0 @@
-package api
-
-import (
-	"reflect"
-)
-
-// Wrap adapts partial api impl to another version
-// proxyT is the proxy type used as input in wrapperT
-// Usage: Wrap(new(v1api.FullNodeStruct), new(v0api.WrapperV1Full), eventsApi).(EventAPI)
-func Wrap(proxyT, wrapperT, impl interface{}) interface{} {
-	proxy := reflect.New(reflect.TypeOf(proxyT).Elem())
-	proxyMethods := proxy.Elem().FieldByName("Internal")
-	ri := reflect.ValueOf(impl)
-
-	for i := 0; i < ri.NumMethod(); i++ {
-		mt := ri.Type().Method(i)
-		if proxyMethods.FieldByName(mt.Name).Kind() == reflect.Invalid {
-			continue
-		}
-
-		fn := ri.Method(i)
-		of := proxyMethods.FieldByName(mt.Name)
-
-		proxyMethods.FieldByName(mt.Name).Set(reflect.MakeFunc(of.Type(), func(args []reflect.Value) (results []reflect.Value) {
-			return fn.Call(args)
-		}))
-	}
-
-	for i := 0; i < proxy.Elem().NumField(); i++ {
-		if proxy.Elem().Type().Field(i).Name == "Internal" {
-			continue
-		}
-
-		subProxy := proxy.Elem().Field(i).FieldByName("Internal")
-		for i := 0; i < ri.NumMethod(); i++ {
-			mt := ri.Type().Method(i)
-			if subProxy.FieldByName(mt.Name).Kind() == reflect.Invalid {
-				continue
-			}
-
-			fn := ri.Method(i)
-			of := subProxy.FieldByName(mt.Name)
-
-			subProxy.FieldByName(mt.Name).Set(reflect.MakeFunc(of.Type(), func(args []reflect.Value) (results []reflect.Value) {
-				return fn.Call(args)
-			}))
-		}
-	}
-
-	wp := reflect.New(reflect.TypeOf(wrapperT).Elem())
-	wp.Elem().Field(0).Set(proxy)
-	return wp.Interface()
-}

blockstore/api.go

@@ -1,66 +0,0 @@
-package blockstore
-
-import (
-	"context"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	"golang.org/x/xerrors"
-)
-
-type ChainIO interface {
-	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
-	ChainHasObj(context.Context, cid.Cid) (bool, error)
-}
-
-type apiBlockstore struct {
-	api ChainIO
-}
-
-// This blockstore is adapted in the constructor.
-var _ BasicBlockstore = (*apiBlockstore)(nil)
-
-func NewAPIBlockstore(cio ChainIO) Blockstore {
-	bs := &apiBlockstore{api: cio}
-	return Adapt(bs) // return an adapted blockstore.
-}
-
-func (a *apiBlockstore) DeleteBlock(cid.Cid) error {
-	return xerrors.New("not supported")
-}
-
-func (a *apiBlockstore) Has(c cid.Cid) (bool, error) {
-	return a.api.ChainHasObj(context.TODO(), c)
-}
-
-func (a *apiBlockstore) Get(c cid.Cid) (blocks.Block, error) {
-	bb, err := a.api.ChainReadObj(context.TODO(), c)
-	if err != nil {
-		return nil, err
-	}
-	return blocks.NewBlockWithCid(bb, c)
-}
-
-func (a *apiBlockstore) GetSize(c cid.Cid) (int, error) {
-	bb, err := a.api.ChainReadObj(context.TODO(), c)
-	if err != nil {
-		return 0, err
-	}
-	return len(bb), nil
-}
-
-func (a *apiBlockstore) Put(blocks.Block) error {
-	return xerrors.New("not supported")
-}
-
-func (a *apiBlockstore) PutMany([]blocks.Block) error {
-	return xerrors.New("not supported")
-}
-
-func (a *apiBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return nil, xerrors.New("not supported")
-}
-
-func (a *apiBlockstore) HashOnRead(enabled bool) {
-	return
-}

blockstore/badger/blockstore.go

@@ -1,581 +0,0 @@
-package badgerbs
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"runtime"
-	"sync"
-
-	"github.com/dgraph-io/badger/v2"
-	"github.com/dgraph-io/badger/v2/options"
-	"github.com/multiformats/go-base32"
-	"go.uber.org/zap"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	logger "github.com/ipfs/go-log/v2"
-	pool "github.com/libp2p/go-buffer-pool"
-
-	"github.com/filecoin-project/lotus/blockstore"
-)
-
-var (
-	// KeyPool is the buffer pool we use to compute storage keys.
-	KeyPool *pool.BufferPool = pool.GlobalPool
-)
-
-var (
-	// ErrBlockstoreClosed is returned from blockstore operations after
-	// the blockstore has been closed.
-	ErrBlockstoreClosed = fmt.Errorf("badger blockstore closed")
-
-	log = logger.Logger("badgerbs")
-)
-
-// aliases to mask badger dependencies.
-const (
-	// FileIO is equivalent to badger/options.FileIO.
-	FileIO = options.FileIO
-	// MemoryMap is equivalent to badger/options.MemoryMap.
-	MemoryMap = options.MemoryMap
-	// LoadToRAM is equivalent to badger/options.LoadToRAM.
-	LoadToRAM = options.LoadToRAM
-)
-
-// Options embeds the badger options themselves, and augments them with
-// blockstore-specific options.
-type Options struct {
-	badger.Options
-
-	// Prefix is an optional prefix to prepend to keys. Default: "".
-	Prefix string
-}
-
-func DefaultOptions(path string) Options {
-	return Options{
-		Options: badger.DefaultOptions(path),
-		Prefix:  "",
-	}
-}
-
-// badgerLogger is a local wrapper for go-log to make the interface
-// compatible with badger.Logger (namely, aliasing Warnf to Warningf)
-type badgerLogger struct {
-	*zap.SugaredLogger // skips 1 caller to get useful line info, skipping over badger.Options.
-
-	skip2 *zap.SugaredLogger // skips 2 callers, just like above + this logger.
-}
-
-// Warningf is required by the badger logger APIs.
-func (b *badgerLogger) Warningf(format string, args ...interface{}) {
-	b.skip2.Warnf(format, args...)
-}
-
-const (
-	stateOpen = iota
-	stateClosing
-	stateClosed
-)
-
-// Blockstore is a badger-backed IPLD blockstore.
-type Blockstore struct {
-	stateLk sync.RWMutex
-	state   int
-	viewers sync.WaitGroup
-
-	DB *badger.DB
-
-	prefixing bool
-	prefix    []byte
-	prefixLen int
-}
-
-var _ blockstore.Blockstore = (*Blockstore)(nil)
-var _ blockstore.Viewer = (*Blockstore)(nil)
-var _ blockstore.BlockstoreIterator = (*Blockstore)(nil)
-var _ blockstore.BlockstoreGC = (*Blockstore)(nil)
-var _ io.Closer = (*Blockstore)(nil)
-
-// Open creates a new badger-backed blockstore, with the supplied options.
-func Open(opts Options) (*Blockstore, error) {
-	opts.Logger = &badgerLogger{
-		SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(),
-		skip2:         log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(),
-	}
-
-	db, err := badger.Open(opts.Options)
-	if err != nil {
-		return nil, fmt.Errorf("failed to open badger blockstore: %w", err)
-	}
-
-	bs := &Blockstore{DB: db}
-	if p := opts.Prefix; p != "" {
-		bs.prefixing = true
-		bs.prefix = []byte(p)
-		bs.prefixLen = len(bs.prefix)
-	}
-
-	return bs, nil
-}
-
-// Close closes the store. If the store has already been closed, this noops and
-// returns an error, even if the first closure resulted in error.
-func (b *Blockstore) Close() error {
-	b.stateLk.Lock()
-	if b.state != stateOpen {
-		b.stateLk.Unlock()
-		return nil
-	}
-	b.state = stateClosing
-	b.stateLk.Unlock()
-
-	defer func() {
-		b.stateLk.Lock()
-		b.state = stateClosed
-		b.stateLk.Unlock()
-	}()
-
-	// wait for all accesses to complete
-	b.viewers.Wait()
-
-	return b.DB.Close()
-}
-
-func (b *Blockstore) access() error {
-	b.stateLk.RLock()
-	defer b.stateLk.RUnlock()
-
-	if b.state != stateOpen {
-		return ErrBlockstoreClosed
-	}
-
-	b.viewers.Add(1)
-	return nil
-}
-
-func (b *Blockstore) isOpen() bool {
-	b.stateLk.RLock()
-	defer b.stateLk.RUnlock()
-
-	return b.state == stateOpen
-}
-
-// CollectGarbage runs garbage collection on the value log
-func (b *Blockstore) CollectGarbage() error {
-	if err := b.access(); err != nil {
-		return err
-	}
-	defer b.viewers.Done()
-
-	// compact first to gather the necessary statistics for GC
-	nworkers := runtime.NumCPU() / 2
-	if nworkers < 2 {
-		nworkers = 2
-	}
-
-	err := b.DB.Flatten(nworkers)
-	if err != nil {
-		return err
-	}
-
-	for err == nil {
-		err = b.DB.RunValueLogGC(0.125)
-	}
-
-	if err == badger.ErrNoRewrite {
-		// not really an error in this case, it signals the end of GC
-		return nil
-	}
-
-	return err
-}
-
-// View implements blockstore.Viewer, which leverages zero-copy read-only
-// access to values.
-func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
-	if err := b.access(); err != nil {
-		return err
-	}
-	defer b.viewers.Done()
-
-	k, pooled := b.PooledStorageKey(cid)
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	return b.DB.View(func(txn *badger.Txn) error {
-		switch item, err := txn.Get(k); err {
-		case nil:
-			return item.Value(fn)
-		case badger.ErrKeyNotFound:
-			return blockstore.ErrNotFound
-		default:
-			return fmt.Errorf("failed to view block from badger blockstore: %w", err)
-		}
-	})
-}
-
-// Has implements Blockstore.Has.
-func (b *Blockstore) Has(cid cid.Cid) (bool, error) {
-	if err := b.access(); err != nil {
-		return false, err
-	}
-	defer b.viewers.Done()
-
-	k, pooled := b.PooledStorageKey(cid)
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	err := b.DB.View(func(txn *badger.Txn) error {
-		_, err := txn.Get(k)
-		return err
-	})
-
-	switch err {
-	case badger.ErrKeyNotFound:
-		return false, nil
-	case nil:
-		return true, nil
-	default:
-		return false, fmt.Errorf("failed to check if block exists in badger blockstore: %w", err)
-	}
-}
-
-// Get implements Blockstore.Get.
-func (b *Blockstore) Get(cid cid.Cid) (blocks.Block, error) {
-	if !cid.Defined() {
-		return nil, blockstore.ErrNotFound
-	}
-
-	if err := b.access(); err != nil {
-		return nil, err
-	}
-	defer b.viewers.Done()
-
-	k, pooled := b.PooledStorageKey(cid)
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	var val []byte
-	err := b.DB.View(func(txn *badger.Txn) error {
-		switch item, err := txn.Get(k); err {
-		case nil:
-			val, err = item.ValueCopy(nil)
-			return err
-		case badger.ErrKeyNotFound:
-			return blockstore.ErrNotFound
-		default:
-			return fmt.Errorf("failed to get block from badger blockstore: %w", err)
-		}
-	})
-	if err != nil {
-		return nil, err
-	}
-	return blocks.NewBlockWithCid(val, cid)
-}
-
-// GetSize implements Blockstore.GetSize.
-func (b *Blockstore) GetSize(cid cid.Cid) (int, error) {
-	if err := b.access(); err != nil {
-		return 0, err
-	}
-	defer b.viewers.Done()
-
-	k, pooled := b.PooledStorageKey(cid)
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	var size int
-	err := b.DB.View(func(txn *badger.Txn) error {
-		switch item, err := txn.Get(k); err {
-		case nil:
-			size = int(item.ValueSize())
-		case badger.ErrKeyNotFound:
-			return blockstore.ErrNotFound
-		default:
-			return fmt.Errorf("failed to get block size from badger blockstore: %w", err)
-		}
-		return nil
-	})
-	if err != nil {
-		size = -1
-	}
-	return size, err
-}
-
-// Put implements Blockstore.Put.
-func (b *Blockstore) Put(block blocks.Block) error {
-	if err := b.access(); err != nil {
-		return err
-	}
-	defer b.viewers.Done()
-
-	k, pooled := b.PooledStorageKey(block.Cid())
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	err := b.DB.Update(func(txn *badger.Txn) error {
-		return txn.Set(k, block.RawData())
-	})
-	if err != nil {
-		err = fmt.Errorf("failed to put block in badger blockstore: %w", err)
-	}
-	return err
-}
-
-// PutMany implements Blockstore.PutMany.
-func (b *Blockstore) PutMany(blocks []blocks.Block) error {
-	if err := b.access(); err != nil {
-		return err
-	}
-	defer b.viewers.Done()
-
-	// toReturn tracks the byte slices to return to the pool, if we're using key
-	// prefixing. we can't return each slice to the pool after each Set, because
-	// badger holds on to the slice.
-	var toReturn [][]byte
-	if b.prefixing {
-		toReturn = make([][]byte, 0, len(blocks))
-		defer func() {
-			for _, b := range toReturn {
-				KeyPool.Put(b)
-			}
-		}()
-	}
-
-	batch := b.DB.NewWriteBatch()
-	defer batch.Cancel()
-
-	for _, block := range blocks {
-		k, pooled := b.PooledStorageKey(block.Cid())
-		if pooled {
-			toReturn = append(toReturn, k)
-		}
-		if err := batch.Set(k, block.RawData()); err != nil {
-			return err
-		}
-	}
-
-	err := batch.Flush()
-	if err != nil {
-		err = fmt.Errorf("failed to put blocks in badger blockstore: %w", err)
-	}
-	return err
-}
-
-// DeleteBlock implements Blockstore.DeleteBlock.
-func (b *Blockstore) DeleteBlock(cid cid.Cid) error {
-	if err := b.access(); err != nil {
-		return err
-	}
-	defer b.viewers.Done()
-
-	k, pooled := b.PooledStorageKey(cid)
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	return b.DB.Update(func(txn *badger.Txn) error {
-		return txn.Delete(k)
-	})
-}
-
-func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
-	if err := b.access(); err != nil {
-		return err
-	}
-	defer b.viewers.Done()
-
-	// toReturn tracks the byte slices to return to the pool, if we're using key
-	// prefixing. we can't return each slice to the pool after each Set, because
-	// badger holds on to the slice.
-	var toReturn [][]byte
-	if b.prefixing {
-		toReturn = make([][]byte, 0, len(cids))
-		defer func() {
-			for _, b := range toReturn {
-				KeyPool.Put(b)
-			}
-		}()
-	}
-
-	batch := b.DB.NewWriteBatch()
-	defer batch.Cancel()
-
-	for _, cid := range cids {
-		k, pooled := b.PooledStorageKey(cid)
-		if pooled {
-			toReturn = append(toReturn, k)
-		}
-		if err := batch.Delete(k); err != nil {
-			return err
-		}
-	}
-
-	err := batch.Flush()
-	if err != nil {
-		err = fmt.Errorf("failed to delete blocks from badger blockstore: %w", err)
-	}
-	return err
-}
-
-// AllKeysChan implements Blockstore.AllKeysChan.
-func (b *Blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	if err := b.access(); err != nil {
-		return nil, err
-	}
-
-	txn := b.DB.NewTransaction(false)
-	opts := badger.IteratorOptions{PrefetchSize: 100}
-	if b.prefixing {
-		opts.Prefix = b.prefix
-	}
-	iter := txn.NewIterator(opts)
-
-	ch := make(chan cid.Cid)
-	go func() {
-		defer b.viewers.Done()
-		defer close(ch)
-		defer iter.Close()
-
-		// NewCidV1 makes a copy of the multihash buffer, so we can reuse it to
-		// contain allocs.
-		var buf []byte
-		for iter.Rewind(); iter.Valid(); iter.Next() {
-			if ctx.Err() != nil {
-				return // context has fired.
-			}
-			if !b.isOpen() {
-				// open iterators will run even after the database is closed...
-				return // closing, yield.
-			}
-			k := iter.Item().Key()
-			if b.prefixing {
-				k = k[b.prefixLen:]
-			}
-
-			if reqlen := base32.RawStdEncoding.DecodedLen(len(k)); len(buf) < reqlen {
-				buf = make([]byte, reqlen)
-			}
-			if n, err := base32.RawStdEncoding.Decode(buf, k); err == nil {
-				select {
-				case ch <- cid.NewCidV1(cid.Raw, buf[:n]):
-				case <-ctx.Done():
-					return
-				}
-			} else {
-				log.Warnf("failed to decode key %s in badger AllKeysChan; err: %s", k, err)
-			}
-		}
-	}()
-
-	return ch, nil
-}
-
-// Implementation of BlockstoreIterator interface
-func (b *Blockstore) ForEachKey(f func(cid.Cid) error) error {
-	if err := b.access(); err != nil {
-		return err
-	}
-	defer b.viewers.Done()
-
-	txn := b.DB.NewTransaction(false)
-	defer txn.Discard()
-
-	opts := badger.IteratorOptions{PrefetchSize: 100}
-	if b.prefixing {
-		opts.Prefix = b.prefix
-	}
-
-	iter := txn.NewIterator(opts)
-	defer iter.Close()
-
-	var buf []byte
-	for iter.Rewind(); iter.Valid(); iter.Next() {
-		if !b.isOpen() {
-			return ErrBlockstoreClosed
-		}
-
-		k := iter.Item().Key()
-		if b.prefixing {
-			k = k[b.prefixLen:]
-		}
-
-		klen := base32.RawStdEncoding.DecodedLen(len(k))
-		if klen > len(buf) {
-			buf = make([]byte, klen)
-		}
-
-		n, err := base32.RawStdEncoding.Decode(buf, k)
-		if err != nil {
-			return err
-		}
-
-		c := cid.NewCidV1(cid.Raw, buf[:n])
-
-		err = f(c)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// HashOnRead implements Blockstore.HashOnRead. It is not supported by this
-// blockstore.
-func (b *Blockstore) HashOnRead(_ bool) {
-	log.Warnf("called HashOnRead on badger blockstore; function not supported; ignoring")
-}
-
-// PooledStorageKey returns the storage key under which this CID is stored.
-//
-// The key is: prefix + base32_no_padding(cid.Hash)
-//
-// This method may return pooled byte slice, which MUST be returned to the
-// KeyPool if pooled=true, or a leak will occur.
-func (b *Blockstore) PooledStorageKey(cid cid.Cid) (key []byte, pooled bool) {
-	h := cid.Hash()
-	size := base32.RawStdEncoding.EncodedLen(len(h))
-	if !b.prefixing { // optimize for branch prediction.
-		k := pool.Get(size)
-		base32.RawStdEncoding.Encode(k, h)
-		return k, true // slicing upto length unnecessary; the pool has already done this.
-	}
-
-	size += b.prefixLen
-	k := pool.Get(size)
-	copy(k, b.prefix)
-	base32.RawStdEncoding.Encode(k[b.prefixLen:], h)
-	return k, true // slicing upto length unnecessary; the pool has already done this.
-}
-
-// Storage acts like PooledStorageKey, but attempts to write the storage key
-// into the provided slice. If the slice capacity is insufficient, it allocates
-// a new byte slice with enough capacity to accommodate the result. This method
-// returns the resulting slice.
-func (b *Blockstore) StorageKey(dst []byte, cid cid.Cid) []byte {
-	h := cid.Hash()
-	reqsize := base32.RawStdEncoding.EncodedLen(len(h)) + b.prefixLen
-	if reqsize > cap(dst) {
-		// passed slice is smaller than required size; create new.
-		dst = make([]byte, reqsize)
-	} else if reqsize > len(dst) {
-		// passed slice has enough capacity, but its length is
-		// restricted, expand.
-		dst = dst[:cap(dst)]
-	}
-
-	if b.prefixing { // optimize for branch prediction.
-		copy(dst, b.prefix)
-		base32.RawStdEncoding.Encode(dst[b.prefixLen:], h)
-	} else {
-		base32.RawStdEncoding.Encode(dst, h)
-	}
-	return dst[:reqsize]
-}

blockstore/badger/blockstore_test.go

@@ -1,91 +0,0 @@
-package badgerbs
-
-import (
-	"io/ioutil"
-	"os"
-	"testing"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/stretchr/testify/require"
-
-	"github.com/filecoin-project/lotus/blockstore"
-)
-
-func TestBadgerBlockstore(t *testing.T) {
-	(&Suite{
-		NewBlockstore:  newBlockstore(DefaultOptions),
-		OpenBlockstore: openBlockstore(DefaultOptions),
-	}).RunTests(t, "non_prefixed")
-
-	prefixed := func(path string) Options {
-		opts := DefaultOptions(path)
-		opts.Prefix = "/prefixed/"
-		return opts
-	}
-
-	(&Suite{
-		NewBlockstore:  newBlockstore(prefixed),
-		OpenBlockstore: openBlockstore(prefixed),
-	}).RunTests(t, "prefixed")
-}
-
-func TestStorageKey(t *testing.T) {
-	bs, _ := newBlockstore(DefaultOptions)(t)
-	bbs := bs.(*Blockstore)
-	defer bbs.Close() //nolint:errcheck
-
-	cid1 := blocks.NewBlock([]byte("some data")).Cid()
-	cid2 := blocks.NewBlock([]byte("more data")).Cid()
-	cid3 := blocks.NewBlock([]byte("a little more data")).Cid()
-	require.NotEqual(t, cid1, cid2) // sanity check
-	require.NotEqual(t, cid2, cid3) // sanity check
-
-	// nil slice; let StorageKey allocate for us.
-	k1 := bbs.StorageKey(nil, cid1)
-	require.Len(t, k1, 55)
-	require.True(t, cap(k1) == len(k1))
-
-	// k1's backing array is reused.
-	k2 := bbs.StorageKey(k1, cid2)
-	require.Len(t, k2, 55)
-	require.True(t, cap(k2) == len(k1))
-
-	// bring k2 to len=0, and verify that its backing array gets reused
-	// (i.e. k1 and k2 are overwritten)
-	k3 := bbs.StorageKey(k2[:0], cid3)
-	require.Len(t, k3, 55)
-	require.True(t, cap(k3) == len(k3))
-
-	// backing array of k1 and k2 has been modified, i.e. memory is shared.
-	require.Equal(t, k3, k1)
-	require.Equal(t, k3, k2)
-}
-
-func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
-	return func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
-		tb.Helper()
-
-		path, err := ioutil.TempDir("", "")
-		if err != nil {
-			tb.Fatal(err)
-		}
-
-		db, err := Open(optsSupplier(path))
-		if err != nil {
-			tb.Fatal(err)
-		}
-
-		tb.Cleanup(func() {
-			_ = os.RemoveAll(path)
-		})
-
-		return db, path
-	}
-}
-
-func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) {
-	return func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) {
-		tb.Helper()
-		return Open(optsSupplier(path))
-	}
-}

blockstore/badger/blockstore_test_suite.go

@@ -1,313 +0,0 @@
-package badgerbs
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"reflect"
-	"strings"
-	"testing"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	u "github.com/ipfs/go-ipfs-util"
-
-	"github.com/filecoin-project/lotus/blockstore"
-
-	"github.com/stretchr/testify/require"
-)
-
-// TODO: move this to go-ipfs-blockstore.
-type Suite struct {
-	NewBlockstore  func(tb testing.TB) (bs blockstore.BasicBlockstore, path string)
-	OpenBlockstore func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error)
-}
-
-func (s *Suite) RunTests(t *testing.T, prefix string) {
-	v := reflect.TypeOf(s)
-	f := func(t *testing.T) {
-		for i := 0; i < v.NumMethod(); i++ {
-			if m := v.Method(i); strings.HasPrefix(m.Name, "Test") {
-				f := m.Func.Interface().(func(*Suite, *testing.T))
-				t.Run(m.Name, func(t *testing.T) {
-					f(s, t)
-				})
-			}
-		}
-	}
-
-	if prefix == "" {
-		f(t)
-	} else {
-		t.Run(prefix, f)
-	}
-}
-
-func (s *Suite) TestGetWhenKeyNotPresent(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	c := cid.NewCidV0(u.Hash([]byte("stuff")))
-	bl, err := bs.Get(c)
-	require.Nil(t, bl)
-	require.Equal(t, blockstore.ErrNotFound, err)
-}
-
-func (s *Suite) TestGetWhenKeyIsNil(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	_, err := bs.Get(cid.Undef)
-	require.Equal(t, blockstore.ErrNotFound, err)
-}
-
-func (s *Suite) TestPutThenGetBlock(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	orig := blocks.NewBlock([]byte("some data"))
-
-	err := bs.Put(orig)
-	require.NoError(t, err)
-
-	fetched, err := bs.Get(orig.Cid())
-	require.NoError(t, err)
-	require.Equal(t, orig.RawData(), fetched.RawData())
-}
-
-func (s *Suite) TestHas(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	orig := blocks.NewBlock([]byte("some data"))
-
-	err := bs.Put(orig)
-	require.NoError(t, err)
-
-	ok, err := bs.Has(orig.Cid())
-	require.NoError(t, err)
-	require.True(t, ok)
-
-	ok, err = bs.Has(blocks.NewBlock([]byte("another thing")).Cid())
-	require.NoError(t, err)
-	require.False(t, ok)
-}
-
-func (s *Suite) TestCidv0v1(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	orig := blocks.NewBlock([]byte("some data"))
-
-	err := bs.Put(orig)
-	require.NoError(t, err)
-
-	fetched, err := bs.Get(cid.NewCidV1(cid.DagProtobuf, orig.Cid().Hash()))
-	require.NoError(t, err)
-	require.Equal(t, orig.RawData(), fetched.RawData())
-}
-
-func (s *Suite) TestPutThenGetSizeBlock(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	block := blocks.NewBlock([]byte("some data"))
-	missingBlock := blocks.NewBlock([]byte("missingBlock"))
-	emptyBlock := blocks.NewBlock([]byte{})
-
-	err := bs.Put(block)
-	require.NoError(t, err)
-
-	blockSize, err := bs.GetSize(block.Cid())
-	require.NoError(t, err)
-	require.Len(t, block.RawData(), blockSize)
-
-	err = bs.Put(emptyBlock)
-	require.NoError(t, err)
-
-	emptySize, err := bs.GetSize(emptyBlock.Cid())
-	require.NoError(t, err)
-	require.Zero(t, emptySize)
-
-	missingSize, err := bs.GetSize(missingBlock.Cid())
-	require.Equal(t, blockstore.ErrNotFound, err)
-	require.Equal(t, -1, missingSize)
-}
-
-func (s *Suite) TestAllKeysSimple(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	keys := insertBlocks(t, bs, 100)
-
-	ctx := context.Background()
-	ch, err := bs.AllKeysChan(ctx)
-	require.NoError(t, err)
-	actual := collect(ch)
-
-	require.ElementsMatch(t, keys, actual)
-}
-
-func (s *Suite) TestAllKeysRespectsContext(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	_ = insertBlocks(t, bs, 100)
-
-	ctx, cancel := context.WithCancel(context.Background())
-	ch, err := bs.AllKeysChan(ctx)
-	require.NoError(t, err)
-
-	// consume 2, then cancel context.
-	v, ok := <-ch
-	require.NotEqual(t, cid.Undef, v)
-	require.True(t, ok)
-
-	v, ok = <-ch
-	require.NotEqual(t, cid.Undef, v)
-	require.True(t, ok)
-
-	cancel()
-	// pull one value out to avoid race
-	_, _ = <-ch
-
-	v, ok = <-ch
-	require.Equal(t, cid.Undef, v)
-	require.False(t, ok)
-}
-
-func (s *Suite) TestDoubleClose(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	c, ok := bs.(io.Closer)
-	if !ok {
-		t.SkipNow()
-	}
-	require.NoError(t, c.Close())
-	require.NoError(t, c.Close())
-}
-
-func (s *Suite) TestReopenPutGet(t *testing.T) {
-	bs, path := s.NewBlockstore(t)
-	c, ok := bs.(io.Closer)
-	if !ok {
-		t.SkipNow()
-	}
-
-	orig := blocks.NewBlock([]byte("some data"))
-	err := bs.Put(orig)
-	require.NoError(t, err)
-
-	err = c.Close()
-	require.NoError(t, err)
-
-	bs, err = s.OpenBlockstore(t, path)
-	require.NoError(t, err)
-
-	fetched, err := bs.Get(orig.Cid())
-	require.NoError(t, err)
-	require.Equal(t, orig.RawData(), fetched.RawData())
-
-	err = bs.(io.Closer).Close()
-	require.NoError(t, err)
-}
-
-func (s *Suite) TestPutMany(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	blks := []blocks.Block{
-		blocks.NewBlock([]byte("foo1")),
-		blocks.NewBlock([]byte("foo2")),
-		blocks.NewBlock([]byte("foo3")),
-	}
-	err := bs.PutMany(blks)
-	require.NoError(t, err)
-
-	for _, blk := range blks {
-		fetched, err := bs.Get(blk.Cid())
-		require.NoError(t, err)
-		require.Equal(t, blk.RawData(), fetched.RawData())
-
-		ok, err := bs.Has(blk.Cid())
-		require.NoError(t, err)
-		require.True(t, ok)
-	}
-
-	ch, err := bs.AllKeysChan(context.Background())
-	require.NoError(t, err)
-
-	cids := collect(ch)
-	require.Len(t, cids, 3)
-}
-
-func (s *Suite) TestDelete(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	blks := []blocks.Block{
-		blocks.NewBlock([]byte("foo1")),
-		blocks.NewBlock([]byte("foo2")),
-		blocks.NewBlock([]byte("foo3")),
-	}
-	err := bs.PutMany(blks)
-	require.NoError(t, err)
-
-	err = bs.DeleteBlock(blks[1].Cid())
-	require.NoError(t, err)
-
-	ch, err := bs.AllKeysChan(context.Background())
-	require.NoError(t, err)
-
-	cids := collect(ch)
-	require.Len(t, cids, 2)
-	require.ElementsMatch(t, cids, []cid.Cid{
-		cid.NewCidV1(cid.Raw, blks[0].Cid().Hash()),
-		cid.NewCidV1(cid.Raw, blks[2].Cid().Hash()),
-	})
-
-	has, err := bs.Has(blks[1].Cid())
-	require.NoError(t, err)
-	require.False(t, has)
-
-}
-
-func insertBlocks(t *testing.T, bs blockstore.BasicBlockstore, count int) []cid.Cid {
-	keys := make([]cid.Cid, count)
-	for i := 0; i < count; i++ {
-		block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
-		err := bs.Put(block)
-		require.NoError(t, err)
-		// NewBlock assigns a CIDv0; we convert it to CIDv1 because that's what
-		// the store returns.
-		keys[i] = cid.NewCidV1(cid.Raw, block.Multihash())
-	}
-	return keys
-}
-
-func collect(ch <-chan cid.Cid) []cid.Cid {
-	var keys []cid.Cid
-	for k := range ch {
-		keys = append(keys, k)
-	}
-	return keys
-}

blockstore/blockstore.go

@@ -1,105 +0,0 @@
-package blockstore
-
-import (
-	cid "github.com/ipfs/go-cid"
-	ds "github.com/ipfs/go-datastore"
-	logging "github.com/ipfs/go-log/v2"
-
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
-)
-
-var log = logging.Logger("blockstore")
-
-var ErrNotFound = blockstore.ErrNotFound
-
-// Blockstore is the blockstore interface used by Lotus. It is the union
-// of the basic go-ipfs blockstore, with other capabilities required by Lotus,
-// e.g. View or Sync.
-type Blockstore interface {
-	blockstore.Blockstore
-	blockstore.Viewer
-	BatchDeleter
-}
-
-// BasicBlockstore is an alias to the original IPFS Blockstore.
-type BasicBlockstore = blockstore.Blockstore
-
-type Viewer = blockstore.Viewer
-
-type BatchDeleter interface {
-	DeleteMany(cids []cid.Cid) error
-}
-
-// BlockstoreIterator is a trait for efficient iteration
-type BlockstoreIterator interface {
-	ForEachKey(func(cid.Cid) error) error
-}
-
-// BlockstoreGC is a trait for blockstores that support online garbage collection
-type BlockstoreGC interface {
-	CollectGarbage() error
-}
-
-// WrapIDStore wraps the underlying blockstore in an "identity" blockstore.
-// The ID store filters out all puts for blocks with CIDs using the "identity"
-// hash function. It also extracts inlined blocks from CIDs using the identity
-// hash function and returns them on get/has, ignoring the contents of the
-// blockstore.
-func WrapIDStore(bstore blockstore.Blockstore) Blockstore {
-	if is, ok := bstore.(*idstore); ok {
-		// already wrapped
-		return is
-	}
-
-	if bs, ok := bstore.(Blockstore); ok {
-		// we need to wrap our own because we don't want to neuter the DeleteMany method
-		// the underlying blockstore has implemented an (efficient) DeleteMany
-		return NewIDStore(bs)
-	}
-
-	// The underlying blockstore does not implement DeleteMany, so we need to shim it.
-	// This is less efficient as it'll iterate and perform single deletes.
-	return NewIDStore(Adapt(bstore))
-}
-
-// FromDatastore creates a new blockstore backed by the given datastore.
-func FromDatastore(dstore ds.Batching) Blockstore {
-	return WrapIDStore(blockstore.NewBlockstore(dstore))
-}
-
-type adaptedBlockstore struct {
-	blockstore.Blockstore
-}
-
-var _ Blockstore = (*adaptedBlockstore)(nil)
-
-func (a *adaptedBlockstore) View(cid cid.Cid, callback func([]byte) error) error {
-	blk, err := a.Get(cid)
-	if err != nil {
-		return err
-	}
-	return callback(blk.RawData())
-}
-
-func (a *adaptedBlockstore) DeleteMany(cids []cid.Cid) error {
-	for _, cid := range cids {
-		err := a.DeleteBlock(cid)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// Adapt adapts a standard blockstore to a Lotus blockstore by
-// enriching it with the extra methods that Lotus requires (e.g. View, Sync).
-//
-// View proxies over to Get and calls the callback with the value supplied by Get.
-// Sync noops.
-func Adapt(bs blockstore.Blockstore) Blockstore {
-	if ret, ok := bs.(Blockstore); ok {
-		return ret
-	}
-	return &adaptedBlockstore{bs}
-}

blockstore/buffered.go

@@ -1,174 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"os"
-
-	block "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-)
-
-// buflog is a logger for the buffered blockstore. It is subscoped from the
-// blockstore logger.
-var buflog = log.Named("buf")
-
-type BufferedBlockstore struct {
-	read  Blockstore
-	write Blockstore
-}
-
-func NewBuffered(base Blockstore) *BufferedBlockstore {
-	var buf Blockstore
-	if os.Getenv("LOTUS_DISABLE_VM_BUF") == "iknowitsabadidea" {
-		buflog.Warn("VM BLOCKSTORE BUFFERING IS DISABLED")
-		buf = base
-	} else {
-		buf = NewMemory()
-	}
-
-	bs := &BufferedBlockstore{
-		read:  base,
-		write: buf,
-	}
-	return bs
-}
-
-func NewTieredBstore(r Blockstore, w Blockstore) *BufferedBlockstore {
-	return &BufferedBlockstore{
-		read:  r,
-		write: w,
-	}
-}
-
-var (
-	_ Blockstore = (*BufferedBlockstore)(nil)
-	_ Viewer     = (*BufferedBlockstore)(nil)
-)
-
-func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	a, err := bs.read.AllKeysChan(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	b, err := bs.write.AllKeysChan(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	out := make(chan cid.Cid)
-	go func() {
-		defer close(out)
-		for a != nil || b != nil {
-			select {
-			case val, ok := <-a:
-				if !ok {
-					a = nil
-				} else {
-					select {
-					case out <- val:
-					case <-ctx.Done():
-						return
-					}
-				}
-			case val, ok := <-b:
-				if !ok {
-					b = nil
-				} else {
-					select {
-					case out <- val:
-					case <-ctx.Done():
-						return
-					}
-				}
-			}
-		}
-	}()
-
-	return out, nil
-}
-
-func (bs *BufferedBlockstore) DeleteBlock(c cid.Cid) error {
-	if err := bs.read.DeleteBlock(c); err != nil {
-		return err
-	}
-
-	return bs.write.DeleteBlock(c)
-}
-
-func (bs *BufferedBlockstore) DeleteMany(cids []cid.Cid) error {
-	if err := bs.read.DeleteMany(cids); err != nil {
-		return err
-	}
-
-	return bs.write.DeleteMany(cids)
-}
-
-func (bs *BufferedBlockstore) View(c cid.Cid, callback func([]byte) error) error {
-	// both stores are viewable.
-	if err := bs.write.View(c, callback); err == ErrNotFound {
-		// not found in write blockstore; fall through.
-	} else {
-		return err // propagate errors, or nil, i.e. found.
-	}
-	return bs.read.View(c, callback)
-}
-
-func (bs *BufferedBlockstore) Get(c cid.Cid) (block.Block, error) {
-	if out, err := bs.write.Get(c); err != nil {
-		if err != ErrNotFound {
-			return nil, err
-		}
-	} else {
-		return out, nil
-	}
-
-	return bs.read.Get(c)
-}
-
-func (bs *BufferedBlockstore) GetSize(c cid.Cid) (int, error) {
-	s, err := bs.read.GetSize(c)
-	if err == ErrNotFound || s == 0 {
-		return bs.write.GetSize(c)
-	}
-
-	return s, err
-}
-
-func (bs *BufferedBlockstore) Put(blk block.Block) error {
-	has, err := bs.read.Has(blk.Cid()) // TODO: consider dropping this check
-	if err != nil {
-		return err
-	}
-
-	if has {
-		return nil
-	}
-
-	return bs.write.Put(blk)
-}
-
-func (bs *BufferedBlockstore) Has(c cid.Cid) (bool, error) {
-	has, err := bs.write.Has(c)
-	if err != nil {
-		return false, err
-	}
-	if has {
-		return true, nil
-	}
-
-	return bs.read.Has(c)
-}
-
-func (bs *BufferedBlockstore) HashOnRead(hor bool) {
-	bs.read.HashOnRead(hor)
-	bs.write.HashOnRead(hor)
-}
-
-func (bs *BufferedBlockstore) PutMany(blks []block.Block) error {
-	return bs.write.PutMany(blks)
-}
-
-func (bs *BufferedBlockstore) Read() Blockstore {
-	return bs.read
-}

blockstore/discard.go

@@ -1,66 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"io"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-)
-
-var _ Blockstore = (*discardstore)(nil)
-
-type discardstore struct {
-	bs Blockstore
-}
-
-func NewDiscardStore(bs Blockstore) Blockstore {
-	return &discardstore{bs: bs}
-}
-
-func (b *discardstore) Has(cid cid.Cid) (bool, error) {
-	return b.bs.Has(cid)
-}
-
-func (b *discardstore) HashOnRead(hor bool) {
-	b.bs.HashOnRead(hor)
-}
-
-func (b *discardstore) Get(cid cid.Cid) (blocks.Block, error) {
-	return b.bs.Get(cid)
-}
-
-func (b *discardstore) GetSize(cid cid.Cid) (int, error) {
-	return b.bs.GetSize(cid)
-}
-
-func (b *discardstore) View(cid cid.Cid, f func([]byte) error) error {
-	return b.bs.View(cid, f)
-}
-
-func (b *discardstore) Put(blk blocks.Block) error {
-	return nil
-}
-
-func (b *discardstore) PutMany(blks []blocks.Block) error {
-	return nil
-}
-
-func (b *discardstore) DeleteBlock(cid cid.Cid) error {
-	return nil
-}
-
-func (b *discardstore) DeleteMany(cids []cid.Cid) error {
-	return nil
-}
-
-func (b *discardstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return b.bs.AllKeysChan(ctx)
-}
-
-func (b *discardstore) Close() error {
-	if c, ok := b.bs.(io.Closer); ok {
-		return c.Close()
-	}
-	return nil
-}

blockstore/doc.go

@@ -1,9 +0,0 @@
-// Package blockstore and subpackages contain most of the blockstore
-// implementations used by Lotus.
-//
-// Blockstores not ultimately constructed out of the building blocks in this
-// package may not work properly.
-//
-// This package re-exports parts of the go-ipfs-blockstore package such that
-// no other package needs to import it directly, for ergonomics and traceability.
-package blockstore

blockstore/fallback.go

@@ -1,106 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"sync"
-	"time"
-
-	"golang.org/x/xerrors"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-)
-
-// UnwrapFallbackStore takes a blockstore, and returns the underlying blockstore
-// if it was a FallbackStore. Otherwise, it just returns the supplied store
-// unmodified.
-func UnwrapFallbackStore(bs Blockstore) (Blockstore, bool) {
-	if fbs, ok := bs.(*FallbackStore); ok {
-		return fbs.Blockstore, true
-	}
-	return bs, false
-}
-
-// FallbackStore is a read-through store that queries another (potentially
-// remote) source if the block is not found locally. If the block is found
-// during the fallback, it stores it in the local store.
-type FallbackStore struct {
-	Blockstore
-
-	lk sync.RWMutex
-	// missFn is the function that will be invoked on a local miss to pull the
-	// block from elsewhere.
-	missFn func(context.Context, cid.Cid) (blocks.Block, error)
-}
-
-var _ Blockstore = (*FallbackStore)(nil)
-
-func (fbs *FallbackStore) SetFallback(missFn func(context.Context, cid.Cid) (blocks.Block, error)) {
-	fbs.lk.Lock()
-	defer fbs.lk.Unlock()
-
-	fbs.missFn = missFn
-}
-
-func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
-	log.Warnf("fallbackstore: block not found locally, fetching from the network; cid: %s", c)
-	fbs.lk.RLock()
-	defer fbs.lk.RUnlock()
-
-	if fbs.missFn == nil {
-		// FallbackStore wasn't configured yet (chainstore/bitswap aren't up yet)
-		// Wait for a bit and retry
-		fbs.lk.RUnlock()
-		time.Sleep(5 * time.Second)
-		fbs.lk.RLock()
-
-		if fbs.missFn == nil {
-			log.Errorw("fallbackstore: missFn not configured yet")
-			return nil, ErrNotFound
-		}
-	}
-
-	ctx, cancel := context.WithTimeout(context.TODO(), 120*time.Second)
-	defer cancel()
-
-	b, err := fbs.missFn(ctx, c)
-	if err != nil {
-		return nil, err
-	}
-
-	// chain bitswap puts blocks in temp blockstore which is cleaned up
-	// every few min (to drop any messages we fetched but don't want)
-	// in this case we want to keep this block around
-	if err := fbs.Put(b); err != nil {
-		return nil, xerrors.Errorf("persisting fallback-fetched block: %w", err)
-	}
-	return b, nil
-}
-
-func (fbs *FallbackStore) Get(c cid.Cid) (blocks.Block, error) {
-	b, err := fbs.Blockstore.Get(c)
-	switch err {
-	case nil:
-		return b, nil
-	case ErrNotFound:
-		return fbs.getFallback(c)
-	default:
-		return b, err
-	}
-}
-
-func (fbs *FallbackStore) GetSize(c cid.Cid) (int, error) {
-	sz, err := fbs.Blockstore.GetSize(c)
-	switch err {
-	case nil:
-		return sz, nil
-	case ErrNotFound:
-		b, err := fbs.getFallback(c)
-		if err != nil {
-			return 0, err
-		}
-		return len(b.RawData()), nil
-	default:
-		return sz, err
-	}
-}

blockstore/idstore.go

@@ -1,174 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"io"
-
-	"golang.org/x/xerrors"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-	mh "github.com/multiformats/go-multihash"
-)
-
-var _ Blockstore = (*idstore)(nil)
-
-type idstore struct {
-	bs Blockstore
-}
-
-func NewIDStore(bs Blockstore) Blockstore {
-	return &idstore{bs: bs}
-}
-
-func decodeCid(cid cid.Cid) (inline bool, data []byte, err error) {
-	if cid.Prefix().MhType != mh.IDENTITY {
-		return false, nil, nil
-	}
-
-	dmh, err := mh.Decode(cid.Hash())
-	if err != nil {
-		return false, nil, err
-	}
-
-	if dmh.Code == mh.IDENTITY {
-		return true, dmh.Digest, nil
-	}
-
-	return false, nil, err
-}
-
-func (b *idstore) Has(cid cid.Cid) (bool, error) {
-	inline, _, err := decodeCid(cid)
-	if err != nil {
-		return false, xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return true, nil
-	}
-
-	return b.bs.Has(cid)
-}
-
-func (b *idstore) Get(cid cid.Cid) (blocks.Block, error) {
-	inline, data, err := decodeCid(cid)
-	if err != nil {
-		return nil, xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return blocks.NewBlockWithCid(data, cid)
-	}
-
-	return b.bs.Get(cid)
-}
-
-func (b *idstore) GetSize(cid cid.Cid) (int, error) {
-	inline, data, err := decodeCid(cid)
-	if err != nil {
-		return 0, xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return len(data), err
-	}
-
-	return b.bs.GetSize(cid)
-}
-
-func (b *idstore) View(cid cid.Cid, cb func([]byte) error) error {
-	inline, data, err := decodeCid(cid)
-	if err != nil {
-		return xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return cb(data)
-	}
-
-	return b.bs.View(cid, cb)
-}
-
-func (b *idstore) Put(blk blocks.Block) error {
-	inline, _, err := decodeCid(blk.Cid())
-	if err != nil {
-		return xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return nil
-	}
-
-	return b.bs.Put(blk)
-}
-
-func (b *idstore) PutMany(blks []blocks.Block) error {
-	toPut := make([]blocks.Block, 0, len(blks))
-	for _, blk := range blks {
-		inline, _, err := decodeCid(blk.Cid())
-		if err != nil {
-			return xerrors.Errorf("error decoding Cid: %w", err)
-		}
-
-		if inline {
-			continue
-		}
-		toPut = append(toPut, blk)
-	}
-
-	if len(toPut) > 0 {
-		return b.bs.PutMany(toPut)
-	}
-
-	return nil
-}
-
-func (b *idstore) DeleteBlock(cid cid.Cid) error {
-	inline, _, err := decodeCid(cid)
-	if err != nil {
-		return xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return nil
-	}
-
-	return b.bs.DeleteBlock(cid)
-}
-
-func (b *idstore) DeleteMany(cids []cid.Cid) error {
-	toDelete := make([]cid.Cid, 0, len(cids))
-	for _, cid := range cids {
-		inline, _, err := decodeCid(cid)
-		if err != nil {
-			return xerrors.Errorf("error decoding Cid: %w", err)
-		}
-
-		if inline {
-			continue
-		}
-		toDelete = append(toDelete, cid)
-	}
-
-	if len(toDelete) > 0 {
-		return b.bs.DeleteMany(toDelete)
-	}
-
-	return nil
-}
-
-func (b *idstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return b.bs.AllKeysChan(ctx)
-}
-
-func (b *idstore) HashOnRead(enabled bool) {
-	b.bs.HashOnRead(enabled)
-}
-
-func (b *idstore) Close() error {
-	if c, ok := b.bs.(io.Closer); ok {
-		return c.Close()
-	}
-	return nil
-}

blockstore/ipfs.go

@@ -1,155 +0,0 @@
-package blockstore
-
-import (
-	"bytes"
-	"context"
-	"io/ioutil"
-
-	"golang.org/x/xerrors"
-
-	"github.com/multiformats/go-multiaddr"
-	"github.com/multiformats/go-multihash"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	httpapi "github.com/ipfs/go-ipfs-http-client"
-	iface "github.com/ipfs/interface-go-ipfs-core"
-	"github.com/ipfs/interface-go-ipfs-core/options"
-	"github.com/ipfs/interface-go-ipfs-core/path"
-)
-
-type IPFSBlockstore struct {
-	ctx             context.Context
-	api, offlineAPI iface.CoreAPI
-}
-
-var _ BasicBlockstore = (*IPFSBlockstore)(nil)
-
-func NewLocalIPFSBlockstore(ctx context.Context, onlineMode bool) (Blockstore, error) {
-	localApi, err := httpapi.NewLocalApi()
-	if err != nil {
-		return nil, xerrors.Errorf("getting local ipfs api: %w", err)
-	}
-	api, err := localApi.WithOptions(options.Api.Offline(!onlineMode))
-	if err != nil {
-		return nil, xerrors.Errorf("setting offline mode: %s", err)
-	}
-
-	offlineAPI := api
-	if onlineMode {
-		offlineAPI, err = localApi.WithOptions(options.Api.Offline(true))
-		if err != nil {
-			return nil, xerrors.Errorf("applying offline mode: %s", err)
-		}
-	}
-
-	bs := &IPFSBlockstore{
-		ctx:        ctx,
-		api:        api,
-		offlineAPI: offlineAPI,
-	}
-
-	return Adapt(bs), nil
-}
-
-func NewRemoteIPFSBlockstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineMode bool) (Blockstore, error) {
-	httpApi, err := httpapi.NewApi(maddr)
-	if err != nil {
-		return nil, xerrors.Errorf("setting remote ipfs api: %w", err)
-	}
-	api, err := httpApi.WithOptions(options.Api.Offline(!onlineMode))
-	if err != nil {
-		return nil, xerrors.Errorf("applying offline mode: %s", err)
-	}
-
-	offlineAPI := api
-	if onlineMode {
-		offlineAPI, err = httpApi.WithOptions(options.Api.Offline(true))
-		if err != nil {
-			return nil, xerrors.Errorf("applying offline mode: %s", err)
-		}
-	}
-
-	bs := &IPFSBlockstore{
-		ctx:        ctx,
-		api:        api,
-		offlineAPI: offlineAPI,
-	}
-
-	return Adapt(bs), nil
-}
-
-func (i *IPFSBlockstore) DeleteBlock(cid cid.Cid) error {
-	return xerrors.Errorf("not supported")
-}
-
-func (i *IPFSBlockstore) Has(cid cid.Cid) (bool, error) {
-	_, err := i.offlineAPI.Block().Stat(i.ctx, path.IpldPath(cid))
-	if err != nil {
-		// The underlying client is running in Offline mode.
-		// Stat() will fail with an err if the block isn't in the
-		// blockstore. If that's the case, return false without
-		// an error since that's the original intention of this method.
-		if err.Error() == "blockservice: key not found" {
-			return false, nil
-		}
-		return false, xerrors.Errorf("getting ipfs block: %w", err)
-	}
-
-	return true, nil
-}
-
-func (i *IPFSBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
-	rd, err := i.api.Block().Get(i.ctx, path.IpldPath(cid))
-	if err != nil {
-		return nil, xerrors.Errorf("getting ipfs block: %w", err)
-	}
-
-	data, err := ioutil.ReadAll(rd)
-	if err != nil {
-		return nil, err
-	}
-
-	return blocks.NewBlockWithCid(data, cid)
-}
-
-func (i *IPFSBlockstore) GetSize(cid cid.Cid) (int, error) {
-	st, err := i.api.Block().Stat(i.ctx, path.IpldPath(cid))
-	if err != nil {
-		return 0, xerrors.Errorf("getting ipfs block: %w", err)
-	}
-
-	return st.Size(), nil
-}
-
-func (i *IPFSBlockstore) Put(block blocks.Block) error {
-	mhd, err := multihash.Decode(block.Cid().Hash())
-	if err != nil {
-		return err
-	}
-
-	_, err = i.api.Block().Put(i.ctx, bytes.NewReader(block.RawData()),
-		options.Block.Hash(mhd.Code, mhd.Length),
-		options.Block.Format(cid.CodecToStr[block.Cid().Type()]))
-	return err
-}
-
-func (i *IPFSBlockstore) PutMany(blocks []blocks.Block) error {
-	// TODO: could be done in parallel
-
-	for _, block := range blocks {
-		if err := i.Put(block); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (i *IPFSBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return nil, xerrors.Errorf("not supported")
-}
-
-func (i *IPFSBlockstore) HashOnRead(enabled bool) {
-	return // TODO: We could technically support this, but..
-}

blockstore/mem.go

@@ -1,102 +0,0 @@
-package blockstore
-
-import (
-	"context"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-)
-
-// NewMemory returns a temporary memory-backed blockstore.
-func NewMemory() MemBlockstore {
-	return make(MemBlockstore)
-}
-
-// MemBlockstore is a terminal blockstore that keeps blocks in memory.
-type MemBlockstore map[cid.Cid]blocks.Block
-
-func (m MemBlockstore) DeleteBlock(k cid.Cid) error {
-	delete(m, k)
-	return nil
-}
-
-func (m MemBlockstore) DeleteMany(ks []cid.Cid) error {
-	for _, k := range ks {
-		delete(m, k)
-	}
-	return nil
-}
-
-func (m MemBlockstore) Has(k cid.Cid) (bool, error) {
-	_, ok := m[k]
-	return ok, nil
-}
-
-func (m MemBlockstore) View(k cid.Cid, callback func([]byte) error) error {
-	b, ok := m[k]
-	if !ok {
-		return ErrNotFound
-	}
-	return callback(b.RawData())
-}
-
-func (m MemBlockstore) Get(k cid.Cid) (blocks.Block, error) {
-	b, ok := m[k]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	return b, nil
-}
-
-// GetSize returns the CIDs mapped BlockSize
-func (m MemBlockstore) GetSize(k cid.Cid) (int, error) {
-	b, ok := m[k]
-	if !ok {
-		return 0, ErrNotFound
-	}
-	return len(b.RawData()), nil
-}
-
-// Put puts a given block to the underlying datastore
-func (m MemBlockstore) Put(b blocks.Block) error {
-	// Convert to a basic block for safety, but try to reuse the existing
-	// block if it's already a basic block.
-	k := b.Cid()
-	if _, ok := b.(*blocks.BasicBlock); !ok {
-		// If we already have the block, abort.
-		if _, ok := m[k]; ok {
-			return nil
-		}
-		// the error is only for debugging.
-		b, _ = blocks.NewBlockWithCid(b.RawData(), b.Cid())
-	}
-	m[b.Cid()] = b
-	return nil
-}
-
-// PutMany puts a slice of blocks at the same time using batching
-// capabilities of the underlying datastore whenever possible.
-func (m MemBlockstore) PutMany(bs []blocks.Block) error {
-	for _, b := range bs {
-		_ = m.Put(b) // can't fail
-	}
-	return nil
-}
-
-// AllKeysChan returns a channel from which
-// the CIDs in the Blockstore can be read. It should respect
-// the given context, closing the channel if it becomes Done.
-func (m MemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	ch := make(chan cid.Cid, len(m))
-	for k := range m {
-		ch <- k
-	}
-	close(ch)
-	return ch, nil
-}
-
-// HashOnRead specifies if every read block should be
-// rehashed to make sure it matches its CID.
-func (m MemBlockstore) HashOnRead(enabled bool) {
-	// no-op
-}

blockstore/metrics.go

@@ -1,154 +0,0 @@
-package blockstore
-
-import (
-	"time"
-
-	"go.opencensus.io/stats"
-	"go.opencensus.io/stats/view"
-	"go.opencensus.io/tag"
-)
-
-//
-// Currently unused, but kept in repo in case we introduce one of the candidate
-// cache implementations (Freecache, Ristretto), both of which report these
-// metrics.
-//
-
-// CacheMetricsEmitInterval is the interval at which metrics are emitted onto
-// OpenCensus.
-var CacheMetricsEmitInterval = 5 * time.Second
-
-var (
-	CacheName, _ = tag.NewKey("cache_name")
-)
-
-// CacheMeasures groups all metrics emitted by the blockstore caches.
-var CacheMeasures = struct {
-	HitRatio       *stats.Float64Measure
-	Hits           *stats.Int64Measure
-	Misses         *stats.Int64Measure
-	Entries        *stats.Int64Measure
-	QueriesServed  *stats.Int64Measure
-	Adds           *stats.Int64Measure
-	Updates        *stats.Int64Measure
-	Evictions      *stats.Int64Measure
-	CostAdded      *stats.Int64Measure
-	CostEvicted    *stats.Int64Measure
-	SetsDropped    *stats.Int64Measure
-	SetsRejected   *stats.Int64Measure
-	QueriesDropped *stats.Int64Measure
-}{
-	HitRatio:       stats.Float64("blockstore/cache/hit_ratio", "Hit ratio of blockstore cache", stats.UnitDimensionless),
-	Hits:           stats.Int64("blockstore/cache/hits", "Total number of hits at blockstore cache", stats.UnitDimensionless),
-	Misses:         stats.Int64("blockstore/cache/misses", "Total number of misses at blockstore cache", stats.UnitDimensionless),
-	Entries:        stats.Int64("blockstore/cache/entry_count", "Total number of entries currently in the blockstore cache", stats.UnitDimensionless),
-	QueriesServed:  stats.Int64("blockstore/cache/queries_served", "Total number of queries served by the blockstore cache", stats.UnitDimensionless),
-	Adds:           stats.Int64("blockstore/cache/adds", "Total number of adds to blockstore cache", stats.UnitDimensionless),
-	Updates:        stats.Int64("blockstore/cache/updates", "Total number of updates in blockstore cache", stats.UnitDimensionless),
-	Evictions:      stats.Int64("blockstore/cache/evictions", "Total number of evictions from blockstore cache", stats.UnitDimensionless),
-	CostAdded:      stats.Int64("blockstore/cache/cost_added", "Total cost (byte size) of entries added into blockstore cache", stats.UnitBytes),
-	CostEvicted:    stats.Int64("blockstore/cache/cost_evicted", "Total cost (byte size) of entries evicted by blockstore cache", stats.UnitBytes),
-	SetsDropped:    stats.Int64("blockstore/cache/sets_dropped", "Total number of sets dropped by blockstore cache", stats.UnitDimensionless),
-	SetsRejected:   stats.Int64("blockstore/cache/sets_rejected", "Total number of sets rejected by blockstore cache", stats.UnitDimensionless),
-	QueriesDropped: stats.Int64("blockstore/cache/queries_dropped", "Total number of queries dropped by blockstore cache", stats.UnitDimensionless),
-}
-
-// CacheViews groups all cache-related default views.
-var CacheViews = struct {
-	HitRatio       *view.View
-	Hits           *view.View
-	Misses         *view.View
-	Entries        *view.View
-	QueriesServed  *view.View
-	Adds           *view.View
-	Updates        *view.View
-	Evictions      *view.View
-	CostAdded      *view.View
-	CostEvicted    *view.View
-	SetsDropped    *view.View
-	SetsRejected   *view.View
-	QueriesDropped *view.View
-}{
-	HitRatio: &view.View{
-		Measure:     CacheMeasures.HitRatio,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Hits: &view.View{
-		Measure:     CacheMeasures.Hits,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Misses: &view.View{
-		Measure:     CacheMeasures.Misses,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Entries: &view.View{
-		Measure:     CacheMeasures.Entries,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	QueriesServed: &view.View{
-		Measure:     CacheMeasures.QueriesServed,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Adds: &view.View{
-		Measure:     CacheMeasures.Adds,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Updates: &view.View{
-		Measure:     CacheMeasures.Updates,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Evictions: &view.View{
-		Measure:     CacheMeasures.Evictions,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	CostAdded: &view.View{
-		Measure:     CacheMeasures.CostAdded,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	CostEvicted: &view.View{
-		Measure:     CacheMeasures.CostEvicted,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	SetsDropped: &view.View{
-		Measure:     CacheMeasures.SetsDropped,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	SetsRejected: &view.View{
-		Measure:     CacheMeasures.SetsRejected,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	QueriesDropped: &view.View{
-		Measure:     CacheMeasures.QueriesDropped,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-}
-
-// DefaultViews exports all default views for this package.
-var DefaultViews = []*view.View{
-	CacheViews.HitRatio,
-	CacheViews.Hits,
-	CacheViews.Misses,
-	CacheViews.Entries,
-	CacheViews.QueriesServed,
-	CacheViews.Adds,
-	CacheViews.Updates,
-	CacheViews.Evictions,
-	CacheViews.CostAdded,
-	CacheViews.CostEvicted,
-	CacheViews.SetsDropped,
-	CacheViews.SetsRejected,
-	CacheViews.QueriesDropped,
-}

blockstore/splitstore/README.md

@@ -1,72 +0,0 @@
-# SplitStore: An actively scalable blockstore for the Filecoin chain
-
-The SplitStore was first introduced in lotus v1.5.1, as an experiment
-in reducing the performance impact of large blockstores.
-
-With lotus v1.11.1, we introduce the next iteration in design and
-implementation, which we call SplitStore v1.
-
-The new design (see [#6474](https://github.com/filecoin-project/lotus/pull/6474)
-evolves the splitstore to be a freestanding compacting blockstore that
-allows us to keep a small (60-100GB) working set in a hot blockstore
-and reliably archive out of scope objects in a coldstore.  The
-coldstore can also be a discard store, whereby out of scope objects
-are discarded or a regular badger blockstore (the default), which can
-be periodically garbage collected according to configurable user
-retention policies.
-
-To enable the splitstore, edit `.lotus/config.toml` and add the following:
-```
-[Chainstore]
-  EnableSplitstore = true
-```
-
-If you intend to use the discard coldstore, your also need to add the following:
-```
-  [Chainstore.Splitstore]
-    ColdStoreType = "discard"
-```
-In general you _should not_ have to use the discard store, unless you
-are running a network booster or have very constrained hardware with
-not enough disk space to maintain a coldstore, even with garbage
-collection.
-
-
-## Operation
-
-When the splitstore is first enabled, the existing blockstore becomes
-the coldstore and a fresh hotstore is initialized.
-
-The hotstore is warmed up on first startup so as to load all chain
-headers and state roots in the current head.  This allows us to
-immediately gain the performance benefits of a smallerblockstore which
-can be substantial for full archival nodes.
-
-All new writes are directed to the hotstore, while reads first hit the
-hotstore, with fallback to the coldstore.
-
-Once 5 finalities have ellapsed, and every finality henceforth, the
-blockstore _compacts_.  Compaction is the process of moving all
-unreachable objects within the last 4 finalities from the hotstore to
-the coldstore. If the system is configured with a discard coldstore,
-these objects are discarded. Note that chain headers, all the way to
-genesis, are considered reachable. Stateroots and messages are
-considered reachable only within the last 4 finalities, unless there
-is a live reference to them.
-
-## Compaction
-
-Compaction works transactionally with the following algorithm:
-- We prepare a transaction, whereby all i/o referenced objects through the API are tracked.
-- We walk the chain and mark reachable objects, keeping 4 finalities of state roots and messages and all headers all the way to genesis.
-- Once the chain walk is complete, we begin full transaction protection with concurrent marking; we walk and mark all references created during the chain walk. On the same time, all I/O through the API concurrently marks objects as live references.
-- We collect cold objects by iterating through the hotstore and checking the mark set; if an object is not marked, then it is candidate for purge.
-- When running with a coldstore, we next copy all cold objects to the coldstore.
-- At this point we are ready to begin purging:
-  - We sort cold objects heaviest first, so as to never delete the consituents of a DAG before the DAG itself (which would leave dangling references)
-  - We delete in small batches taking a lock; each batch is checked again for marks, from the concurrent transactional mark, so as to never delete anything live
-- We then end the transaction and compact/gc the hotstore.
-
-## Coldstore Garbage Collection
-
-TBD -- see [#6577](https://github.com/filecoin-project/lotus/issues/6577)

blockstore/splitstore/debug.go

@@ -1,273 +0,0 @@
-package splitstore
-
-import (
-	"crypto/sha256"
-	"encoding/hex"
-	"fmt"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"runtime/debug"
-	"strings"
-	"sync"
-	"time"
-
-	"go.uber.org/multierr"
-	"golang.org/x/xerrors"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-)
-
-type debugLog struct {
-	readLog, writeLog, deleteLog, stackLog *debugLogOp
-
-	stackMx  sync.Mutex
-	stackMap map[string]string
-}
-
-type debugLogOp struct {
-	path  string
-	mx    sync.Mutex
-	log   *os.File
-	count int
-}
-
-func openDebugLog(path string) (*debugLog, error) {
-	basePath := filepath.Join(path, "debug")
-	err := os.MkdirAll(basePath, 0755)
-	if err != nil {
-		return nil, err
-	}
-
-	readLog, err := openDebugLogOp(basePath, "read.log")
-	if err != nil {
-		return nil, err
-	}
-
-	writeLog, err := openDebugLogOp(basePath, "write.log")
-	if err != nil {
-		_ = readLog.Close()
-		return nil, err
-	}
-
-	deleteLog, err := openDebugLogOp(basePath, "delete.log")
-	if err != nil {
-		_ = readLog.Close()
-		_ = writeLog.Close()
-		return nil, err
-	}
-
-	stackLog, err := openDebugLogOp(basePath, "stack.log")
-	if err != nil {
-		_ = readLog.Close()
-		_ = writeLog.Close()
-		_ = deleteLog.Close()
-		return nil, xerrors.Errorf("error opening stack log: %w", err)
-	}
-
-	return &debugLog{
-		readLog:   readLog,
-		writeLog:  writeLog,
-		deleteLog: deleteLog,
-		stackLog:  stackLog,
-		stackMap:  make(map[string]string),
-	}, nil
-}
-
-func (d *debugLog) LogReadMiss(cid cid.Cid) {
-	if d == nil {
-		return
-	}
-
-	stack := d.getStack()
-	err := d.readLog.Log("%s %s %s\n", d.timestamp(), cid, stack)
-	if err != nil {
-		log.Warnf("error writing read log: %s", err)
-	}
-}
-
-func (d *debugLog) LogWrite(blk blocks.Block) {
-	if d == nil {
-		return
-	}
-
-	var stack string
-	if enableDebugLogWriteTraces {
-		stack = " " + d.getStack()
-	}
-
-	err := d.writeLog.Log("%s %s%s\n", d.timestamp(), blk.Cid(), stack)
-	if err != nil {
-		log.Warnf("error writing write log: %s", err)
-	}
-}
-
-func (d *debugLog) LogWriteMany(blks []blocks.Block) {
-	if d == nil {
-		return
-	}
-
-	var stack string
-	if enableDebugLogWriteTraces {
-		stack = " " + d.getStack()
-	}
-
-	now := d.timestamp()
-	for _, blk := range blks {
-		err := d.writeLog.Log("%s %s%s\n", now, blk.Cid(), stack)
-		if err != nil {
-			log.Warnf("error writing write log: %s", err)
-			break
-		}
-	}
-}
-
-func (d *debugLog) LogDelete(cids []cid.Cid) {
-	if d == nil {
-		return
-	}
-
-	now := d.timestamp()
-	for _, c := range cids {
-		err := d.deleteLog.Log("%s %s\n", now, c)
-		if err != nil {
-			log.Warnf("error writing delete log: %s", err)
-			break
-		}
-	}
-}
-
-func (d *debugLog) Flush() {
-	if d == nil {
-		return
-	}
-
-	// rotate non-empty logs
-	d.readLog.Rotate()
-	d.writeLog.Rotate()
-	d.deleteLog.Rotate()
-	d.stackLog.Rotate()
-}
-
-func (d *debugLog) Close() error {
-	if d == nil {
-		return nil
-	}
-
-	err1 := d.readLog.Close()
-	err2 := d.writeLog.Close()
-	err3 := d.deleteLog.Close()
-	err4 := d.stackLog.Close()
-
-	return multierr.Combine(err1, err2, err3, err4)
-}
-
-func (d *debugLog) getStack() string {
-	sk := d.getNormalizedStackTrace()
-	hash := sha256.Sum256([]byte(sk))
-	key := string(hash[:])
-
-	d.stackMx.Lock()
-	repr, ok := d.stackMap[key]
-	if !ok {
-		repr = hex.EncodeToString(hash[:])
-		d.stackMap[key] = repr
-
-		err := d.stackLog.Log("%s\n%s\n", repr, sk)
-		if err != nil {
-			log.Warnf("error writing stack trace for %s: %s", repr, err)
-		}
-	}
-	d.stackMx.Unlock()
-
-	return repr
-}
-
-func (d *debugLog) getNormalizedStackTrace() string {
-	sk := string(debug.Stack())
-
-	// Normalization for deduplication
-	// skip first line -- it's the goroutine
-	// for each line that ends in a ), remove the call args -- these are the registers
-	lines := strings.Split(sk, "\n")[1:]
-	for i, line := range lines {
-		if len(line) > 0 && line[len(line)-1] == ')' {
-			idx := strings.LastIndex(line, "(")
-			if idx < 0 {
-				continue
-			}
-			lines[i] = line[:idx]
-		}
-	}
-
-	return strings.Join(lines, "\n")
-}
-
-func (d *debugLog) timestamp() string {
-	ts, _ := time.Now().MarshalText()
-	return string(ts)
-}
-
-func openDebugLogOp(basePath, name string) (*debugLogOp, error) {
-	path := filepath.Join(basePath, name)
-	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
-	if err != nil {
-		return nil, xerrors.Errorf("error opening %s: %w", name, err)
-	}
-
-	return &debugLogOp{path: path, log: file}, nil
-}
-
-func (d *debugLogOp) Close() error {
-	d.mx.Lock()
-	defer d.mx.Unlock()
-
-	return d.log.Close()
-}
-
-func (d *debugLogOp) Log(template string, arg ...interface{}) error {
-	d.mx.Lock()
-	defer d.mx.Unlock()
-
-	d.count++
-	_, err := fmt.Fprintf(d.log, template, arg...)
-	return err
-}
-
-func (d *debugLogOp) Rotate() {
-	d.mx.Lock()
-	defer d.mx.Unlock()
-
-	if d.count == 0 {
-		return
-	}
-
-	err := d.log.Close()
-	if err != nil {
-		log.Warnf("error closing log (file: %s): %s", d.path, err)
-		return
-	}
-
-	arxivPath := fmt.Sprintf("%s-%d", d.path, time.Now().Unix())
-	err = os.Rename(d.path, arxivPath)
-	if err != nil {
-		log.Warnf("error moving log (file: %s): %s", d.path, err)
-		return
-	}
-
-	go func() {
-		cmd := exec.Command("gzip", arxivPath)
-		err := cmd.Run()
-		if err != nil {
-			log.Warnf("error compressing log (file: %s): %s", arxivPath, err)
-		}
-	}()
-
-	d.count = 0
-	d.log, err = os.OpenFile(d.path, os.O_WRONLY|os.O_CREATE, 0644)
-	if err != nil {
-		log.Warnf("error opening log (file: %s): %s", d.path, err)
-		return
-	}
-}

blockstore/splitstore/markset.go

@@ -1,38 +0,0 @@
-package splitstore
-
-import (
-	"errors"
-
-	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
-)
-
-var errMarkSetClosed = errors.New("markset closed")
-
-// MarkSet is a utility to keep track of seen CID, and later query for them.
-//
-// * If the expected dataset is large, it can be backed by a datastore (e.g. bbolt).
-// * If a probabilistic result is acceptable, it can be backed by a bloom filter
-type MarkSet interface {
-	Mark(cid.Cid) error
-	Has(cid.Cid) (bool, error)
-	Close() error
-	SetConcurrent()
-}
-
-type MarkSetEnv interface {
-	Create(name string, sizeHint int64) (MarkSet, error)
-	Close() error
-}
-
-func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) {
-	switch mtype {
-	case "bloom":
-		return NewBloomMarkSetEnv()
-	case "map":
-		return NewMapMarkSetEnv()
-	default:
-		return nil, xerrors.Errorf("unknown mark set type %s", mtype)
-	}
-}

blockstore/splitstore/markset_bloom.go

@@ -1,107 +0,0 @@
-package splitstore
-
-import (
-	"crypto/rand"
-	"crypto/sha256"
-	"sync"
-
-	"golang.org/x/xerrors"
-
-	bbloom "github.com/ipfs/bbloom"
-	cid "github.com/ipfs/go-cid"
-)
-
-const (
-	BloomFilterMinSize     = 10_000_000
-	BloomFilterProbability = 0.01
-)
-
-type BloomMarkSetEnv struct{}
-
-var _ MarkSetEnv = (*BloomMarkSetEnv)(nil)
-
-type BloomMarkSet struct {
-	salt []byte
-	mx   sync.RWMutex
-	bf   *bbloom.Bloom
-	ts   bool
-}
-
-var _ MarkSet = (*BloomMarkSet)(nil)
-
-func NewBloomMarkSetEnv() (*BloomMarkSetEnv, error) {
-	return &BloomMarkSetEnv{}, nil
-}
-
-func (e *BloomMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
-	size := int64(BloomFilterMinSize)
-	for size < sizeHint {
-		size += BloomFilterMinSize
-	}
-
-	salt := make([]byte, 4)
-	_, err := rand.Read(salt)
-	if err != nil {
-		return nil, xerrors.Errorf("error reading salt: %w", err)
-	}
-
-	bf, err := bbloom.New(float64(size), BloomFilterProbability)
-	if err != nil {
-		return nil, xerrors.Errorf("error creating bloom filter: %w", err)
-	}
-
-	return &BloomMarkSet{salt: salt, bf: bf}, nil
-}
-
-func (e *BloomMarkSetEnv) Close() error {
-	return nil
-}
-
-func (s *BloomMarkSet) saltedKey(cid cid.Cid) []byte {
-	hash := cid.Hash()
-	key := make([]byte, len(s.salt)+len(hash))
-	n := copy(key, s.salt)
-	copy(key[n:], hash)
-	rehash := sha256.Sum256(key)
-	return rehash[:]
-}
-
-func (s *BloomMarkSet) Mark(cid cid.Cid) error {
-	if s.ts {
-		s.mx.Lock()
-		defer s.mx.Unlock()
-	}
-
-	if s.bf == nil {
-		return errMarkSetClosed
-	}
-
-	s.bf.Add(s.saltedKey(cid))
-	return nil
-}
-
-func (s *BloomMarkSet) Has(cid cid.Cid) (bool, error) {
-	if s.ts {
-		s.mx.RLock()
-		defer s.mx.RUnlock()
-	}
-
-	if s.bf == nil {
-		return false, errMarkSetClosed
-	}
-
-	return s.bf.Has(s.saltedKey(cid)), nil
-}
-
-func (s *BloomMarkSet) Close() error {
-	if s.ts {
-		s.mx.Lock()
-		defer s.mx.Unlock()
-	}
-	s.bf = nil
-	return nil
-}
-
-func (s *BloomMarkSet) SetConcurrent() {
-	s.ts = true
-}

blockstore/splitstore/markset_map.go

@@ -1,75 +0,0 @@
-package splitstore
-
-import (
-	"sync"
-
-	cid "github.com/ipfs/go-cid"
-)
-
-type MapMarkSetEnv struct{}
-
-var _ MarkSetEnv = (*MapMarkSetEnv)(nil)
-
-type MapMarkSet struct {
-	mx  sync.RWMutex
-	set map[string]struct{}
-
-	ts bool
-}
-
-var _ MarkSet = (*MapMarkSet)(nil)
-
-func NewMapMarkSetEnv() (*MapMarkSetEnv, error) {
-	return &MapMarkSetEnv{}, nil
-}
-
-func (e *MapMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
-	return &MapMarkSet{
-		set: make(map[string]struct{}, sizeHint),
-	}, nil
-}
-
-func (e *MapMarkSetEnv) Close() error {
-	return nil
-}
-
-func (s *MapMarkSet) Mark(cid cid.Cid) error {
-	if s.ts {
-		s.mx.Lock()
-		defer s.mx.Unlock()
-	}
-
-	if s.set == nil {
-		return errMarkSetClosed
-	}
-
-	s.set[string(cid.Hash())] = struct{}{}
-	return nil
-}
-
-func (s *MapMarkSet) Has(cid cid.Cid) (bool, error) {
-	if s.ts {
-		s.mx.RLock()
-		defer s.mx.RUnlock()
-	}
-
-	if s.set == nil {
-		return false, errMarkSetClosed
-	}
-
-	_, ok := s.set[string(cid.Hash())]
-	return ok, nil
-}
-
-func (s *MapMarkSet) Close() error {
-	if s.ts {
-		s.mx.Lock()
-		defer s.mx.Unlock()
-	}
-	s.set = nil
-	return nil
-}
-
-func (s *MapMarkSet) SetConcurrent() {
-	s.ts = true
-}

blockstore/splitstore/markset_test.go

@@ -1,138 +0,0 @@
-package splitstore
-
-import (
-	"io/ioutil"
-	"testing"
-
-	cid "github.com/ipfs/go-cid"
-	"github.com/multiformats/go-multihash"
-)
-
-func TestMapMarkSet(t *testing.T) {
-	testMarkSet(t, "map")
-}
-
-func TestBloomMarkSet(t *testing.T) {
-	testMarkSet(t, "bloom")
-}
-
-func testMarkSet(t *testing.T, lsType string) {
-	t.Helper()
-
-	path, err := ioutil.TempDir("", "sweep-test.*")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	env, err := OpenMarkSetEnv(path, lsType)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer env.Close() //nolint:errcheck
-
-	hotSet, err := env.Create("hot", 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	coldSet, err := env.Create("cold", 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	makeCid := func(key string) cid.Cid {
-		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		return cid.NewCidV1(cid.Raw, h)
-	}
-
-	mustHave := func(s MarkSet, cid cid.Cid) {
-		has, err := s.Has(cid)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if !has {
-			t.Fatal("mark not found")
-		}
-	}
-
-	mustNotHave := func(s MarkSet, cid cid.Cid) {
-		has, err := s.Has(cid)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if has {
-			t.Fatal("unexpected mark")
-		}
-	}
-
-	k1 := makeCid("a")
-	k2 := makeCid("b")
-	k3 := makeCid("c")
-	k4 := makeCid("d")
-
-	hotSet.Mark(k1)  //nolint
-	hotSet.Mark(k2)  //nolint
-	coldSet.Mark(k3) //nolint
-
-	mustHave(hotSet, k1)
-	mustHave(hotSet, k2)
-	mustNotHave(hotSet, k3)
-	mustNotHave(hotSet, k4)
-
-	mustNotHave(coldSet, k1)
-	mustNotHave(coldSet, k2)
-	mustHave(coldSet, k3)
-	mustNotHave(coldSet, k4)
-
-	// close them and reopen to redo the dance
-
-	err = hotSet.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = coldSet.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	hotSet, err = env.Create("hot", 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	coldSet, err = env.Create("cold", 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	hotSet.Mark(k3)  //nolint
-	hotSet.Mark(k4)  //nolint
-	coldSet.Mark(k1) //nolint
-
-	mustNotHave(hotSet, k1)
-	mustNotHave(hotSet, k2)
-	mustHave(hotSet, k3)
-	mustHave(hotSet, k4)
-
-	mustHave(coldSet, k1)
-	mustNotHave(coldSet, k2)
-	mustNotHave(coldSet, k3)
-	mustNotHave(coldSet, k4)
-
-	err = hotSet.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = coldSet.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-}

blockstore/splitstore/splitstore.go

@@ -1,551 +0,0 @@
-package splitstore
-
-import (
-	"context"
-	"errors"
-	"os"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"go.uber.org/multierr"
-	"golang.org/x/xerrors"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-	dstore "github.com/ipfs/go-datastore"
-	logging "github.com/ipfs/go-log/v2"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	bstore "github.com/filecoin-project/lotus/blockstore"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/metrics"
-
-	"go.opencensus.io/stats"
-)
-
-var (
-	// baseEpochKey stores the base epoch (last compaction epoch) in the
-	// metadata store.
-	baseEpochKey = dstore.NewKey("/splitstore/baseEpoch")
-
-	// warmupEpochKey stores whether a hot store warmup has been performed.
-	// On first start, the splitstore will walk the state tree and will copy
-	// all active blocks into the hotstore.
-	warmupEpochKey = dstore.NewKey("/splitstore/warmupEpoch")
-
-	// markSetSizeKey stores the current estimate for the mark set size.
-	// this is first computed at warmup and updated in every compaction
-	markSetSizeKey = dstore.NewKey("/splitstore/markSetSize")
-
-	// compactionIndexKey stores the compaction index (serial number)
-	compactionIndexKey = dstore.NewKey("/splitstore/compactionIndex")
-
-	log = logging.Logger("splitstore")
-
-	// set this to true if you are debugging the splitstore to enable debug logging
-	enableDebugLog = false
-	// set this to true if you want to track origin stack traces in the write log
-	enableDebugLogWriteTraces = false
-)
-
-func init() {
-	if os.Getenv("LOTUS_SPLITSTORE_DEBUG_LOG") == "1" {
-		enableDebugLog = true
-	}
-
-	if os.Getenv("LOTUS_SPLITSTORE_DEBUG_LOG_WRITE_TRACES") == "1" {
-		enableDebugLogWriteTraces = true
-	}
-}
-
-type Config struct {
-	// MarkSetType is the type of mark set to use.
-	//
-	// Only current sane value is "map", but we may add an option for a disk-backed
-	// markset for memory-constrained situations.
-	MarkSetType string
-
-	// DiscardColdBlocks indicates whether to skip moving cold blocks to the coldstore.
-	// If the splitstore is running with a noop coldstore then this option is set to true
-	// which skips moving (as it is a noop, but still takes time to read all the cold objects)
-	// and directly purges cold blocks.
-	DiscardColdBlocks bool
-}
-
-// ChainAccessor allows the Splitstore to access the chain. It will most likely
-// be a ChainStore at runtime.
-type ChainAccessor interface {
-	GetTipsetByHeight(context.Context, abi.ChainEpoch, *types.TipSet, bool) (*types.TipSet, error)
-	GetHeaviestTipSet() *types.TipSet
-	SubscribeHeadChanges(change func(revert []*types.TipSet, apply []*types.TipSet) error)
-}
-
-// hotstore is the interface that must be satisfied by the hot blockstore; it is an extension
-// of the Blockstore interface with the traits we need for compaction.
-type hotstore interface {
-	bstore.Blockstore
-	bstore.BlockstoreIterator
-}
-
-type SplitStore struct {
-	compacting int32 // compaction/prune/warmup in progress
-	closing    int32 // the splitstore is closing
-
-	cfg *Config
-
-	mx          sync.Mutex
-	warmupEpoch abi.ChainEpoch // protected by mx
-	baseEpoch   abi.ChainEpoch // protected by compaction lock
-
-	headChangeMx sync.Mutex
-
-	coldPurgeSize int
-
-	chain ChainAccessor
-	ds    dstore.Datastore
-	cold  bstore.Blockstore
-	hot   hotstore
-
-	markSetEnv  MarkSetEnv
-	markSetSize int64
-
-	compactionIndex int64
-
-	ctx    context.Context
-	cancel func()
-
-	debug *debugLog
-
-	// transactional protection for concurrent read/writes during compaction
-	txnLk           sync.RWMutex
-	txnViewsMx      sync.Mutex
-	txnViewsCond    sync.Cond
-	txnViews        int
-	txnViewsWaiting bool
-	txnActive       bool
-	txnProtect      MarkSet
-	txnRefsMx       sync.Mutex
-	txnRefs         map[cid.Cid]struct{}
-	txnMissing      map[cid.Cid]struct{}
-}
-
-var _ bstore.Blockstore = (*SplitStore)(nil)
-
-// Open opens an existing splistore, or creates a new splitstore. The splitstore
-// is backed by the provided hot and cold stores. The returned SplitStore MUST be
-// attached to the ChainStore with Start in order to trigger compaction.
-func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Config) (*SplitStore, error) {
-	// hot blockstore must support the hotstore interface
-	hots, ok := hot.(hotstore)
-	if !ok {
-		// be specific about what is missing
-		if _, ok := hot.(bstore.BlockstoreIterator); !ok {
-			return nil, xerrors.Errorf("hot blockstore does not support efficient iteration: %T", hot)
-		}
-
-		return nil, xerrors.Errorf("hot blockstore does not support the necessary traits: %T", hot)
-	}
-
-	// the markset env
-	markSetEnv, err := OpenMarkSetEnv(path, cfg.MarkSetType)
-	if err != nil {
-		return nil, err
-	}
-
-	// and now we can make a SplitStore
-	ss := &SplitStore{
-		cfg:        cfg,
-		ds:         ds,
-		cold:       cold,
-		hot:        hots,
-		markSetEnv: markSetEnv,
-
-		coldPurgeSize: defaultColdPurgeSize,
-	}
-
-	ss.txnViewsCond.L = &ss.txnViewsMx
-	ss.ctx, ss.cancel = context.WithCancel(context.Background())
-
-	if enableDebugLog {
-		ss.debug, err = openDebugLog(path)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return ss, nil
-}
-
-// Blockstore interface
-func (s *SplitStore) DeleteBlock(_ cid.Cid) error {
-	// afaict we don't seem to be using this method, so it's not implemented
-	return errors.New("DeleteBlock not implemented on SplitStore; don't do this Luke!") //nolint
-}
-
-func (s *SplitStore) DeleteMany(_ []cid.Cid) error {
-	// afaict we don't seem to be using this method, so it's not implemented
-	return errors.New("DeleteMany not implemented on SplitStore; don't do this Luke!") //nolint
-}
-
-func (s *SplitStore) Has(cid cid.Cid) (bool, error) {
-	if isIdentiyCid(cid) {
-		return true, nil
-	}
-
-	s.txnLk.RLock()
-	defer s.txnLk.RUnlock()
-
-	has, err := s.hot.Has(cid)
-
-	if err != nil {
-		return has, err
-	}
-
-	if has {
-		s.trackTxnRef(cid)
-		return true, nil
-	}
-
-	return s.cold.Has(cid)
-}
-
-func (s *SplitStore) Get(cid cid.Cid) (blocks.Block, error) {
-	if isIdentiyCid(cid) {
-		data, err := decodeIdentityCid(cid)
-		if err != nil {
-			return nil, err
-		}
-
-		return blocks.NewBlockWithCid(data, cid)
-	}
-
-	s.txnLk.RLock()
-	defer s.txnLk.RUnlock()
-
-	blk, err := s.hot.Get(cid)
-
-	switch err {
-	case nil:
-		s.trackTxnRef(cid)
-		return blk, nil
-
-	case bstore.ErrNotFound:
-		if s.isWarm() {
-			s.debug.LogReadMiss(cid)
-		}
-
-		blk, err = s.cold.Get(cid)
-		if err == nil {
-			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
-
-		}
-		return blk, err
-
-	default:
-		return nil, err
-	}
-}
-
-func (s *SplitStore) GetSize(cid cid.Cid) (int, error) {
-	if isIdentiyCid(cid) {
-		data, err := decodeIdentityCid(cid)
-		if err != nil {
-			return 0, err
-		}
-
-		return len(data), nil
-	}
-
-	s.txnLk.RLock()
-	defer s.txnLk.RUnlock()
-
-	size, err := s.hot.GetSize(cid)
-
-	switch err {
-	case nil:
-		s.trackTxnRef(cid)
-		return size, nil
-
-	case bstore.ErrNotFound:
-		if s.isWarm() {
-			s.debug.LogReadMiss(cid)
-		}
-
-		size, err = s.cold.GetSize(cid)
-		if err == nil {
-			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
-		}
-		return size, err
-
-	default:
-		return 0, err
-	}
-}
-
-func (s *SplitStore) Put(blk blocks.Block) error {
-	if isIdentiyCid(blk.Cid()) {
-		return nil
-	}
-
-	s.txnLk.RLock()
-	defer s.txnLk.RUnlock()
-
-	err := s.hot.Put(blk)
-	if err != nil {
-		return err
-	}
-
-	s.debug.LogWrite(blk)
-
-	s.trackTxnRef(blk.Cid())
-	return nil
-}
-
-func (s *SplitStore) PutMany(blks []blocks.Block) error {
-	// filter identites
-	idcids := 0
-	for _, blk := range blks {
-		if isIdentiyCid(blk.Cid()) {
-			idcids++
-		}
-	}
-
-	if idcids > 0 {
-		if idcids == len(blks) {
-			// it's all identities
-			return nil
-		}
-
-		filtered := make([]blocks.Block, 0, len(blks)-idcids)
-		for _, blk := range blks {
-			if isIdentiyCid(blk.Cid()) {
-				continue
-			}
-			filtered = append(filtered, blk)
-		}
-
-		blks = filtered
-	}
-
-	batch := make([]cid.Cid, 0, len(blks))
-	for _, blk := range blks {
-		batch = append(batch, blk.Cid())
-	}
-
-	s.txnLk.RLock()
-	defer s.txnLk.RUnlock()
-
-	err := s.hot.PutMany(blks)
-	if err != nil {
-		return err
-	}
-
-	s.debug.LogWriteMany(blks)
-
-	s.trackTxnRefMany(batch)
-	return nil
-}
-
-func (s *SplitStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	ctx, cancel := context.WithCancel(ctx)
-
-	chHot, err := s.hot.AllKeysChan(ctx)
-	if err != nil {
-		cancel()
-		return nil, err
-	}
-
-	chCold, err := s.cold.AllKeysChan(ctx)
-	if err != nil {
-		cancel()
-		return nil, err
-	}
-
-	seen := cid.NewSet()
-	ch := make(chan cid.Cid, 8) // buffer is arbitrary, just enough to avoid context switches
-	go func() {
-		defer cancel()
-		defer close(ch)
-
-		for _, in := range []<-chan cid.Cid{chHot, chCold} {
-			for c := range in {
-				// ensure we only emit each key once
-				if !seen.Visit(c) {
-					continue
-				}
-
-				select {
-				case ch <- c:
-				case <-ctx.Done():
-					return
-				}
-			}
-		}
-	}()
-
-	return ch, nil
-}
-
-func (s *SplitStore) HashOnRead(enabled bool) {
-	s.hot.HashOnRead(enabled)
-	s.cold.HashOnRead(enabled)
-}
-
-func (s *SplitStore) View(cid cid.Cid, cb func([]byte) error) error {
-	if isIdentiyCid(cid) {
-		data, err := decodeIdentityCid(cid)
-		if err != nil {
-			return err
-		}
-
-		return cb(data)
-	}
-
-	// views are (optimistically) protected two-fold:
-	// - if there is an active transaction, then the reference is protected.
-	// - if there is no active transaction, active views are tracked in a
-	//   wait group and compaction is inhibited from starting until they
-	//   have all completed.  this is necessary to ensure that a (very) long-running
-	//   view can't have its data pointer deleted, which would be catastrophic.
-	//   Note that we can't just RLock for the duration of the view, as this could
-	//    lead to deadlock with recursive views.
-	s.protectView(cid)
-	defer s.viewDone()
-
-	err := s.hot.View(cid, cb)
-	switch err {
-	case bstore.ErrNotFound:
-		if s.isWarm() {
-			s.debug.LogReadMiss(cid)
-		}
-
-		err = s.cold.View(cid, cb)
-		if err == nil {
-			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
-		}
-		return err
-
-	default:
-		return err
-	}
-}
-
-func (s *SplitStore) isWarm() bool {
-	s.mx.Lock()
-	defer s.mx.Unlock()
-	return s.warmupEpoch > 0
-}
-
-// State tracking
-func (s *SplitStore) Start(chain ChainAccessor) error {
-	s.chain = chain
-	curTs := chain.GetHeaviestTipSet()
-
-	// should we warmup
-	warmup := false
-
-	// load base epoch from metadata ds
-	// if none, then use current epoch because it's a fresh start
-	bs, err := s.ds.Get(baseEpochKey)
-	switch err {
-	case nil:
-		s.baseEpoch = bytesToEpoch(bs)
-
-	case dstore.ErrNotFound:
-		if curTs == nil {
-			// this can happen in some tests
-			break
-		}
-
-		err = s.setBaseEpoch(curTs.Height())
-		if err != nil {
-			return xerrors.Errorf("error saving base epoch: %w", err)
-		}
-
-	default:
-		return xerrors.Errorf("error loading base epoch: %w", err)
-	}
-
-	// load warmup epoch from metadata ds
-	bs, err = s.ds.Get(warmupEpochKey)
-	switch err {
-	case nil:
-		s.warmupEpoch = bytesToEpoch(bs)
-
-	case dstore.ErrNotFound:
-		warmup = true
-
-	default:
-		return xerrors.Errorf("error loading warmup epoch: %w", err)
-	}
-
-	// load markSetSize from metadata ds to provide a size hint for marksets
-	bs, err = s.ds.Get(markSetSizeKey)
-	switch err {
-	case nil:
-		s.markSetSize = bytesToInt64(bs)
-
-	case dstore.ErrNotFound:
-	default:
-		return xerrors.Errorf("error loading mark set size: %w", err)
-	}
-
-	// load compactionIndex from metadata ds to provide a hint as to when to perform moving gc
-	bs, err = s.ds.Get(compactionIndexKey)
-	switch err {
-	case nil:
-		s.compactionIndex = bytesToInt64(bs)
-
-	case dstore.ErrNotFound:
-		// this is potentially an upgrade from splitstore v0; schedule a warmup as v0 has
-		// some issues with hot references leaking into the coldstore.
-		warmup = true
-	default:
-		return xerrors.Errorf("error loading compaction index: %w", err)
-	}
-
-	log.Infow("starting splitstore", "baseEpoch", s.baseEpoch, "warmupEpoch", s.warmupEpoch)
-
-	if warmup {
-		err = s.warmup(curTs)
-		if err != nil {
-			return xerrors.Errorf("error starting warmup: %w", err)
-		}
-	}
-
-	// watch the chain
-	chain.SubscribeHeadChanges(s.HeadChange)
-
-	return nil
-}
-
-func (s *SplitStore) Close() error {
-	if !atomic.CompareAndSwapInt32(&s.closing, 0, 1) {
-		// already closing
-		return nil
-	}
-
-	if atomic.LoadInt32(&s.compacting) == 1 {
-		log.Warn("close with ongoing compaction in progress; waiting for it to finish...")
-		for atomic.LoadInt32(&s.compacting) == 1 {
-			time.Sleep(time.Second)
-		}
-	}
-
-	s.cancel()
-	return multierr.Combine(s.markSetEnv.Close(), s.debug.Close())
-}
-
-func (s *SplitStore) checkClosing() error {
-	if atomic.LoadInt32(&s.closing) == 1 {
-		return xerrors.Errorf("splitstore is closing")
-	}
-
-	return nil
-}
-
-func (s *SplitStore) setBaseEpoch(epoch abi.ChainEpoch) error {
-	s.baseEpoch = epoch
-	return s.ds.Put(baseEpochKey, epochToBytes(epoch))
-}

blockstore/splitstore/splitstore_expose.go

@@ -1,114 +0,0 @@
-package splitstore
-
-import (
-	"context"
-	"errors"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-
-	bstore "github.com/filecoin-project/lotus/blockstore"
-)
-
-type exposedSplitStore struct {
-	s *SplitStore
-}
-
-var _ bstore.Blockstore = (*exposedSplitStore)(nil)
-
-func (s *SplitStore) Expose() bstore.Blockstore {
-	return &exposedSplitStore{s: s}
-}
-
-func (es *exposedSplitStore) DeleteBlock(_ cid.Cid) error {
-	return errors.New("DeleteBlock: operation not supported")
-}
-
-func (es *exposedSplitStore) DeleteMany(_ []cid.Cid) error {
-	return errors.New("DeleteMany: operation not supported")
-}
-
-func (es *exposedSplitStore) Has(c cid.Cid) (bool, error) {
-	if isIdentiyCid(c) {
-		return true, nil
-	}
-
-	has, err := es.s.hot.Has(c)
-	if has || err != nil {
-		return has, err
-	}
-
-	return es.s.cold.Has(c)
-}
-
-func (es *exposedSplitStore) Get(c cid.Cid) (blocks.Block, error) {
-	if isIdentiyCid(c) {
-		data, err := decodeIdentityCid(c)
-		if err != nil {
-			return nil, err
-		}
-
-		return blocks.NewBlockWithCid(data, c)
-	}
-
-	blk, err := es.s.hot.Get(c)
-	switch err {
-	case bstore.ErrNotFound:
-		return es.s.cold.Get(c)
-	default:
-		return blk, err
-	}
-}
-
-func (es *exposedSplitStore) GetSize(c cid.Cid) (int, error) {
-	if isIdentiyCid(c) {
-		data, err := decodeIdentityCid(c)
-		if err != nil {
-			return 0, err
-		}
-
-		return len(data), nil
-	}
-
-	size, err := es.s.hot.GetSize(c)
-	switch err {
-	case bstore.ErrNotFound:
-		return es.s.cold.GetSize(c)
-	default:
-		return size, err
-	}
-}
-
-func (es *exposedSplitStore) Put(blk blocks.Block) error {
-	return es.s.Put(blk)
-}
-
-func (es *exposedSplitStore) PutMany(blks []blocks.Block) error {
-	return es.s.PutMany(blks)
-}
-
-func (es *exposedSplitStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return es.s.AllKeysChan(ctx)
-}
-
-func (es *exposedSplitStore) HashOnRead(enabled bool) {}
-
-func (es *exposedSplitStore) View(c cid.Cid, f func([]byte) error) error {
-	if isIdentiyCid(c) {
-		data, err := decodeIdentityCid(c)
-		if err != nil {
-			return err
-		}
-
-		return f(data)
-	}
-
-	err := es.s.hot.View(c, f)
-	switch err {
-	case bstore.ErrNotFound:
-		return es.s.cold.View(c, f)
-
-	default:
-		return err
-	}
-}

blockstore/splitstore/splitstore_gc.go

@@ -1,30 +0,0 @@
-package splitstore
-
-import (
-	"fmt"
-	"time"
-
-	bstore "github.com/filecoin-project/lotus/blockstore"
-)
-
-func (s *SplitStore) gcHotstore() {
-	if err := s.gcBlockstoreOnline(s.hot); err != nil {
-		log.Warnf("error garbage collecting hostore: %s", err)
-	}
-}
-
-func (s *SplitStore) gcBlockstoreOnline(b bstore.Blockstore) error {
-	if gc, ok := b.(bstore.BlockstoreGC); ok {
-		log.Info("garbage collecting blockstore")
-		startGC := time.Now()
-
-		if err := gc.CollectGarbage(); err != nil {
-			return err
-		}
-
-		log.Infow("garbage collecting hotstore done", "took", time.Since(startGC))
-		return nil
-	}
-
-	return fmt.Errorf("blockstore doesn't support online gc: %T", b)
-}

blockstore/splitstore/splitstore_test.go

@@ -1,333 +0,0 @@
-package splitstore
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/blockstore"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/chain/types/mock"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-	datastore "github.com/ipfs/go-datastore"
-	dssync "github.com/ipfs/go-datastore/sync"
-	logging "github.com/ipfs/go-log/v2"
-)
-
-func init() {
-	CompactionThreshold = 5
-	CompactionBoundary = 2
-	logging.SetLogLevel("splitstore", "DEBUG")
-}
-
-func testSplitStore(t *testing.T, cfg *Config) {
-	chain := &mockChain{t: t}
-
-	// the myriads of stores
-	ds := dssync.MutexWrap(datastore.NewMapDatastore())
-	hot := newMockStore()
-	cold := newMockStore()
-
-	// this is necessary to avoid the garbage mock puts in the blocks
-	garbage := blocks.NewBlock([]byte{1, 2, 3})
-	err := cold.Put(garbage)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// genesis
-	genBlock := mock.MkBlock(nil, 0, 0)
-	genBlock.Messages = garbage.Cid()
-	genBlock.ParentMessageReceipts = garbage.Cid()
-	genBlock.ParentStateRoot = garbage.Cid()
-	genBlock.Timestamp = uint64(time.Now().Unix())
-
-	genTs := mock.TipSet(genBlock)
-	chain.push(genTs)
-
-	// put the genesis block to cold store
-	blk, err := genBlock.ToStorageBlock()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = cold.Put(blk)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// open the splitstore
-	ss, err := Open("", ds, hot, cold, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer ss.Close() //nolint
-
-	err = ss.Start(chain)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// make some tipsets, but not enough to cause compaction
-	mkBlock := func(curTs *types.TipSet, i int, stateRoot blocks.Block) *types.TipSet {
-		blk := mock.MkBlock(curTs, uint64(i), uint64(i))
-
-		blk.Messages = garbage.Cid()
-		blk.ParentMessageReceipts = garbage.Cid()
-		blk.ParentStateRoot = stateRoot.Cid()
-		blk.Timestamp = uint64(time.Now().Unix())
-
-		sblk, err := blk.ToStorageBlock()
-		if err != nil {
-			t.Fatal(err)
-		}
-		err = ss.Put(stateRoot)
-		if err != nil {
-			t.Fatal(err)
-		}
-		err = ss.Put(sblk)
-		if err != nil {
-			t.Fatal(err)
-		}
-		ts := mock.TipSet(blk)
-		chain.push(ts)
-
-		return ts
-	}
-
-	waitForCompaction := func() {
-		for atomic.LoadInt32(&ss.compacting) == 1 {
-			time.Sleep(100 * time.Millisecond)
-		}
-	}
-
-	curTs := genTs
-	for i := 1; i < 5; i++ {
-		stateRoot := blocks.NewBlock([]byte{byte(i), 3, 3, 7})
-		curTs = mkBlock(curTs, i, stateRoot)
-		waitForCompaction()
-	}
-
-	// count objects in the cold and hot stores
-	countBlocks := func(bs blockstore.Blockstore) int {
-		count := 0
-		_ = bs.(blockstore.BlockstoreIterator).ForEachKey(func(_ cid.Cid) error {
-			count++
-			return nil
-		})
-		return count
-	}
-
-	coldCnt := countBlocks(cold)
-	hotCnt := countBlocks(hot)
-
-	if coldCnt != 2 {
-		t.Errorf("expected %d blocks, but got %d", 2, coldCnt)
-	}
-
-	if hotCnt != 10 {
-		t.Errorf("expected %d blocks, but got %d", 10, hotCnt)
-	}
-
-	// trigger a compaction
-	for i := 5; i < 10; i++ {
-		stateRoot := blocks.NewBlock([]byte{byte(i), 3, 3, 7})
-		curTs = mkBlock(curTs, i, stateRoot)
-		waitForCompaction()
-	}
-
-	coldCnt = countBlocks(cold)
-	hotCnt = countBlocks(hot)
-
-	if coldCnt != 5 {
-		t.Errorf("expected %d cold blocks, but got %d", 5, coldCnt)
-	}
-
-	if hotCnt != 17 {
-		t.Errorf("expected %d hot blocks, but got %d", 17, hotCnt)
-	}
-
-	// Make sure we can revert without panicking.
-	chain.revert(2)
-}
-
-func TestSplitStoreCompaction(t *testing.T) {
-	testSplitStore(t, &Config{MarkSetType: "map"})
-}
-
-type mockChain struct {
-	t testing.TB
-
-	sync.Mutex
-	genesis  *types.BlockHeader
-	tipsets  []*types.TipSet
-	listener func(revert []*types.TipSet, apply []*types.TipSet) error
-}
-
-func (c *mockChain) push(ts *types.TipSet) {
-	c.Lock()
-	c.tipsets = append(c.tipsets, ts)
-	if c.genesis == nil {
-		c.genesis = ts.Blocks()[0]
-	}
-	c.Unlock()
-
-	if c.listener != nil {
-		err := c.listener(nil, []*types.TipSet{ts})
-		if err != nil {
-			c.t.Errorf("mockchain: error dispatching listener: %s", err)
-		}
-	}
-}
-
-func (c *mockChain) revert(count int) {
-	c.Lock()
-	revert := make([]*types.TipSet, count)
-	if count > len(c.tipsets) {
-		c.Unlock()
-		c.t.Fatalf("not enough tipsets to revert")
-	}
-	copy(revert, c.tipsets[len(c.tipsets)-count:])
-	c.tipsets = c.tipsets[:len(c.tipsets)-count]
-	c.Unlock()
-
-	if c.listener != nil {
-		err := c.listener(revert, nil)
-		if err != nil {
-			c.t.Errorf("mockchain: error dispatching listener: %s", err)
-		}
-	}
-}
-
-func (c *mockChain) GetTipsetByHeight(_ context.Context, epoch abi.ChainEpoch, _ *types.TipSet, _ bool) (*types.TipSet, error) {
-	c.Lock()
-	defer c.Unlock()
-
-	iEpoch := int(epoch)
-	if iEpoch > len(c.tipsets) {
-		return nil, fmt.Errorf("bad epoch %d", epoch)
-	}
-
-	return c.tipsets[iEpoch], nil
-}
-
-func (c *mockChain) GetHeaviestTipSet() *types.TipSet {
-	c.Lock()
-	defer c.Unlock()
-
-	return c.tipsets[len(c.tipsets)-1]
-}
-
-func (c *mockChain) SubscribeHeadChanges(change func(revert []*types.TipSet, apply []*types.TipSet) error) {
-	c.listener = change
-}
-
-type mockStore struct {
-	mx  sync.Mutex
-	set map[cid.Cid]blocks.Block
-}
-
-func newMockStore() *mockStore {
-	return &mockStore{set: make(map[cid.Cid]blocks.Block)}
-}
-
-func (b *mockStore) Has(cid cid.Cid) (bool, error) {
-	b.mx.Lock()
-	defer b.mx.Unlock()
-	_, ok := b.set[cid]
-	return ok, nil
-}
-
-func (b *mockStore) HashOnRead(hor bool) {}
-
-func (b *mockStore) Get(cid cid.Cid) (blocks.Block, error) {
-	b.mx.Lock()
-	defer b.mx.Unlock()
-
-	blk, ok := b.set[cid]
-	if !ok {
-		return nil, blockstore.ErrNotFound
-	}
-	return blk, nil
-}
-
-func (b *mockStore) GetSize(cid cid.Cid) (int, error) {
-	blk, err := b.Get(cid)
-	if err != nil {
-		return 0, err
-	}
-
-	return len(blk.RawData()), nil
-}
-
-func (b *mockStore) View(cid cid.Cid, f func([]byte) error) error {
-	blk, err := b.Get(cid)
-	if err != nil {
-		return err
-	}
-	return f(blk.RawData())
-}
-
-func (b *mockStore) Put(blk blocks.Block) error {
-	b.mx.Lock()
-	defer b.mx.Unlock()
-
-	b.set[blk.Cid()] = blk
-	return nil
-}
-
-func (b *mockStore) PutMany(blks []blocks.Block) error {
-	b.mx.Lock()
-	defer b.mx.Unlock()
-
-	for _, blk := range blks {
-		b.set[blk.Cid()] = blk
-	}
-	return nil
-}
-
-func (b *mockStore) DeleteBlock(cid cid.Cid) error {
-	b.mx.Lock()
-	defer b.mx.Unlock()
-
-	delete(b.set, cid)
-	return nil
-}
-
-func (b *mockStore) DeleteMany(cids []cid.Cid) error {
-	b.mx.Lock()
-	defer b.mx.Unlock()
-
-	for _, c := range cids {
-		delete(b.set, c)
-	}
-	return nil
-}
-
-func (b *mockStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return nil, errors.New("not implemented")
-}
-
-func (b *mockStore) ForEachKey(f func(cid.Cid) error) error {
-	b.mx.Lock()
-	defer b.mx.Unlock()
-
-	for c := range b.set {
-		err := f(c)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (b *mockStore) Close() error {
-	return nil
-}

blockstore/splitstore/splitstore_util.go

@@ -1,67 +0,0 @@
-package splitstore
-
-import (
-	"encoding/binary"
-
-	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
-	mh "github.com/multiformats/go-multihash"
-
-	"github.com/filecoin-project/go-state-types/abi"
-)
-
-func epochToBytes(epoch abi.ChainEpoch) []byte {
-	return uint64ToBytes(uint64(epoch))
-}
-
-func bytesToEpoch(buf []byte) abi.ChainEpoch {
-	return abi.ChainEpoch(bytesToUint64(buf))
-}
-
-func int64ToBytes(i int64) []byte {
-	return uint64ToBytes(uint64(i))
-}
-
-func bytesToInt64(buf []byte) int64 {
-	return int64(bytesToUint64(buf))
-}
-
-func uint64ToBytes(i uint64) []byte {
-	buf := make([]byte, 16)
-	n := binary.PutUvarint(buf, i)
-	return buf[:n]
-}
-
-func bytesToUint64(buf []byte) uint64 {
-	i, _ := binary.Uvarint(buf)
-	return i
-}
-
-func isUnitaryObject(c cid.Cid) bool {
-	pre := c.Prefix()
-	switch pre.Codec {
-	case cid.FilCommitmentSealed, cid.FilCommitmentUnsealed:
-		return true
-	default:
-		return pre.MhType == mh.IDENTITY
-	}
-}
-
-func isIdentiyCid(c cid.Cid) bool {
-	return c.Prefix().MhType == mh.IDENTITY
-}
-
-func decodeIdentityCid(c cid.Cid) ([]byte, error) {
-	dmh, err := mh.Decode(c.Hash())
-	if err != nil {
-		return nil, xerrors.Errorf("error decoding identity cid %s: %w", c, err)
-	}
-
-	// sanity check
-	if dmh.Code != mh.IDENTITY {
-		return nil, xerrors.Errorf("error decoding identity cid %s: hash type is not identity", c)
-	}
-
-	return dmh.Digest, nil
-}

blockstore/splitstore/splitstore_warmup.go

@@ -1,126 +0,0 @@
-package splitstore
-
-import (
-	"sync/atomic"
-	"time"
-
-	"golang.org/x/xerrors"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-
-	bstore "github.com/filecoin-project/lotus/blockstore"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-// warmup acuiqres the compaction lock and spawns a goroutine to warm up the hotstore;
-// this is necessary when we sync from a snapshot or when we enable the splitstore
-// on top of an existing blockstore (which becomes the coldstore).
-func (s *SplitStore) warmup(curTs *types.TipSet) error {
-	if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
-		return xerrors.Errorf("error locking compaction")
-	}
-
-	go func() {
-		defer atomic.StoreInt32(&s.compacting, 0)
-
-		log.Info("warming up hotstore")
-		start := time.Now()
-
-		err := s.doWarmup(curTs)
-		if err != nil {
-			log.Errorf("error warming up hotstore: %s", err)
-			return
-		}
-
-		log.Infow("warm up done", "took", time.Since(start))
-	}()
-
-	return nil
-}
-
-// the actual warmup procedure; it walks the chain loading all state roots at the boundary
-// and headers all the way up to genesis.
-// objects are written in batches so as to minimize overhead.
-func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
-	epoch := curTs.Height()
-	batchHot := make([]blocks.Block, 0, batchSize)
-	count := int64(0)
-	xcount := int64(0)
-	missing := int64(0)
-	err := s.walkChain(curTs, epoch, false,
-		func(c cid.Cid) error {
-			if isUnitaryObject(c) {
-				return errStopWalk
-			}
-
-			count++
-
-			has, err := s.hot.Has(c)
-			if err != nil {
-				return err
-			}
-
-			if has {
-				return nil
-			}
-
-			blk, err := s.cold.Get(c)
-			if err != nil {
-				if err == bstore.ErrNotFound {
-					missing++
-					return nil
-				}
-				return err
-			}
-
-			xcount++
-
-			batchHot = append(batchHot, blk)
-			if len(batchHot) == batchSize {
-				err = s.hot.PutMany(batchHot)
-				if err != nil {
-					return err
-				}
-				batchHot = batchHot[:0]
-			}
-
-			return nil
-		})
-
-	if err != nil {
-		return err
-	}
-
-	if len(batchHot) > 0 {
-		err = s.hot.PutMany(batchHot)
-		if err != nil {
-			return err
-		}
-	}
-
-	log.Infow("warmup stats", "visited", count, "warm", xcount, "missing", missing)
-
-	s.markSetSize = count + count>>2 // overestimate a bit
-	err = s.ds.Put(markSetSizeKey, int64ToBytes(s.markSetSize))
-	if err != nil {
-		log.Warnf("error saving mark set size: %s", err)
-	}
-
-	// save the warmup epoch
-	err = s.ds.Put(warmupEpochKey, epochToBytes(epoch))
-	if err != nil {
-		return xerrors.Errorf("error saving warm up epoch: %w", err)
-	}
-	s.mx.Lock()
-	s.warmupEpoch = epoch
-	s.mx.Unlock()
-
-	// also save the compactionIndex, as this is used as an indicator of warmup for upgraded nodes
-	err = s.ds.Put(compactionIndexKey, int64ToBytes(s.compactionIndex))
-	if err != nil {
-		return xerrors.Errorf("error saving compaction index: %w", err)
-	}
-
-	return nil
-}

blockstore/sync.go

@@ -1,81 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"sync"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-)
-
-// NewMemorySync returns a thread-safe in-memory blockstore.
-func NewMemorySync() *SyncBlockstore {
-	return &SyncBlockstore{bs: make(MemBlockstore)}
-}
-
-// SyncBlockstore is a terminal blockstore that is a synchronized version
-// of MemBlockstore.
-type SyncBlockstore struct {
-	mu sync.RWMutex
-	bs MemBlockstore // specifically use a memStore to save indirection overhead.
-}
-
-func (m *SyncBlockstore) DeleteBlock(k cid.Cid) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return m.bs.DeleteBlock(k)
-}
-
-func (m *SyncBlockstore) DeleteMany(ks []cid.Cid) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return m.bs.DeleteMany(ks)
-}
-
-func (m *SyncBlockstore) Has(k cid.Cid) (bool, error) {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	return m.bs.Has(k)
-}
-
-func (m *SyncBlockstore) View(k cid.Cid, callback func([]byte) error) error {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-
-	return m.bs.View(k, callback)
-}
-
-func (m *SyncBlockstore) Get(k cid.Cid) (blocks.Block, error) {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	return m.bs.Get(k)
-}
-
-func (m *SyncBlockstore) GetSize(k cid.Cid) (int, error) {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	return m.bs.GetSize(k)
-}
-
-func (m *SyncBlockstore) Put(b blocks.Block) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return m.bs.Put(b)
-}
-
-func (m *SyncBlockstore) PutMany(bs []blocks.Block) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return m.bs.PutMany(bs)
-}
-
-func (m *SyncBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	// this blockstore implementation doesn't do any async work.
-	return m.bs.AllKeysChan(ctx)
-}
-
-func (m *SyncBlockstore) HashOnRead(enabled bool) {
-	// noop
-}

blockstore/timed.go

@@ -1,185 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"fmt"
-	"sync"
-	"time"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	"github.com/raulk/clock"
-	"go.uber.org/multierr"
-)
-
-// TimedCacheBlockstore is a blockstore that keeps blocks for at least the
-// specified caching interval before discarding them. Garbage collection must
-// be started and stopped by calling Start/Stop.
-//
-// Under the covers, it's implemented with an active and an inactive blockstore
-// that are rotated every cache time interval. This means all blocks will be
-// stored at most 2x the cache interval.
-//
-// Create a new instance by calling the NewTimedCacheBlockstore constructor.
-type TimedCacheBlockstore struct {
-	mu               sync.RWMutex
-	active, inactive MemBlockstore
-	clock            clock.Clock
-	interval         time.Duration
-	closeCh          chan struct{}
-	doneRotatingCh   chan struct{}
-}
-
-func NewTimedCacheBlockstore(interval time.Duration) *TimedCacheBlockstore {
-	b := &TimedCacheBlockstore{
-		active:   NewMemory(),
-		inactive: NewMemory(),
-		interval: interval,
-		clock:    clock.New(),
-	}
-	return b
-}
-
-func (t *TimedCacheBlockstore) Start(_ context.Context) error {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	if t.closeCh != nil {
-		return fmt.Errorf("already started")
-	}
-	t.closeCh = make(chan struct{})
-	go func() {
-		ticker := t.clock.Ticker(t.interval)
-		defer ticker.Stop()
-		for {
-			select {
-			case <-ticker.C:
-				t.rotate()
-				if t.doneRotatingCh != nil {
-					t.doneRotatingCh <- struct{}{}
-				}
-			case <-t.closeCh:
-				return
-			}
-		}
-	}()
-	return nil
-}
-
-func (t *TimedCacheBlockstore) Stop(_ context.Context) error {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	if t.closeCh == nil {
-		return fmt.Errorf("not started")
-	}
-	select {
-	case <-t.closeCh:
-		// already closed
-	default:
-		close(t.closeCh)
-	}
-	return nil
-}
-
-func (t *TimedCacheBlockstore) rotate() {
-	newBs := NewMemory()
-
-	t.mu.Lock()
-	t.inactive, t.active = t.active, newBs
-	t.mu.Unlock()
-}
-
-func (t *TimedCacheBlockstore) Put(b blocks.Block) error {
-	// Don't check the inactive set here. We want to keep this block for at
-	// least one interval.
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	return t.active.Put(b)
-}
-
-func (t *TimedCacheBlockstore) PutMany(bs []blocks.Block) error {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	return t.active.PutMany(bs)
-}
-
-func (t *TimedCacheBlockstore) View(k cid.Cid, callback func([]byte) error) error {
-	// The underlying blockstore is always a "mem" blockstore so there's no difference,
-	// from a performance perspective, between view & get. So we call Get to avoid
-	// calling an arbitrary callback while holding a lock.
-	t.mu.RLock()
-	block, err := t.active.Get(k)
-	if err == ErrNotFound {
-		block, err = t.inactive.Get(k)
-	}
-	t.mu.RUnlock()
-
-	if err != nil {
-		return err
-	}
-	return callback(block.RawData())
-}
-
-func (t *TimedCacheBlockstore) Get(k cid.Cid) (blocks.Block, error) {
-	t.mu.RLock()
-	defer t.mu.RUnlock()
-	b, err := t.active.Get(k)
-	if err == ErrNotFound {
-		b, err = t.inactive.Get(k)
-	}
-	return b, err
-}
-
-func (t *TimedCacheBlockstore) GetSize(k cid.Cid) (int, error) {
-	t.mu.RLock()
-	defer t.mu.RUnlock()
-	size, err := t.active.GetSize(k)
-	if err == ErrNotFound {
-		size, err = t.inactive.GetSize(k)
-	}
-	return size, err
-}
-
-func (t *TimedCacheBlockstore) Has(k cid.Cid) (bool, error) {
-	t.mu.RLock()
-	defer t.mu.RUnlock()
-	if has, err := t.active.Has(k); err != nil {
-		return false, err
-	} else if has {
-		return true, nil
-	}
-	return t.inactive.Has(k)
-}
-
-func (t *TimedCacheBlockstore) HashOnRead(_ bool) {
-	// no-op
-}
-
-func (t *TimedCacheBlockstore) DeleteBlock(k cid.Cid) error {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	return multierr.Combine(t.active.DeleteBlock(k), t.inactive.DeleteBlock(k))
-}
-
-func (t *TimedCacheBlockstore) DeleteMany(ks []cid.Cid) error {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	return multierr.Combine(t.active.DeleteMany(ks), t.inactive.DeleteMany(ks))
-}
-
-func (t *TimedCacheBlockstore) AllKeysChan(_ context.Context) (<-chan cid.Cid, error) {
-	t.mu.RLock()
-	defer t.mu.RUnlock()
-
-	ch := make(chan cid.Cid, len(t.active)+len(t.inactive))
-	for c := range t.active {
-		ch <- c
-	}
-	for c := range t.inactive {
-		if _, ok := t.active[c]; ok {
-			continue
-		}
-		ch <- c
-	}
-	close(ch)
-	return ch, nil
-}

blockstore/timed_test.go

@@ -1,85 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"github.com/raulk/clock"
-	"github.com/stretchr/testify/require"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-)
-
-func TestTimedCacheBlockstoreSimple(t *testing.T) {
-	tc := NewTimedCacheBlockstore(10 * time.Millisecond)
-	mClock := clock.NewMock()
-	mClock.Set(time.Now())
-	tc.clock = mClock
-	tc.doneRotatingCh = make(chan struct{})
-
-	_ = tc.Start(context.Background())
-	mClock.Add(1) // IDK why it is needed but it makes it work
-
-	defer func() {
-		_ = tc.Stop(context.Background())
-	}()
-
-	b1 := blocks.NewBlock([]byte("foo"))
-	require.NoError(t, tc.Put(b1))
-
-	b2 := blocks.NewBlock([]byte("bar"))
-	require.NoError(t, tc.Put(b2))
-
-	b3 := blocks.NewBlock([]byte("baz"))
-
-	b1out, err := tc.Get(b1.Cid())
-	require.NoError(t, err)
-	require.Equal(t, b1.RawData(), b1out.RawData())
-
-	has, err := tc.Has(b1.Cid())
-	require.NoError(t, err)
-	require.True(t, has)
-
-	mClock.Add(10 * time.Millisecond)
-	<-tc.doneRotatingCh
-
-	// We should still have everything.
-	has, err = tc.Has(b1.Cid())
-	require.NoError(t, err)
-	require.True(t, has)
-
-	has, err = tc.Has(b2.Cid())
-	require.NoError(t, err)
-	require.True(t, has)
-
-	// extend b2, add b3.
-	require.NoError(t, tc.Put(b2))
-	require.NoError(t, tc.Put(b3))
-
-	// all keys once.
-	allKeys, err := tc.AllKeysChan(context.Background())
-	var ks []cid.Cid
-	for k := range allKeys {
-		ks = append(ks, k)
-	}
-	require.NoError(t, err)
-	require.ElementsMatch(t, ks, []cid.Cid{b1.Cid(), b2.Cid(), b3.Cid()})
-
-	mClock.Add(10 * time.Millisecond)
-	<-tc.doneRotatingCh
-	// should still have b2, and b3, but not b1
-
-	has, err = tc.Has(b1.Cid())
-	require.NoError(t, err)
-	require.False(t, has)
-
-	has, err = tc.Has(b2.Cid())
-	require.NoError(t, err)
-	require.True(t, has)
-
-	has, err = tc.Has(b3.Cid())
-	require.NoError(t, err)
-	require.True(t, has)
-}

blockstore/union.go

@@ -1,119 +0,0 @@
-package blockstore
-
-import (
-	"context"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-)
-
-type unionBlockstore []Blockstore
-
-// Union returns an unioned blockstore.
-//
-// * Reads return from the first blockstore that has the value, querying in the
-//   supplied order.
-// * Writes (puts and deltes) are broadcast to all stores.
-//
-func Union(stores ...Blockstore) Blockstore {
-	return unionBlockstore(stores)
-}
-
-func (m unionBlockstore) Has(cid cid.Cid) (has bool, err error) {
-	for _, bs := range m {
-		if has, err = bs.Has(cid); has || err != nil {
-			break
-		}
-	}
-	return has, err
-}
-
-func (m unionBlockstore) Get(cid cid.Cid) (blk blocks.Block, err error) {
-	for _, bs := range m {
-		if blk, err = bs.Get(cid); err == nil || err != ErrNotFound {
-			break
-		}
-	}
-	return blk, err
-}
-
-func (m unionBlockstore) View(cid cid.Cid, callback func([]byte) error) (err error) {
-	for _, bs := range m {
-		if err = bs.View(cid, callback); err == nil || err != ErrNotFound {
-			break
-		}
-	}
-	return err
-}
-
-func (m unionBlockstore) GetSize(cid cid.Cid) (size int, err error) {
-	for _, bs := range m {
-		if size, err = bs.GetSize(cid); err == nil || err != ErrNotFound {
-			break
-		}
-	}
-	return size, err
-}
-
-func (m unionBlockstore) Put(block blocks.Block) (err error) {
-	for _, bs := range m {
-		if err = bs.Put(block); err != nil {
-			break
-		}
-	}
-	return err
-}
-
-func (m unionBlockstore) PutMany(blks []blocks.Block) (err error) {
-	for _, bs := range m {
-		if err = bs.PutMany(blks); err != nil {
-			break
-		}
-	}
-	return err
-}
-
-func (m unionBlockstore) DeleteBlock(cid cid.Cid) (err error) {
-	for _, bs := range m {
-		if err = bs.DeleteBlock(cid); err != nil {
-			break
-		}
-	}
-	return err
-}
-
-func (m unionBlockstore) DeleteMany(cids []cid.Cid) (err error) {
-	for _, bs := range m {
-		if err = bs.DeleteMany(cids); err != nil {
-			break
-		}
-	}
-	return err
-}
-
-func (m unionBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	// this does not deduplicate; this interface needs to be revisited.
-	outCh := make(chan cid.Cid)
-
-	go func() {
-		defer close(outCh)
-
-		for _, bs := range m {
-			ch, err := bs.AllKeysChan(ctx)
-			if err != nil {
-				return
-			}
-			for cid := range ch {
-				outCh <- cid
-			}
-		}
-	}()
-
-	return outCh, nil
-}
-
-func (m unionBlockstore) HashOnRead(enabled bool) {
-	for _, bs := range m {
-		bs.HashOnRead(enabled)
-	}
-}

blockstore/union_test.go

@@ -1,102 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"testing"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/stretchr/testify/require"
-)
-
-var (
-	b0 = blocks.NewBlock([]byte("abc"))
-	b1 = blocks.NewBlock([]byte("foo"))
-	b2 = blocks.NewBlock([]byte("bar"))
-)
-
-func TestUnionBlockstore_Get(t *testing.T) {
-	m1 := NewMemory()
-	m2 := NewMemory()
-
-	_ = m1.Put(b1)
-	_ = m2.Put(b2)
-
-	u := Union(m1, m2)
-
-	v1, err := u.Get(b1.Cid())
-	require.NoError(t, err)
-	require.Equal(t, b1.RawData(), v1.RawData())
-
-	v2, err := u.Get(b2.Cid())
-	require.NoError(t, err)
-	require.Equal(t, b2.RawData(), v2.RawData())
-}
-
-func TestUnionBlockstore_Put_PutMany_Delete_AllKeysChan(t *testing.T) {
-	m1 := NewMemory()
-	m2 := NewMemory()
-
-	u := Union(m1, m2)
-
-	err := u.Put(b0)
-	require.NoError(t, err)
-
-	var has bool
-
-	// write was broadcasted to all stores.
-	has, _ = m1.Has(b0.Cid())
-	require.True(t, has)
-
-	has, _ = m2.Has(b0.Cid())
-	require.True(t, has)
-
-	has, _ = u.Has(b0.Cid())
-	require.True(t, has)
-
-	// put many.
-	err = u.PutMany([]blocks.Block{b1, b2})
-	require.NoError(t, err)
-
-	// write was broadcasted to all stores.
-	has, _ = m1.Has(b1.Cid())
-	require.True(t, has)
-
-	has, _ = m1.Has(b2.Cid())
-	require.True(t, has)
-
-	has, _ = m2.Has(b1.Cid())
-	require.True(t, has)
-
-	has, _ = m2.Has(b2.Cid())
-	require.True(t, has)
-
-	// also in the union store.
-	has, _ = u.Has(b1.Cid())
-	require.True(t, has)
-
-	has, _ = u.Has(b2.Cid())
-	require.True(t, has)
-
-	// deleted from all stores.
-	err = u.DeleteBlock(b1.Cid())
-	require.NoError(t, err)
-
-	has, _ = u.Has(b1.Cid())
-	require.False(t, has)
-
-	has, _ = m1.Has(b1.Cid())
-	require.False(t, has)
-
-	has, _ = m2.Has(b1.Cid())
-	require.False(t, has)
-
-	// check that AllKeysChan returns b0 and b2, twice (once per backing store)
-	ch, err := u.AllKeysChan(context.Background())
-	require.NoError(t, err)
-
-	var i int
-	for range ch {
-		i++
-	}
-	require.Equal(t, 4, i)
-}

build/bootstrap.go

@@ -2,33 +2,44 @@ package build
 
 import (
 	"context"
-	"embed"
+	"os"
-	"path"
 	"strings"
 
 	"github.com/filecoin-project/lotus/lib/addrutil"
+	"golang.org/x/xerrors"
 
+	rice "github.com/GeertJohan/go.rice"
 	"github.com/libp2p/go-libp2p-core/peer"
 )
 
-//go:embed bootstrap
-var bootstrapfs embed.FS
-
 func BuiltinBootstrap() ([]peer.AddrInfo, error) {
-	if DisableBuiltinAssets {
+	var out []peer.AddrInfo
-		return nil, nil
+
-	}
+	b := rice.MustFindBox("bootstrap")
-	if BootstrappersFile != "" {
+	err := b.Walk("", func(path string, info os.FileInfo, err error) error {
-		spi, err := bootstrapfs.ReadFile(path.Join("bootstrap", BootstrappersFile))
 		if err != nil {
-			return nil, err
+			return xerrors.Errorf("failed to walk box: %w", err)
-		}
-		if len(spi) == 0 {
-			return nil, nil
 		}
 
-		return addrutil.ParseAddresses(context.TODO(), strings.Split(strings.TrimSpace(string(spi)), "\n"))
+		if !strings.HasSuffix(path, ".pi") {
+			return nil
+		}
+		spi := b.MustString(path)
+		if spi == "" {
+			return nil
+		}
+		pi, err := addrutil.ParseAddresses(context.TODO(), strings.Split(strings.TrimSpace(spi), "\n"))
+		out = append(out, pi...)
+		return err
+	})
+	return out, err
+}
+
+func DrandBootstrap() ([]peer.AddrInfo, error) {
+	addrs := []string{
+		"/dnsaddr/pl-eu.testnet.drand.sh/",
+		"/dnsaddr/pl-us.testnet.drand.sh/",
+		"/dnsaddr/pl-sin.testnet.drand.sh/",
 	}
-
+	return addrutil.ParseAddresses(context.TODO(), addrs)
-	return nil, nil
 }

build/bootstrap/bootstrappers.pi

@@ -0,0 +1,6 @@
+/dns4/t01000.miner.interopnet.kittyhawk.wtf/tcp/1347/p2p/12D3KooWLVgeFhXA7ztuSS1uvk4NVzZwRZLxg8PBy1mJuHfc8J6E
+/ip4/34.217.110.132/tcp/1347/p2p/12D3KooWLVgeFhXA7ztuSS1uvk4NVzZwRZLxg8PBy1mJuHfc8J6E
+/dns4/peer0.interopnet.kittyhawk.wtf/tcp/1347/p2p/12D3KooWBkXWpT1hLHXz7VzR8ZHCJ42P6c9Nb9532ATSguBpLTKJ
+/ip4/54.187.182.170/tcp/1347/p2p/12D3KooWBkXWpT1hLHXz7VzR8ZHCJ42P6c9Nb9532ATSguBpLTKJ
+/dns4/peer1.interopnet.kittyhawk.wtf/tcp/1347/p2p/12D3KooWJk5hQHNdCsidtX4jhKQMBXKJdo2kjhrNHUZBPmCZKwKt
+/ip4/52.24.84.39/tcp/1347/p2p/12D3KooWJk5hQHNdCsidtX4jhKQMBXKJdo2kjhrNHUZBPmCZKwKt