New interop network info

.circleci/config.yml

@@ -1,13 +1,11 @@
 version: 2.1
 orbs:
   go: gotest/tools@0.0.13
-  aws-cli: circleci/aws-cli@1.3.2
-  packer: salaxander/packer@0.0.3
 
 executors:
   golang:
     docker:
-      - image: circleci/golang:1.15.5
+      - image: circleci/golang:1.14.2
     resource_class: 2xlarge
   ubuntu:
     docker:
@@ -36,7 +34,7 @@ commands:
           condition: << parameters.linux >>
           steps:
             - run: sudo apt-get update
-            - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
+            - run: sudo apt-get install ocl-icd-opencl-dev
       - run: git submodule sync
       - run: git submodule update --init
   download-params:
@@ -47,7 +45,7 @@ commands:
             - 'v25-2k-lotus-params'
           paths:
             - /var/tmp/filecoin-proof-parameters/
-      - run:  ./lotus fetch-params 2048
+      - run:  ./lotus fetch-params --proving-params 2048
       - save_cache:
           name: Save parameters cache
           key: 'v25-2k-lotus-params'
@@ -95,10 +93,10 @@ jobs:
       - store_artifacts:
           path: lotus
       - store_artifacts:
-          path: lotus-miner
+          path: lotus-storage-miner
       - store_artifacts:
-          path: lotus-worker
-      - run: mkdir linux && mv lotus lotus-miner lotus-worker linux/
+          path: lotus-seal-worker
+      - run: mkdir linux && mv lotus lotus-storage-miner lotus-seal-worker linux/
       - persist_to_workspace:
           root: "."
           paths:
@@ -115,7 +113,7 @@ jobs:
   test: &test
     description: |
       Run tests with gotestsum.
-    parameters: &test-params
+    parameters:
       executor:
         type: executor
         default: golang
@@ -136,7 +134,7 @@ jobs:
         description: Test suite name to report to CircleCI.
       gotestsum-format:
         type: string
-        default: pkgname-and-test-fails
+        default: short
         description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
       coverage:
         type: string
@@ -158,28 +156,21 @@ jobs:
       - download-params
       - go/install-gotestsum:
           gobin: $HOME/.local/bin
-          version: 0.5.2
       - run:
           name: go test
           environment:
+            GOTESTSUM_JUNITFILE: /tmp/test-reports/<< parameters.test-suite-name >>/junit.xml
+            GOTESTSUM_FORMAT: << parameters.gotestsum-format >>
             LOTUS_TEST_WINDOW_POST: << parameters.winpost-test >>
-            SKIP_CONFORMANCE: "1"
           command: |
             mkdir -p /tmp/test-reports/<< parameters.test-suite-name >>
-            mkdir -p /tmp/test-artifacts
-            gotestsum \
-              --format << parameters.gotestsum-format >> \
-              --junitfile /tmp/test-reports/<< parameters.test-suite-name >>/junit.xml \
-              --jsonfile /tmp/test-artifacts/<< parameters.test-suite-name >>.json \
-              -- \
+            gotestsum -- \
               << parameters.coverage >> \
               << parameters.go-test-flags >> \
               << parameters.packages >>
           no_output_timeout: 30m
       - store_test_results:
           path: /tmp/test-reports
-      - store_artifacts:
-          path: /tmp/test-artifacts/<< parameters.test-suite-name >>.json
       - when:
           condition: << parameters.codecov-upload >>
           steps:
@@ -190,146 +181,10 @@ jobs:
                 command: |
                   bash <(curl -s https://codecov.io/bash)
 
-  test-chain:
-    <<: *test
-  test-node:
-    <<: *test
-  test-storage:
-    <<: *test
-  test-cli:
-    <<: *test
   test-short:
     <<: *test
   test-window-post:
     <<: *test
-  test-window-post-dispute:
-    <<: *test
-  test-terminate:
-    <<: *test
-  test-conformance:
-    description: |
-      Run tests using a corpus of interoperable test vectors for Filecoin 
-      implementations to test their correctness and compliance with the Filecoin
-      specifications.
-    parameters:
-      <<: *test-params
-      vectors-branch:
-        type: string
-        default: ""
-        description: |
-          Branch on github.com/filecoin-project/test-vectors to checkout and
-          test with. If empty (the default) the commit defined by the git
-          submodule is used.
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          command: make deps lotus
-          no_output_timeout: 30m
-      - download-params
-      - when:
-          condition:
-            not:
-              equal: [ "", << parameters.vectors-branch >> ]
-          steps:
-            - run:
-                name: checkout vectors branch
-                command: |
-                  cd extern/test-vectors
-                  git fetch
-                  git checkout origin/<< parameters.vectors-branch >>
-      - go/install-gotestsum:
-          gobin: $HOME/.local/bin
-          version: 0.5.2
-      - run:
-          name: install statediff globally
-          command: |
-            ## statediff is optional; we succeed even if compilation fails.
-            mkdir -p /tmp/statediff
-            git clone https://github.com/filecoin-project/statediff.git /tmp/statediff
-            cd /tmp/statediff
-            go install ./cmd/statediff || exit 0
-      - run:
-          name: go test
-          environment:
-            SKIP_CONFORMANCE: "0"
-          command: |
-            mkdir -p /tmp/test-reports
-            mkdir -p /tmp/test-artifacts
-            gotestsum \
-              --format pkgname-and-test-fails \
-              --junitfile /tmp/test-reports/junit.xml \
-              -- \
-              -v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/
-            go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html
-          no_output_timeout: 30m
-      - store_test_results:
-          path: /tmp/test-reports
-      - store_artifacts:
-          path: /tmp/test-artifacts/conformance-coverage.html
-  build-ntwk-calibration:
-    description: |
-      Compile lotus binaries for the calibration network
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: make calibnet
-      - run: mkdir linux-calibnet && mv lotus lotus-miner lotus-worker linux-calibnet
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux-calibnet
-  build-lotus-soup:
-    description: |
-      Compile `lotus-soup` Testground test plan
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: cd extern/filecoin-ffi && make
-      - run:
-          name: "go get lotus@master"
-          command: cd testplans/lotus-soup && go mod edit -replace=github.com/filecoin-project/lotus=../..
-      - run:
-          name: "build lotus-soup testplan"
-          command: pushd testplans/lotus-soup && go build -tags=testground .
-  trigger-testplans:
-    description: |
-      Trigger `lotus-soup` test cases on TaaS
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run:
-          name: "download testground"
-          command: wget https://gist.github.com/nonsense/5fbf3167cac79945f658771aed32fc44/raw/2e17eb0debf7ec6bdf027c1bdafc2c92dd97273b/testground-d3e9603 -O ~/testground-cli && chmod +x ~/testground-cli
-      - run:
-          name: "prepare .env.toml"
-          command: pushd testplans/lotus-soup && mkdir -p $HOME/testground && cp env-ci.toml $HOME/testground/.env.toml && echo 'endpoint="https://ci.testground.ipfs.team"' >> $HOME/testground/.env.toml && echo 'user="circleci"' >> $HOME/testground/.env.toml
-      - run:
-          name: "prepare testground home dir and link test plans"
-          command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup && ln -s $(pwd)/testplans/graphsync $HOME/testground/plans/graphsync
-      - run:
-          name: "go get lotus@master"
-          command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master
-      - run:
-          name: "trigger deals baseline testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/baseline-k8s-3-1.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-      - run:
-          name: "trigger payment channel stress testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-      - run:
-          name: "trigger graphsync testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/graphsync/_compositions/stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-
 
   build-macos:
     description: build darwin lotus binary
@@ -343,8 +198,8 @@ jobs:
       - run:
           name: Install go
           command: |
-            curl -O https://dl.google.com/go/go1.15.5.darwin-amd64.pkg && \
-            sudo installer -pkg go1.15.5.darwin-amd64.pkg -target /
+            curl -O https://dl.google.com/go/go1.14.2.darwin-amd64.pkg && \
+            sudo installer -pkg go1.14.2.darwin-amd64.pkg -target /
       - run:
           name: Install pkg-config
           command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
@@ -368,10 +223,10 @@ jobs:
       - store_artifacts:
           path: lotus
       - store_artifacts:
-          path: lotus-miner
+          path: lotus-storage-miner
       - store_artifacts:
-          path: lotus-worker
-      - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
+          path: lotus-seal-worker
+      - run: mkdir darwin && mv lotus lotus-storage-miner lotus-seal-worker darwin/
       - persist_to_workspace:
           root: "."
           paths:
@@ -391,28 +246,6 @@ jobs:
       - run:
           command: "! go fmt ./... 2>&1 | read"
 
-  cbor-gen-check:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run: make deps
-      - run: go install golang.org/x/tools/cmd/goimports
-      - run: go install github.com/hannahhoward/cbor-gen-for
-      - run: go generate ./...
-      - run: git --no-pager diff
-      - run: git --no-pager diff --quiet
-
-  docs-check:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run: make deps
-      - run: make docsgen
-      - run: git --no-pager diff
-      - run: git --no-pager diff --quiet
-
   lint: &lint
     description: |
       Run golangci-lint.
@@ -449,6 +282,9 @@ jobs:
           command: |
             $HOME/.local/bin/golangci-lint run -v --timeout 2m \
               --concurrency << parameters.concurrency >> << parameters.args >>
+  lint-changes:
+    <<: *lint
+
   lint-all:
     <<: *lint
 
@@ -472,205 +308,26 @@ jobs:
           name: Publish release
           command: ./scripts/publish-release.sh
 
-  build-and-push-image:
-    description: build and push docker images to public AWS ECR registry
-    executor: aws-cli/default
-    parameters:
-      profile-name:
-        type: string
-        default: "default"
-        description: AWS profile name to be configured.
-
-      aws-access-key-id:
-        type: env_var_name
-        default: AWS_ACCESS_KEY_ID
-        description: >
-          AWS access key id for IAM role. Set this to the name of
-          the environment variable you will set to hold this
-          value, i.e. AWS_ACCESS_KEY.
-
-      aws-secret-access-key:
-        type: env_var_name
-        default: AWS_SECRET_ACCESS_KEY
-        description: >
-          AWS secret key for IAM role. Set this to the name of
-          the environment variable you will set to hold this
-          value, i.e. AWS_SECRET_ACCESS_KEY.
-
-      region:
-        type: env_var_name
-        default: AWS_REGION
-        description: >
-          Name of env var storing your AWS region information,
-          defaults to AWS_REGION
-
-      account-url:
-        type: env_var_name
-        default: AWS_ECR_ACCOUNT_URL
-        description: >
-          Env var storing Amazon ECR account URL that maps to an AWS account,
-          e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com
-          defaults to AWS_ECR_ACCOUNT_URL
-
-      dockerfile:
-        type: string
-        default: Dockerfile
-        description: Name of dockerfile to use. Defaults to Dockerfile.
-
-      path:
-        type: string
-        default: .
-        description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory).
-
-      extra-build-args:
-        type: string
-        default: ""
-        description: >
-          Extra flags to pass to docker build. For examples, see
-          https://docs.docker.com/engine/reference/commandline/build
-
-      repo:
-        type: string
-        description: Name of an Amazon ECR repository
-
-      tag:
-        type: string
-        default: "latest"
-        description: A comma-separated string containing docker image tags to build and push (default = latest)
-
-    steps:
-      - run:
-          name: Confirm that environment variables are set
-          command: |
-            if [ -z "$AWS_ACCESS_KEY_ID" ]; then
-              echo "No AWS_ACCESS_KEY_ID is set. Skipping build-and-push job ..."
-              circleci-agent step halt
-            fi
-
-      - aws-cli/setup:
-          profile-name: <<parameters.profile-name>>
-          aws-access-key-id: <<parameters.aws-access-key-id>>
-          aws-secret-access-key: <<parameters.aws-secret-access-key>>
-          aws-region: <<parameters.region>>
-
-      - run:
-          name: Log into Amazon ECR
-          command: |
-            aws ecr-public get-login-password --region $<<parameters.region>> --profile <<parameters.profile-name>> | docker login --username AWS --password-stdin $<<parameters.account-url>>
-
-      - checkout
-
-      - setup_remote_docker:
-          version: 19.03.13
-          docker_layer_caching: false
-
-      - run:
-          name: Build docker image
-          command: |
-            registry_id=$(echo $<<parameters.account-url>> | sed "s;\..*;;g")
-
-            docker_tag_args=""
-            IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
-            for tag in "${DOCKER_TAGS[@]}"; do
-              docker_tag_args="$docker_tag_args -t $<<parameters.account-url>>/<<parameters.repo>>:$tag"
-            done
-
-            docker build \
-              <<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
-              -f <<parameters.path>>/<<parameters.dockerfile>> \
-              $docker_tag_args \
-              <<parameters.path>>
-
-      - run:
-          name: Push image to Amazon ECR
-          command: |
-            IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
-            for tag in "${DOCKER_TAGS[@]}"; do
-              docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
-            done
-
-  publish-packer:
-    description: build and push AWS IAM and DigitalOcean droplet.
-    executor:
-      name: packer/default
-      packer-version: 1.6.6
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
-      - packer/build:
-          template: tools/packer/lotus.pkr.hcl
-          args: "-var ci_workspace_bins=./linux-calibnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
 
 workflows:
   version: 2.1
   ci:
     jobs:
-      - lint-all:
-          concurrency: "16"   # expend all docker 2xlarge CPUs.
+      - lint-changes:
+          args: "--new-from-rev origin/next" 
       - mod-tidy-check
       - gofmt
-      - cbor-gen-check
-      - docs-check
       - test:
           codecov-upload: true
-          test-suite-name: full
-      - test-chain:
-          codecov-upload: true
-          test-suite-name: chain
-          packages: "./chain/..."
-      - test-node:
-          codecov-upload: true
-          test-suite-name: node
-          packages: "./node/..."
-      - test-storage:
-          codecov-upload: true
-          test-suite-name: storage
-          packages: "./storage/... ./extern/..."
-      - test-cli:
-          codecov-upload: true
-          test-suite-name: cli
-          packages: "./cli/... ./cmd/... ./api/..."
       - test-window-post:
-          codecov-upload: true
           go-test-flags: "-run=TestWindowedPost"
           winpost-test: "1"
-          test-suite-name: window-post
-      - test-window-post-dispute:
-          codecov-upload: true
-          go-test-flags: "-run=TestWindowPostDispute"
-          winpost-test: "1"
-          test-suite-name: window-post-dispute
-      - test-terminate:
-          codecov-upload: true
-          go-test-flags: "-run=TestTerminate"
-          winpost-test: "1"
-          test-suite-name: terminate
       - test-short:
           go-test-flags: "--timeout 10m --short"
-          test-suite-name: short
           filters:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+$/
-      - test-conformance:
-          test-suite-name: conformance
-          packages: "./conformance"
-      - test-conformance:
-          name: test-conformance-bleeding-edge
-          test-suite-name: conformance-bleeding-edge
-          packages: "./conformance"
-          vectors-branch: master
-      - build-ntwk-calibration
-      - build-lotus-soup
-      - trigger-testplans:
-          filters:
-            branches:
-              only:
-                - master
       - build-debug
       - build-all:
           requires:
@@ -700,20 +357,3 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+$/
-      - build-and-push-image:
-          dockerfile: Dockerfile.lotus
-          path: .
-          repo: lotus-dev
-          tag: '${CIRCLE_SHA1:0:8}'
-      - publish-packer:
-          requires:
-            - build-all
-            - build-ntwk-calibration
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+$/
-         

.codecov.yml

@@ -1,19 +1,3 @@
 comment: off
 ignore:
-  - "**/cbor_gen.go"
-  - "api/test/**/*"
-  - "api/test/*"
-  - "gen/**/*"
-  - "gen/*"
-  - "cmd/lotus-shed/*"
-  - "cmd/tvx/*"
-  - "cmd/lotus-pcr/*"
-  - "cmd/tvx/*"
-  - "cmd/lotus-chainwatch/*"
-  - "cmd/lotus-health/*"
-  - "cmd/lotus-fountain/*"
-  - "cmd/lotus-townhall/*"
-  - "cmd/lotus-stats/*"
-  - "cmd/lotus-pcr/*"
-github_checks:
-  annotations: false
+  - "cbor_gen.go"

.github/CODEOWNERS

@@ -1,16 +0,0 @@
-## filecoin-project/lotus CODEOWNERS
-## Refer to https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners.
-##
-## These users or groups will be automatically assigned as reviewers every time
-## a PR is submitted that modifies code in the specified locations.
-##
-## The Lotus repo configuration requires that at least ONE codeowner approves
-## the PR before merging.
-
-### Global owners.
-*                       @magik6k @whyrusleeping @Kubuxu
-
-### Conformance testing.
-conformance/            @raulk
-extern/test-vectors     @raulk
-cmd/tvx                 @raulk

.github/ISSUE_TEMPLATE/bug-report.md

@@ -1,33 +0,0 @@
----
-name: Bug Report
-about: Create a report to help us improve
-title: "[BUG] "
-labels: hint/needs-triaging, kind/bug
-assignees: ''
-
----
-
-> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
-
-**Describe the bug**
-A clear and concise description of what the bug is. 
-(If you are not sure what the bug is, try to figure it out via a [discussion](https://github.com/filecoin-project/lotus/discussions/new) first!
-
-**Version (run `lotus version`):**
-
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Run '...'
-2. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Logs**
-Provide daemon/miner/worker logs, and goroutines(if available) for troubleshooting.
- 
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,27 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Run '...'
+2. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Version (run `lotus --version`):**
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/deal-making-issues.md

@@ -1,49 +0,0 @@
----
-name: Deal Making Issues
-about: Create a report for help with deal making failures.
-title: "[Deal Making Issue]"
-labels: hint/needs-triaging, area/markets
-assignees: ''
-
----
-
-> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
-
-Please provide all the information requested here to help us troubleshoot "deal making failed" issues.
-If the information requested is missing, we will probably have to just ask you to provide it anyway,
-before we can help debug.
-
-**Basic Information**
-Including information like, Are you the client or the miner? Is this a storage deal or a retrieval deal? Is it an offline deal?
-
-**Describe the problem**
-
-A brief description of the problem you encountered while trying to make a deal. 
-
-**Version**
-
-The output of `lotus --version`.
-
-**Setup**
-
-You miner(if applicable) and daemon setup, i.e: What hardware do you use, how much ram and etc.
-
-**To Reproduce**
- Steps to reproduce the behavior:
- 1. Run '...'
- 2. See error
-
-**Deal status**
-
-The output of `lotus client list-deals -v` and/or `lotus-miner storage-deals|retrieval-deals|data-transfers list [-v]` commands for the deal(s) in question.
-
-**Lotus daemon and miner logs**
-
-Please go through the logs of your daemon and miner(if applicable), and include screenshots of any error/warning-like messages you find.
-
-Alternatively please upload full log files and share a link here
-
-** Code modifications **
-
-If you have modified parts of lotus, please describe which areas were modified,
-and the scope of those modifications

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,20 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-title: "[Feature Request]"
-labels: hint/needs-triaging
-assignees: ''
-
----
-
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/mining-issues.md

@@ -1,35 +0,0 @@
----
-name: Mining Issues
-about: Create a report for help with mining failures.
-title: "[Mining Issue]"
-labels: hint/needs-triaging, area/mining
-assignees: ''
-
----
-
-> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
-
-Please provide all the information requested here to help us troubleshoot "mining/WinningPoSt failed" issues.
-If the information requested is missing, you may be asked you to provide it.
-
-**Describe the problem**
-A brief description of the problem you encountered while mining new blocks.
-
-**Version**
-
-The output of `lotus --version`.
-
-**Setup**
-
-You miner and daemon setup, including what hardware do you use, your environment variable settings, how do you run your miner and worker, do you use GPU and etc.
-
-**Lotus daemon and miner logs**
-
-Please go through the logs of your daemon and miner, and include screenshots of any error/warning-like messages you find, highlighting the one has "winning post" in it.
-
-Alternatively please upload full log files and share a link here
-
-** Code modifications **
-
-If you have modified parts of lotus, please describe which areas were modified,
-and the scope of those modifications

.github/ISSUE_TEMPLATE/proving-issues.md

@@ -1,46 +0,0 @@
----
-name: Proving Issues
-about: Create a report for help with proving failures.
-title: "[Proving Issue]"
-labels: area/proving, hint/needs-triaging
-assignees: ''
-
----
-
-> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
-
-Please provide all the information requested here to help us troubleshoot "proving/window PoSt failed" issues.
-If the information requested is missing, we will probably have to just ask you to provide it anyway,
-before we can help debug.
-
-**Describe the problem**
-A brief description of the problem you encountered while proving the storage.
-
-**Version**
-
-The output of `lotus --version`.
-
-**Setup**
-
-You miner and daemon setup, including what hardware do you use, your environment variable settings, how do you run your miner and worker, do you use GPU and etc.
-
-**Proving status**
-
-The output of `lotus-miner proving` info.
-
-**Lotus miner logs**
-
-Please go through the logs of your miner, and include screenshots of any error-like messages you find, highlighting the one has "window post" in it.
-
-Alternatively please upload full log files and share a link here
-
-**Lotus miner diagnostic info**
-
-Please collect the following diagnostic information, and share a link here
-
-* lotus-miner diagnostic info `lotus-miner info all > allinfo.txt`
-
-** Code modifications **
-
-If you have modified parts of lotus, please describe which areas were modified,
-and the scope of those modifications

.github/ISSUE_TEMPLATE/sealing-issues.md

@@ -1,50 +0,0 @@
----
-name: Sealing Issues
-about: Create a report for help with sealing (commit) failures.
-title: "[Sealing Issue]"
-labels: hint/needs-triaging, area/sealing
-assignees: ''
-
----
-
-> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
-
-Please provide all the information requested here to help us troubleshoot "commit failed" issues.
-If the information requested is missing, we will probably have to just ask you to provide it anyway,
-before we can help debug.
-
-**Describe the problem**
-A brief description of the problem you encountered while sealing a sector.
-
-**Version**
-
-The output of `lotus --version`.
-
-**Setup**
-
-You miner and daemon setup, including what hardware do you use, your environment variable settings, how do you run your miner and worker, do you use GPU and etc.
-
-**Commands**
-
-Commands you ran.
-
-**Sectors status**
-
-The output of `lotus-miner sectors status --log <sectorId>` for the failed sector(s).
-
-**Lotus miner logs**
-
-Please go through the logs of your miner, and include screenshots of any error-like messages you find.
-
-Alternatively please upload full log files and share a link here
-
-**Lotus miner diagnostic info**
-
-Please collect the following diagnostic information, and share a link here
-
-* lotus-miner diagnostic info `lotus-miner info all > allinfo`
-
-** Code modifications **
-
-If you have modified parts of lotus, please describe which areas were modified,
-and the scope of those modifications

.github/ISSUE_TEMPLATE/sealingfailed.md

@@ -0,0 +1,34 @@
+---
+name: Sealing Issues
+about: Create a report for help with sealing (commit) failures.
+title: ''
+labels: 'sealing'
+assignees: ''
+
+---
+
+Please provide all the information requested here to help us troubleshoot "commit failed" issues.
+If the information requested is missing, we will probably have to just ask you to provide it anyway,
+before we can help debug.
+
+**Describe the problem**
+
+A brief description of the problem you encountered while proving (sealing) a sector.
+
+Including what commands you ran, and a description of your setup, is very helpful.
+
+**Sectors list**
+
+The output of `./lotus-storage-miner sectors list`.
+
+**Sectors status**
+
+The output of `./lotus-storage-miner sectors status --log <sectorId>` for the failed sector(s).
+
+**Lotus storage miner logs**
+
+Please go through the logs of your storage miner, and include screenshots of any error-like messages you find.
+
+**Version**
+
+The output of `./lotus --version`.

.github/labels.yml

@@ -1,248 +0,0 @@
-###
-### Special magic GitHub labels
-### https://help.github.com/en/github/building-a-strong-community/encouraging-helpful-contributions-to-your-project-with-labels
-#
-- name: "good first issue"
-  color: 7057ff
-  description: "Good for newcomers"
-- name: "help wanted"
-  color: 008672
-  description: "Extra attention is needed"
-
-###
-### Goals
-#
-- name: goal/incentives
-  color: ff004d
-  description: "Incentinet"
-
-###
-### Areas
-#
-- name: area/ux
-  color: 00A4E0
-  description: "Area: UX"
-- name: area/chain/vm
-  color: 00A4E2
-  description: "Area: Chain/VM"
-- name: area/chain/sync
-  color: 00A4E4
-  description: "Area: Chain/Sync"
-- name: area/chain/misc
-  color: 00A4E6
-  description: "Area: Chain/Misc"
-- name: area/markets
-  color: 00A4E8
-  description: "Area: Markets"  
-- name: area/sealing/fsm
-  color: 0bb1ed
-  description: "Area: Sealing/FSM"
-- name: area/sealing/storage
-  color: 0EB4F0
-  description: "Area: Sealing/Storage"
-- name: area/proving
-  color: 0EB4F0
-  description: "Area: Proving"
-- name: area/mining
-  color: 10B6F2
-  description: "Area: Mining"
-- name: area/client/storage
-  color: 13B9F5
-  description: "Area: Client/Storage"
-- name: area/client/retrieval
-  color: 15BBF7
-  description: "Area: Client/Retrieval"
-- name: area/wallet
-  color: 15BBF7
-  description: "Area: Wallet"
-- name: area/payment-channel
-  color: ff6767
-  description: "Area: Payment Channel"
-- name: area/multisig
-  color: fff0ff
-  description: "Area: Multisig"
-- name: area/networking
-  color: 273f8a
-  description: "Area: Networking"
-
-###
-### Kinds
-#
-- name: kind/bug
-  color: c92712
-  description: "Kind: Bug"
-- name: kind/chore
-  color: fcf0b5
-  description: "Kind: Chore"
-- name: kind/feature
-  color: FFF3B8
-  description: "Kind: Feature"
-- name: kind/improvement
-  color: FFF5BA
-  description: "Kind: Improvement"
-- name: kind/test
-  color: FFF8BD
-  description: "Kind: Test"
-- name: kind/question
-  color: FFFDC2
-  description: "Kind: Question"
-- name: kind/enhancement
-  color: FFFFC5
-  description: "Kind: Enhancement"
-- name: kind/discussion
-  color: FFFFC7
-  description: "Kind: Discussion"
-
-###
-### Difficulties
-#
-- name: dif/trivial
-  color: b2b7ff
-  description: "Can be confidently tackled by newcomers, who are widely unfamiliar with lotus"
-- name: dif/easy
-  color: 7886d7
-  description: "An existing lotus user should be able to pick this up"
-- name: dif/medium
-  color: 6574cd
-  description: "Prior development experience with lotus is likely helpful"
-- name: dif/hard
-  color: 5661b3
-  description: "Suggests that having worked on the specific component affected by this issue is important"
-- name: dif/expert
-  color: 2f365f
-  description: "Requires extensive knowledge of the history, implications, ramifications of the issue"
-
-###
-### Efforts
-#
-- name: effort/minutes
-  color: e8fffe
-  description: "Effort: Minutes"
-- name: effort/hours
-  color: a0f0ed
-  description: "Effort: Hours"
-- name: effort/day
-  color: 64d5ca
-  description: "Effort: One Day"
-- name: effort/days
-  color: 4dc0b5
-  description: "Effort: Multiple Days"
-- name: effort/week
-  color: 38a89d
-  description: "Effort: One Week"
-- name: effort/weeks
-  color: 20504f
-  description: "Effort: Multiple Weeks"
-
-###
-### Impacts
-#
-- name: impact/regression
-  color: f1f5f8
-  description: "Impact: Regression"
-- name: impact/api-breakage
-  color: ECF0F3
-  description: "Impact: API Breakage"
-- name: impact/quality
-  color: E7EBEE
-  description: "Impact: Quality"
-- name: impact/dx
-  color: E2E6E9
-  description: "Impact: Developer Experience"
-- name: impact/test-flakiness
-  color: DDE1E4
-  description: "Impact: Test Flakiness"
-- name: impact/consensus
-  color: b20014
-  description: "Impact: Consensus"
-
-###
-### Topics
-#
-- name: topic/interoperability
-  color: bf0f73
-  description: "Topic: Interoperability"
-- name: topic/specs
-  color: CC1C80
-  description: "Topic: Specs"
-- name: topic/docs
-  color: D9298D
-  description: "Topic: Documentation"
-- name: topic/architecture
-  color: E53599
-  description: "Topic: Architecture"
-
-###
-### Priorities
-###
-- name: P0
-  color: dd362a
-  description: "P0: Critical Blocker"
-- name: P1
-  color: ce8048
-  description: "P1: Must be resolved"
-- name: P2
-  color: dbd81a
-  description: "P2: Should be resolved"
-- name: P3
-  color: 9fea8f
-  description: "P3: Might get resolved"
-
-###
-### Hints
-#
-#- name: hint/good-first-issue
-#  color: 7057ff
-#  description: "Hint: Good First Issue"
-#- name: hint/help-wanted
-#  color: 008672
-#  description: "Hint: Help Wanted"
-- name: hint/needs-decision
-  color: 33B9A5
-  description: "Hint: Needs Decision"
-- name: hint/needs-triage
-  color: 1AA08C
-  description: "Hint: Needs Triage"
-- name: hint/needs-analysis
-  color: 26AC98
-  description: "Hint: Needs Analysis"
-- name: hint/needs-author-input
-  color: 33B9A5
-  description: "Hint: Needs Author Input"
-- name: hint/needs-team-input
-  color: 40C6B2
-  description: "Hint: Needs Team Input"
-- name: hint/needs-community-input
-  color: 4DD3BF
-  description: "Hint: Needs Community Input"
-- name: hint/needs-review
-  color: 5AE0CC
-  description: "Hint: Needs Review"
-
-###
-### Statuses
-#
-- name: status/done
-  color: edb3a6
-  description: "Status: Done"
-- name: status/deferred
-  color: E0A699
-  description: "Status: Deferred"
-- name: status/in-progress
-  color: D49A8D
-  description: "Status: In Progress"
-- name: status/blocked
-  color: C78D80
-  description: "Status: Blocked"
-- name: status/inactive
-  color: BA8073
-  description: "Status: Inactive"
-- name: status/waiting
-  color: AD7366
-  description: "Status: Waiting"
-- name: status/rotten
-  color: 7A4033
-  description: "Status: Rotten"
-- name: status/discarded
-  color: 6D3326
-  description: "Status: Discarded / Won't fix"

.github/workflows/label-syncer.yml

@@ -1,17 +0,0 @@
-
-name: Label syncer
-on:
-  push:
-    paths:
-      - '.github/labels.yml'
-    branches:
-      - master
-jobs:
-  build:
-    name: Sync labels
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@1.0.0
-      - uses: micnncim/action-label-syncer@v1.0.0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1,25 +1,19 @@
 /lotus
-/lotus-miner
-/lotus-worker
+/lotus-storage-miner
+/lotus-seal-worker
 /lotus-seed
 /lotus-health
-/lotus-chainwatch
 /lotus-shed
-/lotus-pond
-/lotus-townhall
-/lotus-fountain
-/lotus-stats
-/lotus-bench
-/lotus-gateway
-/lotus-pcr
-/lotus-wallet
-/lotus-keygen
+/pond
+/townhall
+/fountain
+/stats
+/bench
 /bench.json
 /lotuspond/front/node_modules
 /lotuspond/front/build
 /cmd/lotus-townhall/townhall/node_modules
 /cmd/lotus-townhall/townhall/build
-/cmd/lotus-townhall/townhall/package-lock.json
 extern/filecoin-ffi/rust/target
 **/*.a
 **/*.pc
@@ -30,6 +24,7 @@ build/paramfetch.sh
 /vendor
 /blocks.dot
 /blocks.svg
+/chainwatch
 /chainwatch.db
 /bundle
 /darwin
@@ -40,4 +35,3 @@ build/paramfetch.sh
 bin/ipget
 bin/tmp/*
 .idea
-scratchpad

.gitmodules

@@ -1,9 +1,7 @@
 [submodule "extern/filecoin-ffi"]
 	path = extern/filecoin-ffi
 	url = https://github.com/filecoin-project/filecoin-ffi.git
+	branch = master
 [submodule "extern/serialization-vectors"]
 	path = extern/serialization-vectors
-	url = https://github.com/filecoin-project/serialization-vectors.git
-[submodule "extern/test-vectors"]
-	path = extern/test-vectors
-	url = https://github.com/filecoin-project/test-vectors.git
+	url = https://github.com/filecoin-project/serialization-vectors

.golangci.yml

@@ -23,14 +23,6 @@ issues:
     - "Potential file inclusion via variable"
     - "should have( a package)? comment"
     - "Error return value of `logging.SetLogLevel` is not checked"
-    - "comment on exported"
-    - "(func|method) \\w+ should be \\w+"
-    - "(type|var|struct field|(method|func) parameter) `\\w+` should be `\\w+`"
-    - "(G306|G301|G307|G108|G302|G204|G104)"
-    - "don't use ALL_CAPS in Go names"
-    - "string .* has .* occurrences, make it a constant"
-    - "a blank import should be only in a main or test package, or have a comment justifying it"
-    - "package comment should be of the form"
 
   exclude-use-default: false
   exclude-rules:
@@ -54,19 +46,6 @@ issues:
       linters:
         - gosec
 
-    - path: chain/vectors/gen/.*
-      linters:
-        - gosec
-
-    - path: cmd/lotus-bench/.*
-      linters:
-        - gosec
-
-    - path: api/test/.*
-      text: "context.Context should be the first parameter"
-      linters:
-        - golint
-
 linters-settings:
   goconst:
     min-occurrences: 6

Dockerfile.lotus

@@ -1,74 +0,0 @@
-FROM golang:1.15.6 AS builder-deps
-MAINTAINER Lotus Development Team
-
-RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
-
-ARG RUST_VERSION=nightly
-ENV XDG_CACHE_HOME="/tmp"
-
-ENV RUSTUP_HOME=/usr/local/rustup \
-    CARGO_HOME=/usr/local/cargo \
-    PATH=/usr/local/cargo/bin:$PATH
-
-RUN wget "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init"; \
-    chmod +x rustup-init; \
-    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION; \
-    rm rustup-init; \
-    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
-    rustup --version; \
-    cargo --version; \
-    rustc --version;
-
-
-FROM builder-deps AS builder-local
-MAINTAINER Lotus Development Team
-
-COPY ./ /opt/filecoin
-WORKDIR /opt/filecoin
-RUN make clean deps
-
-
-FROM builder-local AS builder
-MAINTAINER Lotus Development Team
-
-WORKDIR /opt/filecoin
-
-ARG RUSTFLAGS=""
-ARG GOFLAGS=""
-
-RUN make deps lotus lotus-miner lotus-worker lotus-shed lotus-chainwatch lotus-stats
-
-
-FROM ubuntu:20.04 AS base
-MAINTAINER Lotus Development Team
-
-# Base resources
-COPY --from=builder /etc/ssl/certs                           /etc/ssl/certs
-COPY --from=builder /lib/x86_64-linux-gnu/libdl.so.2         /lib/
-COPY --from=builder /lib/x86_64-linux-gnu/librt.so.1         /lib/
-COPY --from=builder /lib/x86_64-linux-gnu/libgcc_s.so.1      /lib/
-COPY --from=builder /lib/x86_64-linux-gnu/libutil.so.1       /lib/
-COPY --from=builder /usr/lib/x86_64-linux-gnu/libltdl.so.7   /lib/
-COPY --from=builder /usr/lib/x86_64-linux-gnu/libnuma.so.1   /lib/
-COPY --from=builder /usr/lib/x86_64-linux-gnu/libhwloc.so.5  /lib/
-COPY --from=builder /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /lib/
-
-RUN useradd -r -u 532 -U fc
-
-
-FROM base AS lotus
-MAINTAINER Lotus Development Team
-
-COPY --from=builder /opt/filecoin/lotus      /usr/local/bin/
-COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/
-
-ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV LOTUS_PATH /var/lib/lotus
-
-RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters && chown fc /var/lib/lotus /var/tmp/filecoin-proof-parameters
-
-USER fc
-
-ENTRYPOINT ["/usr/local/bin/lotus"]
-
-CMD ["-help"]

Makefile

@@ -5,10 +5,10 @@ all: build
 
 unexport GOFLAGS
 
-GOVERSION:=$(shell go version | cut -d' ' -f 3 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
-ifeq ($(shell expr $(GOVERSION) \< 1015005), 1)
-$(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell expr $(GOVERSION) % 1000000 / 1000).$(shell expr $(GOVERSION) % 1000))
-$(error Update Golang to version to at least 1.15.5)
+GOVERSION:=$(shell go version | cut -d' ' -f 3 | cut -d. -f 2)
+ifeq ($(shell expr $(GOVERSION) \< 13), 1)
+$(warning Your Golang version is go 1.$(GOVERSION))
+$(error Update Golang to version $(shell grep '^go' go.mod))
 endif
 
 # git modules that need to be loaded
@@ -41,14 +41,8 @@ MODULES+=$(FFI_PATH)
 BUILD_DEPS+=build/.filecoin-install
 CLEAN+=build/.filecoin-install
 
-ffi-version-check:
-	@[[ "$$(awk '/const Version/{print $$5}' extern/filecoin-ffi/version.go)" -eq 2 ]] || (echo "FFI version mismatch, update submodules"; exit 1)
-BUILD_DEPS+=ffi-version-check
-
-.PHONY: ffi-version-check
-
-
 $(MODULES): build/.update-modules ;
+
 # dummy file that marks the last time modules were updated
 build/.update-modules:
 	git submodule update --init --recursive
@@ -64,19 +58,10 @@ deps: $(BUILD_DEPS)
 .PHONY: deps
 
 debug: GOFLAGS+=-tags=debug
-debug: lotus lotus-miner lotus-worker lotus-seed
+debug: lotus lotus-storage-miner lotus-seal-worker lotus-seed
 
 2k: GOFLAGS+=-tags=2k
-2k: lotus lotus-miner lotus-worker lotus-seed
-
-calibnet: GOFLAGS+=-tags=calibnet
-calibnet: lotus lotus-miner lotus-worker lotus-seed
-
-nerpanet: GOFLAGS+=-tags=nerpanet
-nerpanet: lotus lotus-miner lotus-worker lotus-seed
-
-butterflynet: GOFLAGS+=-tags=butterflynet
-butterflynet: lotus lotus-miner lotus-worker lotus-seed
+2k: lotus lotus-storage-miner lotus-seal-worker lotus-seed
 
 lotus: $(BUILD_DEPS)
 	rm -f lotus
@@ -86,19 +71,19 @@ lotus: $(BUILD_DEPS)
 .PHONY: lotus
 BINS+=lotus
 
-lotus-miner: $(BUILD_DEPS)
-	rm -f lotus-miner
-	go build $(GOFLAGS) -o lotus-miner ./cmd/lotus-storage-miner
-	go run github.com/GeertJohan/go.rice/rice append --exec lotus-miner -i ./build
-.PHONY: lotus-miner
-BINS+=lotus-miner
+lotus-storage-miner: $(BUILD_DEPS)
+	rm -f lotus-storage-miner
+	go build $(GOFLAGS) -o lotus-storage-miner ./cmd/lotus-storage-miner
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus-storage-miner -i ./build
+.PHONY: lotus-storage-miner
+BINS+=lotus-storage-miner
 
-lotus-worker: $(BUILD_DEPS)
-	rm -f lotus-worker
-	go build $(GOFLAGS) -o lotus-worker ./cmd/lotus-seal-worker
-	go run github.com/GeertJohan/go.rice/rice append --exec lotus-worker -i ./build
-.PHONY: lotus-worker
-BINS+=lotus-worker
+lotus-seal-worker: $(BUILD_DEPS)
+	rm -f lotus-seal-worker
+	go build $(GOFLAGS) -o lotus-seal-worker ./cmd/lotus-seal-worker
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus-seal-worker -i ./build
+.PHONY: lotus-seal-worker
+BINS+=lotus-seal-worker
 
 lotus-shed: $(BUILD_DEPS)
 	rm -f lotus-shed
@@ -107,28 +92,29 @@ lotus-shed: $(BUILD_DEPS)
 .PHONY: lotus-shed
 BINS+=lotus-shed
 
-lotus-gateway: $(BUILD_DEPS)
-	rm -f lotus-gateway
-	go build $(GOFLAGS) -o lotus-gateway ./cmd/lotus-gateway
-.PHONY: lotus-gateway
-BINS+=lotus-gateway
-
-build: lotus lotus-miner lotus-worker
+build: lotus lotus-storage-miner lotus-seal-worker
 	@[[ $$(type -P "lotus") ]] && echo "Caution: you have \
 an existing lotus binary in your PATH. This may cause problems if you don't run 'sudo make install'" || true
 
 .PHONY: build
 
-install: install-daemon install-miner install-worker
-
-install-daemon:
+install:
 	install -C ./lotus /usr/local/bin/lotus
+	install -C ./lotus-storage-miner /usr/local/bin/lotus-storage-miner
+	install -C ./lotus-seal-worker /usr/local/bin/lotus-seal-worker
 
-install-miner:
-	install -C ./lotus-miner /usr/local/bin/lotus-miner
+install-services: install
+	mkdir -p /usr/local/lib/systemd/system
+	install -C -m 0644 ./scripts/lotus-daemon.service /usr/local/lib/systemd/system/lotus-daemon.service
+	install -C -m 0644 ./scripts/lotus-miner.service /usr/local/lib/systemd/system/lotus-miner.service
+	systemctl daemon-reload
+	@echo
+	@echo "lotus and lotus-miner services installed. Don't forget to 'systemctl enable lotus|lotus-miner' for it to be enabled on startup."
 
-install-worker:
-	install -C ./lotus-worker /usr/local/bin/lotus-worker
+clean-services:
+	rm -f /usr/local/lib/systemd/system/lotus-daemon.service
+	rm -f /usr/local/lib/systemd/system/lotus-miner.service
+	systemctl daemon-reload
 
 # TOOLS
 
@@ -146,148 +132,55 @@ benchmarks:
 	@curl -X POST 'http://benchmark.kittyhawk.wtf/benchmark' -d '@bench.json' -u "${benchmark_http_cred}"
 .PHONY: benchmarks
 
-lotus-pond: 2k
-	go build -o lotus-pond ./lotuspond
-.PHONY: lotus-pond
-BINS+=lotus-pond
-
-lotus-pond-front:
+pond: 2k
+	go build -o pond ./lotuspond
 	(cd lotuspond/front && npm i && CI=false npm run build)
-.PHONY: lotus-pond-front
+.PHONY: pond
+BINS+=pond
 
-lotus-pond-app: lotus-pond-front lotus-pond
-.PHONY: lotus-pond-app
-
-lotus-townhall:
-	rm -f lotus-townhall
-	go build -o lotus-townhall ./cmd/lotus-townhall
-.PHONY: lotus-townhall
-BINS+=lotus-townhall
-
-lotus-townhall-front:
+townhall:
+	rm -f townhall
+	go build -o townhall ./cmd/lotus-townhall
 	(cd ./cmd/lotus-townhall/townhall && npm i && npm run build)
-.PHONY: lotus-townhall-front
+	go run github.com/GeertJohan/go.rice/rice append --exec townhall -i ./cmd/lotus-townhall -i ./build
+.PHONY: townhall
+BINS+=townhall
 
-lotus-townhall-app: lotus-touch lotus-townhall-front
-	go run github.com/GeertJohan/go.rice/rice append --exec lotus-townhall -i ./cmd/lotus-townhall -i ./build
-.PHONY: lotus-townhall-app
+fountain:
+	rm -f fountain
+	go build -o fountain ./cmd/lotus-fountain
+	go run github.com/GeertJohan/go.rice/rice append --exec fountain -i ./cmd/lotus-fountain -i ./build
+.PHONY: fountain
+BINS+=fountain
 
-lotus-fountain:
-	rm -f lotus-fountain
-	go build -o lotus-fountain ./cmd/lotus-fountain
-	go run github.com/GeertJohan/go.rice/rice append --exec lotus-fountain -i ./cmd/lotus-fountain -i ./build
-.PHONY: lotus-fountain
-BINS+=lotus-fountain
+chainwatch:
+	rm -f chainwatch
+	go build -o chainwatch ./cmd/lotus-chainwatch
+	go run github.com/GeertJohan/go.rice/rice append --exec chainwatch -i ./cmd/lotus-chainwatch -i ./build
+.PHONY: chainwatch
+BINS+=chainwatch
 
-lotus-chainwatch:
-	rm -f lotus-chainwatch
-	go build $(GOFLAGS) -o lotus-chainwatch ./cmd/lotus-chainwatch
-.PHONY: lotus-chainwatch
-BINS+=lotus-chainwatch
+bench:
+	rm -f bench
+	go build -o bench ./cmd/lotus-bench
+	go run github.com/GeertJohan/go.rice/rice append --exec bench -i ./build
+.PHONY: bench
+BINS+=bench
 
-lotus-bench:
-	rm -f lotus-bench
-	go build -o lotus-bench ./cmd/lotus-bench
-	go run github.com/GeertJohan/go.rice/rice append --exec lotus-bench -i ./build
-.PHONY: lotus-bench
-BINS+=lotus-bench
+stats:
+	rm -f stats
+	go build -o stats ./tools/stats
+	go run github.com/GeertJohan/go.rice/rice append --exec stats -i ./build
+.PHONY: stats
+BINS+=stats
 
-lotus-stats:
-	rm -f lotus-stats
-	go build $(GOFLAGS) -o lotus-stats ./cmd/lotus-stats
-	go run github.com/GeertJohan/go.rice/rice append --exec lotus-stats -i ./build
-.PHONY: lotus-stats
-BINS+=lotus-stats
-
-lotus-pcr:
-	rm -f lotus-pcr
-	go build $(GOFLAGS) -o lotus-pcr ./cmd/lotus-pcr
-	go run github.com/GeertJohan/go.rice/rice append --exec lotus-pcr -i ./build
-.PHONY: lotus-pcr
-BINS+=lotus-pcr
-
-lotus-health:
+health:
 	rm -f lotus-health
 	go build -o lotus-health ./cmd/lotus-health
 	go run github.com/GeertJohan/go.rice/rice append --exec lotus-health -i ./build
-.PHONY: lotus-health
-BINS+=lotus-health
 
-lotus-wallet:
-	rm -f lotus-wallet
-	go build -o lotus-wallet ./cmd/lotus-wallet
-.PHONY: lotus-wallet
-BINS+=lotus-wallet
-
-lotus-keygen:
-	rm -f lotus-keygen
-	go build -o lotus-keygen ./cmd/lotus-keygen
-.PHONY: lotus-keygen
-BINS+=lotus-keygen
-
-testground:
-	go build -tags testground -o /dev/null ./cmd/lotus
-.PHONY: testground
-BINS+=testground
-
-install-chainwatch: lotus-chainwatch
-	install -C ./lotus-chainwatch /usr/local/bin/lotus-chainwatch
-
-# SYSTEMD
-
-install-daemon-service: install-daemon
-	mkdir -p /etc/systemd/system
-	mkdir -p /var/log/lotus
-	install -C -m 0644 ./scripts/lotus-daemon.service /etc/systemd/system/lotus-daemon.service
-	systemctl daemon-reload
-	@echo
-	@echo "lotus-daemon service installed. Don't forget to run 'sudo systemctl start lotus-daemon' to start it and 'sudo systemctl enable lotus-daemon' for it to be enabled on startup."
-
-install-miner-service: install-miner install-daemon-service
-	mkdir -p /etc/systemd/system
-	mkdir -p /var/log/lotus
-	install -C -m 0644 ./scripts/lotus-miner.service /etc/systemd/system/lotus-miner.service
-	systemctl daemon-reload
-	@echo
-	@echo "lotus-miner service installed. Don't forget to run 'sudo systemctl start lotus-miner' to start it and 'sudo systemctl enable lotus-miner' for it to be enabled on startup."
-
-install-chainwatch-service: install-chainwatch install-daemon-service
-	mkdir -p /etc/systemd/system
-	mkdir -p /var/log/lotus
-	install -C -m 0644 ./scripts/lotus-chainwatch.service /etc/systemd/system/lotus-chainwatch.service
-	systemctl daemon-reload
-	@echo
-	@echo "chainwatch service installed. Don't forget to run 'sudo systemctl start lotus-chainwatch' to start it and 'sudo systemctl enable lotus-chainwatch' for it to be enabled on startup."
-
-install-main-services: install-miner-service
-
-install-all-services: install-main-services install-chainwatch-service
-
-install-services: install-main-services
-
-clean-daemon-service: clean-miner-service clean-chainwatch-service
-	-systemctl stop lotus-daemon
-	-systemctl disable lotus-daemon
-	rm -f /etc/systemd/system/lotus-daemon.service
-	systemctl daemon-reload
-
-clean-miner-service:
-	-systemctl stop lotus-miner
-	-systemctl disable lotus-miner
-	rm -f /etc/systemd/system/lotus-miner.service
-	systemctl daemon-reload
-
-clean-chainwatch-service:
-	-systemctl stop lotus-chainwatch
-	-systemctl disable lotus-chainwatch
-	rm -f /etc/systemd/system/lotus-chainwatch.service
-	systemctl daemon-reload
-
-clean-main-services: clean-daemon-service
-
-clean-all-services: clean-main-services
-
-clean-services: clean-all-services
+.PHONY: health
+BINS+=health
 
 # MISC
 
@@ -295,15 +188,15 @@ buildall: $(BINS)
 
 completions:
 	./scripts/make-completions.sh lotus
-	./scripts/make-completions.sh lotus-miner
+	./scripts/make-completions.sh lotus-storage-miner
 .PHONY: completions
 
 install-completions:
 	mkdir -p /usr/share/bash-completion/completions /usr/local/share/zsh/site-functions/
 	install -C ./scripts/bash-completion/lotus /usr/share/bash-completion/completions/lotus
-	install -C ./scripts/bash-completion/lotus-miner /usr/share/bash-completion/completions/lotus-miner
+	install -C ./scripts/bash-completion/lotus-storage-miner /usr/share/bash-completion/completions/lotus-storage-miner
 	install -C ./scripts/zsh-completion/lotus /usr/local/share/zsh/site-functions/_lotus
-	install -C ./scripts/zsh-completion/lotus-miner /usr/local/share/zsh/site-functions/_lotus-miner
+	install -C ./scripts/zsh-completion/lotus-storage-miner /usr/local/share/zsh/site-functions/_lotus-storage-miner
 
 clean:
 	rm -rf $(CLEAN) $(BINS)
@@ -317,17 +210,11 @@ dist-clean:
 
 type-gen:
 	go run ./gen/main.go
-	go generate ./...
 
 method-gen:
 	(cd ./lotuspond/front/src/chain && go run ./methodgen.go)
 
 gen: type-gen method-gen
 
-docsgen:
-	go run ./api/docgen "api/api_full.go" "FullNode" > documentation/en/api-methods.md
-	go run ./api/docgen "api/api_storage.go" "StorageMiner" > documentation/en/api-methods-miner.md
-	go run ./api/docgen "api/api_worker.go" "WorkerAPI" > documentation/en/api-methods-worker.md
-
 print-%:
 	@echo $*=$($*)

README.md

@@ -1,54 +1,16 @@
-<p align="center">
-  <a href="https://docs.filecoin.io/" title="Filecoin Docs">
-    <img src="documentation/images/lotus_logo_h.png" alt="Project Lotus Logo" width="244" />
-  </a>
-</p>
+![Lotus](documentation/images/lotus_logo_h.png)
 
-<h1 align="center">Project Lotus - 莲</h1>
+# Project Lotus - 莲
 
-<p align="center">
-  <a href="https://circleci.com/gh/filecoin-project/lotus"><img src="https://circleci.com/gh/filecoin-project/lotus.svg?style=svg"></a>
-  <a href="https://codecov.io/gh/filecoin-project/lotus"><img src="https://codecov.io/gh/filecoin-project/lotus/branch/master/graph/badge.svg"></a>
-  <a href="https://goreportcard.com/report/github.com/filecoin-project/lotus"><img src="https://goreportcard.com/badge/github.com/filecoin-project/lotus" /></a>  
-  <a href=""><img src="https://img.shields.io/badge/golang-%3E%3D1.15.5-blue.svg" /></a>
-  <br>
-</p>
+Lotus is an implementation of the Filecoin Distributed Storage Network. For more details about Filecoin, check out the [Filecoin Spec](https://github.com/filecoin-project/specs).
 
-Lotus is an implementation of the Filecoin Distributed Storage Network. For more details about Filecoin, check out the [Filecoin Spec](https://spec.filecoin.io).
+## Development
+
+All work is tracked via issues. An attempt at keeping an up-to-date view on remaining work is in the [lotus testnet github project board](https://github.com/filecoin-project/lotus/projects/1).
 
 ## Building & Documentation
 
-For instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/).
-
-## Reporting a Vulnerability
-
-Please send an email to security@filecoin.org. See our [security policy](SECURITY.md) for more details.
-
-## Related packages
-
-These repos are independent and reusable modules, but are tightly integrated into Lotus to make up a fully featured Filecoin implementation:
-
-- [go-fil-markets](https://github.com/filecoin-project/go-fil-markets) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/markets-shared-components-5daa144a7046a60001c6e253/board)
-- [specs-actors](https://github.com/filecoin-project/specs-actors) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/actors-5ee6f3aa87591f0016c05685/board)
-
-## Contribute
-
-Lotus is a universally open project and welcomes contributions of all kinds: code, docs, and more. However, before making a contribution, we ask you to heed these recommendations:
-
-1. If the proposal entails a protocol change, please first submit a [Filecoin Improvement Proposal](https://github.com/filecoin-project/FIPs).
-2. If the change is complex and requires prior discussion, [open an issue](github.com/filecoin-project/lotus/issues) or a [discussion](https://github.com/filecoin-project/lotus/discussions) to request feedback before you start working on a pull request. This is to avoid disappointment and sunk costs, in case the change is not actually needed or accepted.
-3. Please refrain from submitting PRs to adapt existing code to subjective preferences. The changeset should contain functional or technical improvements/enhancements, bug fixes, new features, or some other clear material contribution. Simple stylistic changes are likely to be rejected in order to reduce code churn.
-
-When implementing a change:
-
-1. Adhere to the standard Go formatting guidelines, e.g. [Effective Go](https://golang.org/doc/effective_go.html). Run `go fmt`.
-2. Stick to the idioms and patterns used in the codebase. Familiar-looking code has a higher chance of being accepted than eerie code. Pay attention to commonly used variable and parameter names, avoidance of naked returns, error handling patterns, etc.
-3. Comments: follow the advice on the [Commentary](https://golang.org/doc/effective_go.html#commentary) section of Effective Go.
-4. Minimize code churn. Modify only what is strictly necessary. Well-encapsulated changesets will get a quicker response from maintainers.
-5. Lint your code with [`golangci-lint`](https://golangci-lint.run) (CI will reject your PR if unlinted).
-6. Add tests.
-7. Title the PR in a meaningful way and describe the rationale and the thought process in the PR description.
-8. Write clean, thoughtful, and detailed [commit messages](https://chris.beams.io/posts/git-commit/). This is even more important than the PR description, because commit messages are stored _inside_ the Git history. One good rule is: if you are happy posting the commit message as the PR description, then it's a good commit message.
+For instructions on how to build lotus from source, please visit [https://docs.lotu.sh](https://docs.lotu.sh) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation).
 
 ## License
 

SECURITY.md

@@ -1,23 +0,0 @@
-# Security Policy
-
-## Reporting a Vulnerability
-
-For reporting security vulnerabilities/bugs, please consult our Security Policy and Responsible Disclosure Program information at https://github.com/filecoin-project/community/blob/master/SECURITY.md. Security vulnerabilities should be reported via our [Vulnerability Reporting channels](https://github.com/filecoin-project/community/blob/master/SECURITY.md#vulnerability-reporting) and will be eligible for a [Bug Bounty](https://security.filecoin.io/bug-bounty/).
-
-Please try to provide a clear description of any bugs reported, along with how to reproduce the bug if possible. More detailed bug reports (especially those with a PoC included) will help us move forward much faster. Additionally, please avoid reporting bugs that already have open issues. Take a moment to search the issue list of the related GitHub repositories before writing up a new report.
-
-Here are some examples of bugs we would consider to be security vulnerabilities:
-
-* If you can spend from a `multisig` wallet you do not control the keys for.
-* If you can cause a miner to be slashed without them actually misbehaving.
-* If you can maintain power without submitting windowed posts regularly.
-* If you can craft a message that causes lotus nodes to panic.
-* If you can cause your miner to win significantly more blocks than it should.
-* If you can craft a message that causes a persistent fork in the network.
-* If you can cause the total amount of Filecoin in the network to no longer be 2 billion.
-
-This is not an exhaustive list, but should provide some idea of what we consider as a security vulnerability, . 
-
-## Reporting a non security bug
-
-For non-security bugs, please simply file a GitHub [issue](https://github.com/filecoin-project/lotus/issues/new?template=bug_report.md). 

api/api_common.go

@@ -4,23 +4,20 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/google/uuid"
-
-	"github.com/filecoin-project/go-jsonrpc/auth"
-	metrics "github.com/libp2p/go-libp2p-core/metrics"
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
-	protocol "github.com/libp2p/go-libp2p-core/protocol"
+
+	"github.com/filecoin-project/go-jsonrpc/auth"
+
+	"github.com/filecoin-project/lotus/build"
 )
 
 type Common interface {
-
-	// MethodGroup: Auth
-
+	// Auth
 	AuthVerify(ctx context.Context, token string) ([]auth.Permission, error)
 	AuthNew(ctx context.Context, perms []auth.Permission) ([]byte, error)
 
-	// MethodGroup: Net
+	// network
 
 	NetConnectedness(context.Context, peer.ID) (network.Connectedness, error)
 	NetPeers(context.Context) ([]peer.AddrInfo, error)
@@ -29,56 +26,29 @@ type Common interface {
 	NetDisconnect(context.Context, peer.ID) error
 	NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error)
 	NetPubsubScores(context.Context) ([]PubsubScore, error)
-	NetAutoNatStatus(context.Context) (NatInfo, error)
-	NetAgentVersion(ctx context.Context, p peer.ID) (string, error)
-	NetPeerInfo(context.Context, peer.ID) (*ExtendedPeerInfo, error)
-
-	// NetBandwidthStats returns statistics about the nodes total bandwidth
-	// usage and current rate across all peers and protocols.
-	NetBandwidthStats(ctx context.Context) (metrics.Stats, error)
-
-	// NetBandwidthStatsByPeer returns statistics about the nodes bandwidth
-	// usage and current rate per peer
-	NetBandwidthStatsByPeer(ctx context.Context) (map[string]metrics.Stats, error)
-
-	// NetBandwidthStatsByProtocol returns statistics about the nodes bandwidth
-	// usage and current rate per protocol
-	NetBandwidthStatsByProtocol(ctx context.Context) (map[protocol.ID]metrics.Stats, error)
-
-	// ConnectionGater API
-	NetBlockAdd(ctx context.Context, acl NetBlockList) error
-	NetBlockRemove(ctx context.Context, acl NetBlockList) error
-	NetBlockList(ctx context.Context) (NetBlockList, error)
-
-	// MethodGroup: Common
 
 	// ID returns peerID of libp2p node backing this API
 	ID(context.Context) (peer.ID, error)
 
 	// Version provides information about API provider
-	Version(context.Context) (APIVersion, error)
+	Version(context.Context) (Version, error)
 
 	LogList(context.Context) ([]string, error)
 	LogSetLevel(context.Context, string, string) error
 
 	// trigger graceful shutdown
 	Shutdown(context.Context) error
-
-	// Session returns a random UUID of api provider session
-	Session(context.Context) (uuid.UUID, error)
-
-	Closing(context.Context) (<-chan struct{}, error)
 }
 
-// APIVersion provides various build-time information
-type APIVersion struct {
+// Version provides various build-time information
+type Version struct {
 	Version string
 
 	// APIVersion is a binary encoded semver version of the remote implementing
 	// this api
 	//
 	// See APIVersion in build/version.go
-	APIVersion Version
+	APIVersion build.Version
 
 	// TODO: git commit / os / genesis cid?
 
@@ -86,11 +56,6 @@ type APIVersion struct {
 	BlockDelay uint64
 }
 
-func (v APIVersion) String() string {
+func (v Version) String() string {
 	return fmt.Sprintf("%s+api%s", v.Version, v.APIVersion.String())
 }
-
-type NatInfo struct {
-	Reachability network.Reachability
-	PublicAddr   string
-}

api/api_gateway.go

@@ -1,47 +0,0 @@
-package api
-
-import (
-	"context"
-
-	"github.com/ipfs/go-cid"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/dline"
-	"github.com/filecoin-project/go-state-types/network"
-
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-type GatewayAPI interface {
-	ChainHasObj(context.Context, cid.Cid) (bool, error)
-	ChainHead(ctx context.Context) (*types.TipSet, error)
-	ChainGetBlockMessages(context.Context, cid.Cid) (*BlockMessages, error)
-	ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
-	ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error)
-	ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error)
-	ChainNotify(context.Context) (<-chan []*HeadChange, error)
-	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
-	GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
-	MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
-	MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
-	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
-	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
-	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
-	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
-	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
-	StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)
-	StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
-	StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
-	StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MarketBalance, error)
-	StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*MarketDeal, error)
-	StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error)
-	StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
-	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error)
-	StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error)
-	StateSearchMsg(ctx context.Context, msg cid.Cid) (*MsgLookup, error)
-	StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
-	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
-	StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*MsgLookup, error)
-}

api/api_storage.go

@@ -3,25 +3,15 @@ package api
 import (
 	"bytes"
 	"context"
-	"time"
 
-	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
-	"github.com/libp2p/go-libp2p-core/peer"
 
 	"github.com/filecoin-project/go-address"
-	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-fil-markets/piecestore"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
-	"github.com/filecoin-project/specs-storage/storage"
-
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	"github.com/filecoin-project/sector-storage/stores"
+	"github.com/filecoin-project/sector-storage/storiface"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 )
 
 // StorageMiner is a low-level interface to the Filecoin network storage miner node
@@ -31,119 +21,41 @@ type StorageMiner interface {
 	ActorAddress(context.Context) (address.Address, error)
 
 	ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error)
-	ActorAddressConfig(ctx context.Context) (AddressConfig, error)
 
 	MiningBase(context.Context) (*types.TipSet, error)
 
 	// Temp api for testing
-	PledgeSector(context.Context) (abi.SectorID, error)
+	PledgeSector(context.Context) error
 
 	// Get the status of a given sector by ID
-	SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (SectorInfo, error)
+	SectorsStatus(context.Context, abi.SectorNumber) (SectorInfo, error)
 
 	// List all staged sectors
 	SectorsList(context.Context) ([]abi.SectorNumber, error)
 
-	// Get summary info of sectors
-	SectorsSummary(ctx context.Context) (map[SectorState]int, error)
-
-	// List sectors in particular states
-	SectorsListInStates(context.Context, []SectorState) ([]abi.SectorNumber, error)
-
 	SectorsRefs(context.Context) (map[string][]SealedRef, error)
 
-	// SectorStartSealing can be called on sectors in Empty or WaitDeals states
-	// to trigger sealing early
-	SectorStartSealing(context.Context, abi.SectorNumber) error
-	// SectorSetSealDelay sets the time that a newly-created sector
-	// waits for more deals before it starts sealing
-	SectorSetSealDelay(context.Context, time.Duration) error
-	// SectorGetSealDelay gets the time that a newly-created sector
-	// waits for more deals before it starts sealing
-	SectorGetSealDelay(context.Context) (time.Duration, error)
-	// SectorSetExpectedSealDuration sets the expected time for a sector to seal
-	SectorSetExpectedSealDuration(context.Context, time.Duration) error
-	// SectorGetExpectedSealDuration gets the expected time for a sector to seal
-	SectorGetExpectedSealDuration(context.Context) (time.Duration, error)
 	SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error
-	// SectorRemove removes the sector from storage. It doesn't terminate it on-chain, which can
-	// be done with SectorTerminate. Removing and not terminating live sectors will cause additional penalties.
-	SectorRemove(context.Context, abi.SectorNumber) error
-	// SectorTerminate terminates the sector on-chain (adding it to a termination batch first), then
-	// automatically removes it from storage
-	SectorTerminate(context.Context, abi.SectorNumber) error
-	// SectorTerminateFlush immediately sends a terminate message with sectors batched for termination.
-	// Returns null if message wasn't sent
-	SectorTerminateFlush(ctx context.Context) (*cid.Cid, error)
-	// SectorTerminatePending returns a list of pending sector terminations to be sent in the next batch message
-	SectorTerminatePending(ctx context.Context) ([]abi.SectorID, error)
-	SectorMarkForUpgrade(ctx context.Context, id abi.SectorNumber) error
 
 	StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error)
 	StorageLocal(ctx context.Context) (map[stores.ID]string, error)
-	StorageStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error)
+	StorageStat(ctx context.Context, id stores.ID) (stores.FsStat, error)
 
 	// WorkerConnect tells the node to connect to workers RPC
 	WorkerConnect(context.Context, string) error
-	WorkerStats(context.Context) (map[uuid.UUID]storiface.WorkerStats, error)
-	WorkerJobs(context.Context) (map[uuid.UUID][]storiface.WorkerJob, error)
-	storiface.WorkerReturn
-
-	// SealingSchedDiag dumps internal sealing scheduler state
-	SealingSchedDiag(ctx context.Context, doSched bool) (interface{}, error)
-	SealingAbort(ctx context.Context, call storiface.CallID) error
+	WorkerStats(context.Context) (map[uint64]storiface.WorkerStats, error)
 
 	stores.SectorIndex
 
 	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error
-	MarketListDeals(ctx context.Context) ([]MarketDeal, error)
-	MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error)
-	MarketGetDealUpdates(ctx context.Context) (<-chan storagemarket.MinerDeal, error)
+	MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error)
 	MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error)
-	MarketSetAsk(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error
-	MarketGetAsk(ctx context.Context) (*storagemarket.SignedStorageAsk, error)
-	MarketSetRetrievalAsk(ctx context.Context, rask *retrievalmarket.Ask) error
-	MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error)
-	MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)
-	MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)
-	// MarketRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer
-	MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error
-	// MarketCancelDataTransfer cancels a data transfer with the given transfer ID and other peer
-	MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error
-	MarketPendingDeals(ctx context.Context) (PendingDealInfo, error)
-	MarketPublishPendingDeals(ctx context.Context) error
+	MarketSetPrice(context.Context, types.BigInt) error
 
 	DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error
-	DealsList(ctx context.Context) ([]MarketDeal, error)
-	DealsConsiderOnlineStorageDeals(context.Context) (bool, error)
-	DealsSetConsiderOnlineStorageDeals(context.Context, bool) error
-	DealsConsiderOnlineRetrievalDeals(context.Context) (bool, error)
-	DealsSetConsiderOnlineRetrievalDeals(context.Context, bool) error
-	DealsPieceCidBlocklist(context.Context) ([]cid.Cid, error)
-	DealsSetPieceCidBlocklist(context.Context, []cid.Cid) error
-	DealsConsiderOfflineStorageDeals(context.Context) (bool, error)
-	DealsSetConsiderOfflineStorageDeals(context.Context, bool) error
-	DealsConsiderOfflineRetrievalDeals(context.Context) (bool, error)
-	DealsSetConsiderOfflineRetrievalDeals(context.Context, bool) error
-	DealsConsiderVerifiedStorageDeals(context.Context) (bool, error)
-	DealsSetConsiderVerifiedStorageDeals(context.Context, bool) error
-	DealsConsiderUnverifiedStorageDeals(context.Context) (bool, error)
-	DealsSetConsiderUnverifiedStorageDeals(context.Context, bool) error
+	DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error)
 
 	StorageAddLocal(ctx context.Context, path string) error
-
-	PiecesListPieces(ctx context.Context) ([]cid.Cid, error)
-	PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error)
-	PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error)
-	PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error)
-
-	// CreateBackup creates node backup onder the specified file name. The
-	// method requires that the lotus-miner is running with the
-	// LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that
-	// the path specified when calling CreateBackup is within the base path
-	CreateBackup(ctx context.Context, fpath string) error
-
-	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error)
 }
 
 type SealRes struct {
@@ -163,40 +75,24 @@ type SectorLog struct {
 }
 
 type SectorInfo struct {
-	SectorID     abi.SectorNumber
-	State        SectorState
-	CommD        *cid.Cid
-	CommR        *cid.Cid
-	Proof        []byte
-	Deals        []abi.DealID
-	Ticket       SealTicket
-	Seed         SealSeed
-	PreCommitMsg *cid.Cid
-	CommitMsg    *cid.Cid
-	Retries      uint64
-	ToUpgrade    bool
+	SectorID abi.SectorNumber
+	State    SectorState
+	CommD    *cid.Cid
+	CommR    *cid.Cid
+	Proof    []byte
+	Deals    []abi.DealID
+	Ticket   SealTicket
+	Seed     SealSeed
+	Retries  uint64
 
 	LastErr string
 
 	Log []SectorLog
-
-	// On Chain Info
-	SealProof          abi.RegisteredSealProof // The seal proof type implies the PoSt proof/s
-	Activation         abi.ChainEpoch          // Epoch during which the sector proof was accepted
-	Expiration         abi.ChainEpoch          // Epoch during which the sector expires
-	DealWeight         abi.DealWeight          // Integral of active deals over sector lifetime
-	VerifiedDealWeight abi.DealWeight          // Integral of active verified deals over sector lifetime
-	InitialPledge      abi.TokenAmount         // Pledge collected to commit this sector
-	// Expiration Info
-	OnTime abi.ChainEpoch
-	// non-zero if sector is faulty, epoch at which it will be permanently
-	// removed if it doesn't recover
-	Early abi.ChainEpoch
 }
 
 type SealedRef struct {
 	SectorID abi.SectorNumber
-	Offset   abi.PaddedPieceSize
+	Offset   uint64
 	Size     abi.UnpaddedPieceSize
 }
 
@@ -223,30 +119,3 @@ func (st *SealSeed) Equals(ost *SealSeed) bool {
 }
 
 type SectorState string
-
-type AddrUse int
-
-const (
-	PreCommitAddr AddrUse = iota
-	CommitAddr
-	PoStAddr
-
-	TerminateSectorsAddr
-)
-
-type AddressConfig struct {
-	PreCommitControl []address.Address
-	CommitControl    []address.Address
-	TerminateControl []address.Address
-
-	DisableOwnerFallback  bool
-	DisableWorkerFallback bool
-}
-
-// PendingDealInfo has info about pending deals and when they are due to be
-// published
-type PendingDealInfo struct {
-	Deals              []market.ClientDealProposal
-	PublishPeriodStart time.Time
-	PublishPeriod      time.Duration
-}

api/api_test.go

@@ -1,16 +1,12 @@
 package api
 
 import (
-	"encoding/json"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"reflect"
 	"runtime"
 	"strings"
 	"testing"
-
-	"github.com/stretchr/testify/require"
 )
 
 func goCmd() string {
@@ -36,68 +32,3 @@ func TestDoesntDependOnFFI(t *testing.T) {
 		}
 	}
 }
-
-func TestReturnTypes(t *testing.T) {
-	errType := reflect.TypeOf(new(error)).Elem()
-	bareIface := reflect.TypeOf(new(interface{})).Elem()
-	jmarsh := reflect.TypeOf(new(json.Marshaler)).Elem()
-
-	tst := func(api interface{}) func(t *testing.T) {
-		return func(t *testing.T) {
-			ra := reflect.TypeOf(api).Elem()
-			for i := 0; i < ra.NumMethod(); i++ {
-				m := ra.Method(i)
-				switch m.Type.NumOut() {
-				case 1: // if 1 return value, it must be an error
-					require.Equal(t, errType, m.Type.Out(0), m.Name)
-
-				case 2: // if 2 return values, first cant be an interface/function, second must be an error
-					seen := map[reflect.Type]struct{}{}
-					todo := []reflect.Type{m.Type.Out(0)}
-					for len(todo) > 0 {
-						typ := todo[len(todo)-1]
-						todo = todo[:len(todo)-1]
-
-						if _, ok := seen[typ]; ok {
-							continue
-						}
-						seen[typ] = struct{}{}
-
-						if typ.Kind() == reflect.Interface && typ != bareIface && !typ.Implements(jmarsh) {
-							t.Error("methods can't return interfaces", m.Name)
-						}
-
-						switch typ.Kind() {
-						case reflect.Ptr:
-							fallthrough
-						case reflect.Array:
-							fallthrough
-						case reflect.Slice:
-							fallthrough
-						case reflect.Chan:
-							todo = append(todo, typ.Elem())
-						case reflect.Map:
-							todo = append(todo, typ.Elem())
-							todo = append(todo, typ.Key())
-						case reflect.Struct:
-							for i := 0; i < typ.NumField(); i++ {
-								todo = append(todo, typ.Field(i).Type)
-							}
-						}
-					}
-
-					require.NotEqual(t, reflect.Func.String(), m.Type.Out(0).Kind().String(), m.Name)
-					require.Equal(t, errType, m.Type.Out(1), m.Name)
-
-				default:
-					t.Error("methods can only have 1 or 2 return values", m.Name)
-				}
-			}
-		}
-	}
-
-	t.Run("common", tst(new(Common)))
-	t.Run("full", tst(new(FullNode)))
-	t.Run("miner", tst(new(StorageMiner)))
-	t.Run("worker", tst(new(WorkerAPI)))
-}

api/api_wallet.go

@@ -1,47 +0,0 @@
-package api
-
-import (
-	"context"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/crypto"
-
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-type MsgType string
-
-const (
-	MTUnknown = "unknown"
-
-	// Signing message CID. MsgMeta.Extra contains raw cbor message bytes
-	MTChainMsg = "message"
-
-	// Signing a blockheader. signing raw cbor block bytes (MsgMeta.Extra is empty)
-	MTBlock = "block"
-
-	// Signing a deal proposal. signing raw cbor proposal bytes (MsgMeta.Extra is empty)
-	MTDealProposal = "dealproposal"
-
-	// TODO: Deals, Vouchers, VRF
-)
-
-type MsgMeta struct {
-	Type MsgType
-
-	// Additional data related to what is signed. Should be verifiable with the
-	// signed bytes (e.g. CID(Extra).Bytes() == toSign)
-	Extra []byte
-}
-
-type WalletAPI interface {
-	WalletNew(context.Context, types.KeyType) (address.Address, error)
-	WalletHas(context.Context, address.Address) (bool, error)
-	WalletList(context.Context) ([]address.Address, error)
-
-	WalletSign(ctx context.Context, signer address.Address, toSign []byte, meta MsgMeta) (*crypto.Signature, error)
-
-	WalletExport(context.Context, address.Address) (*types.KeyInfo, error)
-	WalletImport(context.Context, *types.KeyInfo) (address.Address, error)
-	WalletDelete(context.Context, address.Address) error
-}

api/api_worker.go

@@ -2,46 +2,35 @@ package api
 
 import (
 	"context"
+	"io"
 
-	"github.com/google/uuid"
+	"github.com/ipfs/go-cid"
 
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	"github.com/filecoin-project/sector-storage/sealtasks"
+	"github.com/filecoin-project/sector-storage/stores"
+	"github.com/filecoin-project/sector-storage/storiface"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-storage/storage"
+
+	"github.com/filecoin-project/lotus/build"
 )
 
 type WorkerAPI interface {
-	Version(context.Context) (Version, error)
+	Version(context.Context) (build.Version, error)
 	// TODO: Info() (name, ...) ?
 
 	TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) // TaskType -> Weight
 	Paths(context.Context) ([]stores.StoragePath, error)
 	Info(context.Context) (storiface.WorkerInfo, error)
 
-	storiface.WorkerCalls
+	storage.Sealer
 
-	TaskDisable(ctx context.Context, tt sealtasks.TaskType) error
-	TaskEnable(ctx context.Context, tt sealtasks.TaskType) error
+	MoveStorage(ctx context.Context, sector abi.SectorID) error
 
-	// Storage / Other
-	Remove(ctx context.Context, sector abi.SectorID) error
+	UnsealPiece(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error
+	ReadPiece(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) error
 
-	StorageAddLocal(ctx context.Context, path string) error
+	Fetch(context.Context, abi.SectorID, stores.SectorFileType, stores.PathType, stores.AcquireMode) error
 
-	// SetEnabled marks the worker as enabled/disabled. Not that this setting
-	// may take a few seconds to propagate to task scheduler
-	SetEnabled(ctx context.Context, enabled bool) error
-
-	Enabled(ctx context.Context) (bool, error)
-
-	// WaitQuiet blocks until there are no tasks running
-	WaitQuiet(ctx context.Context) error
-
-	// returns a random UUID of worker session, generated randomly when worker
-	// process starts
-	ProcessSession(context.Context) (uuid.UUID, error)
-
-	// Like ProcessSession, but returns an error when worker is disabled
-	Session(context.Context) (uuid.UUID, error)
+	Closing(context.Context) (<-chan struct{}, error)
 }

api/apibstore/apibstore.go

@@ -0,0 +1,67 @@
+package apibstore
+
+import (
+	"context"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	"golang.org/x/xerrors"
+)
+
+type ChainIO interface {
+	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
+	ChainHasObj(context.Context, cid.Cid) (bool, error)
+}
+
+type apiBStore struct {
+	api ChainIO
+}
+
+func NewAPIBlockstore(cio ChainIO) blockstore.Blockstore {
+	return &apiBStore{
+		api: cio,
+	}
+}
+
+func (a *apiBStore) DeleteBlock(cid.Cid) error {
+	return xerrors.New("not supported")
+}
+
+func (a *apiBStore) Has(c cid.Cid) (bool, error) {
+	return a.api.ChainHasObj(context.TODO(), c)
+}
+
+func (a *apiBStore) Get(c cid.Cid) (blocks.Block, error) {
+	bb, err := a.api.ChainReadObj(context.TODO(), c)
+	if err != nil {
+		return nil, err
+	}
+	return blocks.NewBlockWithCid(bb, c)
+}
+
+func (a *apiBStore) GetSize(c cid.Cid) (int, error) {
+	bb, err := a.api.ChainReadObj(context.TODO(), c)
+	if err != nil {
+		return 0, err
+	}
+	return len(bb), nil
+}
+
+func (a *apiBStore) Put(blocks.Block) error {
+	return xerrors.New("not supported")
+}
+
+func (a *apiBStore) PutMany([]blocks.Block) error {
+	return xerrors.New("not supported")
+}
+
+func (a *apiBStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+	return nil, xerrors.New("not supported")
+}
+
+func (a *apiBStore) HashOnRead(enabled bool) {
+	return
+}
+
+var _ blockstore.Blockstore = &apiBStore{}

api/apistruct/permissioned.go

@@ -36,9 +36,3 @@ func PermissionedWorkerAPI(a api.WorkerAPI) api.WorkerAPI {
 	auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
 	return &out
 }
-
-func PermissionedWalletAPI(a api.WalletAPI) api.WalletAPI {
-	var out WalletStruct
-	auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
-	return &out
-}

api/cbor_gen.go

@@ -5,18 +5,14 @@ package api
 import (
 	"fmt"
 	"io"
-	"sort"
 
-	abi "github.com/filecoin-project/go-state-types/abi"
-	paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
-	cid "github.com/ipfs/go-cid"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/builtin/paych"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	xerrors "golang.org/x/xerrors"
 )
 
 var _ = xerrors.Errorf
-var _ = cid.Undef
-var _ = sort.Sort
 
 func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	if t == nil {
@@ -37,7 +33,7 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Channel"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Channel")); err != nil {
+	if _, err := io.WriteString(w, "Channel"); err != nil {
 		return err
 	}
 
@@ -45,20 +41,26 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	// t.WaitSentinel (cid.Cid) (struct)
-	if len("WaitSentinel") > cbg.MaxLength {
-		return xerrors.Errorf("Value in field \"WaitSentinel\" was too long")
+	// t.ChannelMessage (cid.Cid) (struct)
+	if len("ChannelMessage") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"ChannelMessage\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("WaitSentinel"))); err != nil {
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ChannelMessage"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("WaitSentinel")); err != nil {
+	if _, err := io.WriteString(w, "ChannelMessage"); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteCidBuf(scratch, w, t.WaitSentinel); err != nil {
-		return xerrors.Errorf("failed to write cid field t.WaitSentinel: %w", err)
+	if t.ChannelMessage == nil {
+		if _, err := w.Write(cbg.CborNull); err != nil {
+			return err
+		}
+	} else {
+		if err := cbg.WriteCidBuf(scratch, w, *t.ChannelMessage); err != nil {
+			return xerrors.Errorf("failed to write cid field t.ChannelMessage: %w", err)
+		}
 	}
 
 	// t.Vouchers ([]*paych.SignedVoucher) (slice)
@@ -69,7 +71,7 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Vouchers"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Vouchers")); err != nil {
+	if _, err := io.WriteString(w, "Vouchers"); err != nil {
 		return err
 	}
 
@@ -89,8 +91,6 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 }
 
 func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
-	*t = PaymentInfo{}
-
 	br := cbg.GetPeeker(r)
 	scratch := make([]byte, 8)
 
@@ -131,17 +131,29 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 				}
 
 			}
-			// t.WaitSentinel (cid.Cid) (struct)
-		case "WaitSentinel":
+			// t.ChannelMessage (cid.Cid) (struct)
+		case "ChannelMessage":
 
 			{
 
-				c, err := cbg.ReadCid(br)
+				pb, err := br.PeekByte()
 				if err != nil {
-					return xerrors.Errorf("failed to read cid field t.WaitSentinel: %w", err)
+					return err
 				}
+				if pb == cbg.CborNull[0] {
+					var nbuf [1]byte
+					if _, err := br.Read(nbuf[:]); err != nil {
+						return err
+					}
+				} else {
 
-				t.WaitSentinel = c
+					c, err := cbg.ReadCid(br)
+					if err != nil {
+						return xerrors.Errorf("failed to read cid field t.ChannelMessage: %w", err)
+					}
+
+					t.ChannelMessage = &c
+				}
 
 			}
 			// t.Vouchers ([]*paych.SignedVoucher) (slice)
@@ -175,8 +187,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			// Field doesn't exist on this type, so ignore it
-			cbg.ScanForLinks(r, func(cid.Cid) {})
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
 		}
 	}
 
@@ -201,7 +212,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SectorID"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("SectorID")); err != nil {
+	if _, err := io.WriteString(w, "SectorID"); err != nil {
 		return err
 	}
 
@@ -209,7 +220,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	// t.Offset (abi.PaddedPieceSize) (uint64)
+	// t.Offset (uint64) (uint64)
 	if len("Offset") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"Offset\" was too long")
 	}
@@ -217,7 +228,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Offset"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Offset")); err != nil {
+	if _, err := io.WriteString(w, "Offset"); err != nil {
 		return err
 	}
 
@@ -233,7 +244,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Size"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Size")); err != nil {
+	if _, err := io.WriteString(w, "Size"); err != nil {
 		return err
 	}
 
@@ -245,8 +256,6 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 }
 
 func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
-	*t = SealedRef{}
-
 	br := cbg.GetPeeker(r)
 	scratch := make([]byte, 8)
 
@@ -292,7 +301,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 				t.SectorID = abi.SectorNumber(extra)
 
 			}
-			// t.Offset (abi.PaddedPieceSize) (uint64)
+			// t.Offset (uint64) (uint64)
 		case "Offset":
 
 			{
@@ -304,7 +313,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 				if maj != cbg.MajUnsignedInt {
 					return fmt.Errorf("wrong type for uint64 field")
 				}
-				t.Offset = abi.PaddedPieceSize(extra)
+				t.Offset = uint64(extra)
 
 			}
 			// t.Size (abi.UnpaddedPieceSize) (uint64)
@@ -324,8 +333,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			// Field doesn't exist on this type, so ignore it
-			cbg.ScanForLinks(r, func(cid.Cid) {})
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
 		}
 	}
 
@@ -350,7 +358,7 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Refs"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Refs")); err != nil {
+	if _, err := io.WriteString(w, "Refs"); err != nil {
 		return err
 	}
 
@@ -370,8 +378,6 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
 }
 
 func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
-	*t = SealedRefs{}
-
 	br := cbg.GetPeeker(r)
 	scratch := make([]byte, 8)
 
@@ -433,8 +439,7 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			// Field doesn't exist on this type, so ignore it
-			cbg.ScanForLinks(r, func(cid.Cid) {})
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
 		}
 	}
 
@@ -459,7 +464,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Value")); err != nil {
+	if _, err := io.WriteString(w, "Value"); err != nil {
 		return err
 	}
 
@@ -471,7 +476,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	if _, err := w.Write(t.Value[:]); err != nil {
+	if _, err := w.Write(t.Value); err != nil {
 		return err
 	}
 
@@ -483,7 +488,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Epoch")); err != nil {
+	if _, err := io.WriteString(w, "Epoch"); err != nil {
 		return err
 	}
 
@@ -500,8 +505,6 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 }
 
 func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
-	*t = SealTicket{}
-
 	br := cbg.GetPeeker(r)
 	scratch := make([]byte, 8)
 
@@ -546,12 +549,8 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
 			if maj != cbg.MajByteString {
 				return fmt.Errorf("expected byte array")
 			}
-
-			if extra > 0 {
-				t.Value = make([]uint8, extra)
-			}
-
-			if _, err := io.ReadFull(br, t.Value[:]); err != nil {
+			t.Value = make([]byte, extra)
+			if _, err := io.ReadFull(br, t.Value); err != nil {
 				return err
 			}
 			// t.Epoch (abi.ChainEpoch) (int64)
@@ -582,8 +581,7 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			// Field doesn't exist on this type, so ignore it
-			cbg.ScanForLinks(r, func(cid.Cid) {})
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
 		}
 	}
 
@@ -608,7 +606,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Value")); err != nil {
+	if _, err := io.WriteString(w, "Value"); err != nil {
 		return err
 	}
 
@@ -620,7 +618,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	if _, err := w.Write(t.Value[:]); err != nil {
+	if _, err := w.Write(t.Value); err != nil {
 		return err
 	}
 
@@ -632,7 +630,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil {
 		return err
 	}
-	if _, err := io.WriteString(w, string("Epoch")); err != nil {
+	if _, err := io.WriteString(w, "Epoch"); err != nil {
 		return err
 	}
 
@@ -649,8 +647,6 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 }
 
 func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
-	*t = SealSeed{}
-
 	br := cbg.GetPeeker(r)
 	scratch := make([]byte, 8)
 
@@ -695,12 +691,8 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
 			if maj != cbg.MajByteString {
 				return fmt.Errorf("expected byte array")
 			}
-
-			if extra > 0 {
-				t.Value = make([]uint8, extra)
-			}
-
-			if _, err := io.ReadFull(br, t.Value[:]); err != nil {
+			t.Value = make([]byte, extra)
+			if _, err := io.ReadFull(br, t.Value); err != nil {
 				return err
 			}
 			// t.Epoch (abi.ChainEpoch) (int64)
@@ -731,8 +723,7 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			// Field doesn't exist on this type, so ignore it
-			cbg.ScanForLinks(r, func(cid.Cid) {})
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
 		}
 	}
 

api/client/client.go

@@ -1,23 +1,18 @@
 package client
 
 import (
-	"context"
 	"net/http"
-	"net/url"
-	"path"
-	"time"
 
 	"github.com/filecoin-project/go-jsonrpc"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/apistruct"
-	"github.com/filecoin-project/lotus/lib/rpcenc"
 )
 
 // NewCommonRPC creates a new http jsonrpc client.
-func NewCommonRPC(ctx context.Context, addr string, requestHeader http.Header) (api.Common, jsonrpc.ClientCloser, error) {
+func NewCommonRPC(addr string, requestHeader http.Header) (api.Common, jsonrpc.ClientCloser, error) {
 	var res apistruct.CommonStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
 		[]interface{}{
 			&res.Internal,
 		},
@@ -28,9 +23,9 @@ func NewCommonRPC(ctx context.Context, addr string, requestHeader http.Header) (
 }
 
 // NewFullNodeRPC creates a new http jsonrpc client.
-func NewFullNodeRPC(ctx context.Context, addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
+func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
 	var res apistruct.FullNodeStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
 		[]interface{}{
 			&res.CommonStruct.Internal,
 			&res.Internal,
@@ -39,67 +34,23 @@ func NewFullNodeRPC(ctx context.Context, addr string, requestHeader http.Header)
 	return &res, closer, err
 }
 
-// NewStorageMinerRPC creates a new http jsonrpc client for miner
-func NewStorageMinerRPC(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.StorageMiner, jsonrpc.ClientCloser, error) {
+// NewStorageMinerRPC creates a new http jsonrpc client for storage miner
+func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMiner, jsonrpc.ClientCloser, error) {
 	var res apistruct.StorageMinerStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
 		[]interface{}{
 			&res.CommonStruct.Internal,
 			&res.Internal,
 		},
 		requestHeader,
-		opts...,
 	)
 
 	return &res, closer, err
 }
 
-func NewWorkerRPC(ctx context.Context, addr string, requestHeader http.Header) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
-	u, err := url.Parse(addr)
-	if err != nil {
-		return nil, nil, err
-	}
-	switch u.Scheme {
-	case "ws":
-		u.Scheme = "http"
-	case "wss":
-		u.Scheme = "https"
-	}
-	///rpc/v0 -> /rpc/streams/v0/push
-
-	u.Path = path.Join(u.Path, "../streams/v0/push")
-
+func NewWorkerRPC(addr string, requestHeader http.Header) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
 	var res apistruct.WorkerStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
-		[]interface{}{
-			&res.Internal,
-		},
-		requestHeader,
-		rpcenc.ReaderParamEncoder(u.String()),
-		jsonrpc.WithNoReconnect(),
-		jsonrpc.WithTimeout(30*time.Second),
-	)
-
-	return &res, closer, err
-}
-
-// NewGatewayRPC creates a new http jsonrpc client for a gateway node.
-func NewGatewayRPC(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.GatewayAPI, jsonrpc.ClientCloser, error) {
-	var res apistruct.GatewayStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
-		[]interface{}{
-			&res.Internal,
-		},
-		requestHeader,
-		opts...,
-	)
-
-	return &res, closer, err
-}
-
-func NewWalletRPC(ctx context.Context, addr string, requestHeader http.Header) (api.WalletAPI, jsonrpc.ClientCloser, error) {
-	var res apistruct.WalletStruct
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
+	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
 		[]interface{}{
 			&res.Internal,
 		},

api/docgen/docgen.go

@@ -6,44 +6,27 @@ import (
 	"go/ast"
 	"go/parser"
 	"go/token"
-	"os"
 	"reflect"
 	"sort"
 	"strings"
 	"time"
 	"unicode"
 
-	"github.com/google/uuid"
-	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-filestore"
-	metrics "github.com/libp2p/go-libp2p-core/metrics"
-	"github.com/libp2p/go-libp2p-core/network"
-	"github.com/libp2p/go-libp2p-core/peer"
-	protocol "github.com/libp2p/go-libp2p-core/protocol"
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
-	"github.com/multiformats/go-multiaddr"
-
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
-	datatransfer "github.com/filecoin-project/go-data-transfer"
-	filestore2 "github.com/filecoin-project/go-fil-markets/filestore"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-jsonrpc/auth"
-	"github.com/filecoin-project/go-multistore"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/crypto"
-	"github.com/filecoin-project/go-state-types/exitcode"
-
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/api/apistruct"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
-	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
+	"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-filestore"
+	"github.com/libp2p/go-libp2p-core/network"
+	peer "github.com/libp2p/go-libp2p-peer"
+	"github.com/multiformats/go-multiaddr"
 )
 
 var ExampleValues = map[reflect.Type]interface{}{
@@ -82,23 +65,17 @@ func init() {
 
 	ExampleValues[reflect.TypeOf(addr)] = addr
 
-	pid, err := peer.Decode("12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf")
+	pid, err := peer.IDB58Decode("12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf")
 	if err != nil {
 		panic(err)
 	}
 	addExample(pid)
-	addExample(&pid)
 
 	addExample(bitfield.NewFromSet([]uint64{5}))
-	addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1)
-	addExample(abi.RegisteredPoStProof_StackedDrgWindow32GiBV1)
+	addExample(abi.RegisteredProof_StackedDRG32GiBPoSt)
 	addExample(abi.ChainEpoch(10101))
 	addExample(crypto.SigTypeBLS)
-	addExample(types.KTBLS)
 	addExample(int64(9))
-	addExample(12.3)
-	addExample(123)
-	addExample(uintptr(0))
 	addExample(abi.MethodNum(1))
 	addExample(exitcode.ExitCode(0))
 	addExample(crypto.DomainSeparationTag_ElectionProofProduction)
@@ -113,54 +90,21 @@ func init() {
 	addExample(network.Connected)
 	addExample(dtypes.NetworkName("lotus"))
 	addExample(api.SyncStateStage(1))
-	addExample(api.FullAPIVersion)
+	addExample(build.APIVersion)
 	addExample(api.PCHInbound)
 	addExample(time.Minute)
-	addExample(datatransfer.TransferID(3))
-	addExample(datatransfer.Ongoing)
-	addExample(multistore.StoreID(50))
-	addExample(retrievalmarket.ClientEventDealAccepted)
-	addExample(retrievalmarket.DealStatusNew)
-	addExample(network.ReachabilityPublic)
-	addExample(build.NewestNetworkVersion)
-	addExample(map[string]int{"name": 42})
-	addExample(map[string]time.Time{"name": time.Unix(1615243938, 0).UTC()})
-	addExample(&types.ExecutionTrace{
-		Msg:    exampleValue("init", reflect.TypeOf(&types.Message{}), nil).(*types.Message),
-		MsgRct: exampleValue("init", reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt),
+	addExample(&types.ExecutionResult{
+		Msg:    exampleValue(reflect.TypeOf(&types.Message{})).(*types.Message),
+		MsgRct: exampleValue(reflect.TypeOf(&types.MessageReceipt{})).(*types.MessageReceipt),
 	})
 	addExample(map[string]types.Actor{
-		"t01236": exampleValue("init", reflect.TypeOf(types.Actor{}), nil).(types.Actor),
+		"t01236": exampleValue(reflect.TypeOf(types.Actor{})).(types.Actor),
 	})
 	addExample(map[string]api.MarketDeal{
-		"t026363": exampleValue("init", reflect.TypeOf(api.MarketDeal{}), nil).(api.MarketDeal),
+		"t026363": exampleValue(reflect.TypeOf(api.MarketDeal{})).(api.MarketDeal),
 	})
 	addExample(map[string]api.MarketBalance{
-		"t026363": exampleValue("init", reflect.TypeOf(api.MarketBalance{}), nil).(api.MarketBalance),
-	})
-	addExample(map[string]*pubsub.TopicScoreSnapshot{
-		"/blocks": {
-			TimeInMesh:               time.Minute,
-			FirstMessageDeliveries:   122,
-			MeshMessageDeliveries:    1234,
-			InvalidMessageDeliveries: 3,
-		},
-	})
-	addExample(map[string]metrics.Stats{
-		"12D3KooWSXmXLJmBR1M7i9RW9GQPNUhZSzXKzxDHWtAgNuJAbyEJ": {
-			RateIn:   100,
-			RateOut:  50,
-			TotalIn:  174000,
-			TotalOut: 12500,
-		},
-	})
-	addExample(map[protocol.ID]metrics.Stats{
-		"/fil/hello/1.0.0": {
-			RateIn:   100,
-			RateOut:  50,
-			TotalIn:  174000,
-			TotalOut: 12500,
-		},
+		"t026363": exampleValue(reflect.TypeOf(api.MarketBalance{})).(api.MarketBalance),
 	})
 
 	maddr, err := multiaddr.NewMultiaddr("/ip4/52.36.61.156/tcp/1347/p2p/12D3KooWFETiESTf1v4PGUvtnxMAcEFMzLZbJGg4tjWfGEimYior")
@@ -171,89 +115,9 @@ func init() {
 	// because reflect.TypeOf(maddr) returns the concrete type...
 	ExampleValues[reflect.TypeOf(struct{ A multiaddr.Multiaddr }{}).Field(0).Type] = maddr
 
-	// miner specific
-	addExample(filestore2.Path(".lotusminer/fstmp123"))
-	si := multistore.StoreID(12)
-	addExample(&si)
-	addExample(retrievalmarket.DealID(5))
-	addExample(abi.ActorID(1000))
-	addExample(map[string][]api.SealedRef{
-		"98000": {
-			api.SealedRef{
-				SectorID: 100,
-				Offset:   10 << 20,
-				Size:     1 << 20,
-			},
-		},
-	})
-	addExample(api.SectorState(sealing.Proving))
-	addExample(stores.ID("76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8"))
-	addExample(storiface.FTUnsealed)
-	addExample(storiface.PathSealing)
-	addExample(map[stores.ID][]stores.Decl{
-		"76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8": {
-			{
-				SectorID:       abi.SectorID{Miner: 1000, Number: 100},
-				SectorFileType: storiface.FTSealed,
-			},
-		},
-	})
-	addExample(map[stores.ID]string{
-		"76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8": "/data/path",
-	})
-	addExample(map[uuid.UUID][]storiface.WorkerJob{
-		uuid.MustParse("ef8d99a2-6865-4189-8ffa-9fef0f806eee"): {
-			{
-				ID: storiface.CallID{
-					Sector: abi.SectorID{Miner: 1000, Number: 100},
-					ID:     uuid.MustParse("76081ba0-61bd-45a5-bc08-af05f1c26e5d"),
-				},
-				Sector:   abi.SectorID{Miner: 1000, Number: 100},
-				Task:     sealtasks.TTPreCommit2,
-				RunWait:  0,
-				Start:    time.Unix(1605172927, 0).UTC(),
-				Hostname: "host",
-			},
-		},
-	})
-	addExample(map[uuid.UUID]storiface.WorkerStats{
-		uuid.MustParse("ef8d99a2-6865-4189-8ffa-9fef0f806eee"): {
-			Info: storiface.WorkerInfo{
-				Hostname: "host",
-				Resources: storiface.WorkerResources{
-					MemPhysical: 256 << 30,
-					MemSwap:     120 << 30,
-					MemReserved: 2 << 30,
-					CPUs:        64,
-					GPUs:        []string{"aGPU 1337"},
-				},
-			},
-			Enabled:    true,
-			MemUsedMin: 0,
-			MemUsedMax: 0,
-			GpuUsed:    false,
-			CpuUse:     0,
-		},
-	})
-	addExample(storiface.ErrorCode(0))
-	addExample(map[abi.SectorNumber]string{
-		123: "can't acquire read lock",
-	})
-	addExample(map[api.SectorState]int{
-		api.SectorState(sealing.Proving): 120,
-	})
-	addExample([]abi.SectorNumber{123, 124})
-
-	// worker specific
-	addExample(storiface.AcquireMove)
-	addExample(storiface.UnpaddedByteIndex(abi.PaddedPieceSize(1 << 20).Unpadded()))
-	addExample(map[sealtasks.TaskType]struct{}{
-		sealtasks.TTPreCommit2: {},
-	})
-	addExample(sealtasks.TTCommit2)
 }
 
-func exampleValue(method string, t, parent reflect.Type) interface{} {
+func exampleValue(t reflect.Type) interface{} {
 	v, ok := ExampleValues[t]
 	if ok {
 		return v
@@ -262,25 +126,25 @@ func exampleValue(method string, t, parent reflect.Type) interface{} {
 	switch t.Kind() {
 	case reflect.Slice:
 		out := reflect.New(t).Elem()
-		reflect.Append(out, reflect.ValueOf(exampleValue(method, t.Elem(), t)))
+		reflect.Append(out, reflect.ValueOf(exampleValue(t.Elem())))
 		return out.Interface()
 	case reflect.Chan:
-		return exampleValue(method, t.Elem(), nil)
+		return exampleValue(t.Elem())
 	case reflect.Struct:
-		es := exampleStruct(method, t, parent)
+		es := exampleStruct(t)
 		v := reflect.ValueOf(es).Elem().Interface()
 		ExampleValues[t] = v
 		return v
 	case reflect.Array:
 		out := reflect.New(t).Elem()
 		for i := 0; i < t.Len(); i++ {
-			out.Index(i).Set(reflect.ValueOf(exampleValue(method, t.Elem(), t)))
+			out.Index(i).Set(reflect.ValueOf(exampleValue(t.Elem())))
 		}
 		return out.Interface()
 
 	case reflect.Ptr:
 		if t.Elem().Kind() == reflect.Struct {
-			es := exampleStruct(method, t.Elem(), t)
+			es := exampleStruct(t.Elem())
 			//ExampleValues[t] = es
 			return es
 		}
@@ -288,18 +152,15 @@ func exampleValue(method string, t, parent reflect.Type) interface{} {
 		return struct{}{}
 	}
 
-	panic(fmt.Sprintf("No example value for type: %s (method '%s')", t, method))
+	panic(fmt.Sprintf("No example value for type: %s", t))
 }
 
-func exampleStruct(method string, t, parent reflect.Type) interface{} {
+func exampleStruct(t reflect.Type) interface{} {
 	ns := reflect.New(t)
 	for i := 0; i < t.NumField(); i++ {
 		f := t.Field(i)
-		if f.Type == parent {
-			continue
-		}
 		if strings.Title(f.Name) == f.Name {
-			ns.Elem().Field(i).Set(reflect.ValueOf(exampleValue(method, f.Type, t)))
+			ns.Elem().Field(i).Set(reflect.ValueOf(exampleValue(f.Type)))
 		}
 	}
 
@@ -307,7 +168,6 @@ func exampleStruct(method string, t, parent reflect.Type) interface{} {
 }
 
 type Visitor struct {
-	Root    string
 	Methods map[string]ast.Node
 }
 
@@ -317,7 +177,7 @@ func (v *Visitor) Visit(node ast.Node) ast.Visitor {
 		return v
 	}
 
-	if st.Name.Name != v.Root {
+	if st.Name.Name != "FullNode" {
 		return nil
 	}
 
@@ -333,7 +193,7 @@ func (v *Visitor) Visit(node ast.Node) ast.Visitor {
 
 const noComment = "There are not yet any comments for this method."
 
-func parseApiASTInfo(apiFile, iface string) (map[string]string, map[string]string) { //nolint:golint
+func parseApiASTInfo() (map[string]string, map[string]string) { //nolint:golint
 	fset := token.NewFileSet()
 	pkgs, err := parser.ParseDir(fset, "./api", nil, parser.AllErrors|parser.ParseComments)
 	if err != nil {
@@ -342,11 +202,11 @@ func parseApiASTInfo(apiFile, iface string) (map[string]string, map[string]strin
 
 	ap := pkgs["api"]
 
-	f := ap.Files[apiFile]
+	f := ap.Files["api/api_full.go"]
 
 	cmap := ast.NewCommentMap(fset, f, f.Comments)
 
-	v := &Visitor{iface, make(map[string]ast.Node)}
+	v := &Visitor{make(map[string]ast.Node)}
 	ast.Walk(v, pkgs["api"])
 
 	groupDocs := make(map[string]string)
@@ -402,30 +262,13 @@ func methodGroupFromName(mn string) string {
 }
 
 func main() {
-	comments, groupComments := parseApiASTInfo(os.Args[1], os.Args[2])
+
+	comments, groupComments := parseApiASTInfo()
 
 	groups := make(map[string]*MethodGroup)
 
-	var t reflect.Type
-	var permStruct, commonPermStruct reflect.Type
-
-	switch os.Args[2] {
-	case "FullNode":
-		t = reflect.TypeOf(new(struct{ api.FullNode })).Elem()
-		permStruct = reflect.TypeOf(apistruct.FullNodeStruct{}.Internal)
-		commonPermStruct = reflect.TypeOf(apistruct.CommonStruct{}.Internal)
-	case "StorageMiner":
-		t = reflect.TypeOf(new(struct{ api.StorageMiner })).Elem()
-		permStruct = reflect.TypeOf(apistruct.StorageMinerStruct{}.Internal)
-		commonPermStruct = reflect.TypeOf(apistruct.CommonStruct{}.Internal)
-	case "WorkerAPI":
-		t = reflect.TypeOf(new(struct{ api.WorkerAPI })).Elem()
-		permStruct = reflect.TypeOf(apistruct.WorkerStruct{}.Internal)
-		commonPermStruct = reflect.TypeOf(apistruct.WorkerStruct{}.Internal)
-	default:
-		panic("unknown type")
-	}
-
+	var api struct{ api.FullNode }
+	t := reflect.TypeOf(api)
 	for i := 0; i < t.NumMethod(); i++ {
 		m := t.Method(i)
 
@@ -443,17 +286,17 @@ func main() {
 		ft := m.Func.Type()
 		for j := 2; j < ft.NumIn(); j++ {
 			inp := ft.In(j)
-			args = append(args, exampleValue(m.Name, inp, nil))
+			args = append(args, exampleValue(inp))
 		}
 
-		v, err := json.MarshalIndent(args, "", "  ")
+		v, err := json.Marshal(args)
 		if err != nil {
 			panic(err)
 		}
 
-		outv := exampleValue(m.Name, ft.Out(0), nil)
+		outv := exampleValue(ft.Out(0))
 
-		ov, err := json.MarshalIndent(outv, "", "  ")
+		ov, err := json.Marshal(outv)
 		if err != nil {
 			panic(err)
 		}
@@ -475,15 +318,6 @@ func main() {
 		return groupslice[i].GroupName < groupslice[j].GroupName
 	})
 
-	fmt.Printf("# Groups\n")
-
-	for _, g := range groupslice {
-		fmt.Printf("* [%s](#%s)\n", g.GroupName, g.GroupName)
-		for _, method := range g.Methods {
-			fmt.Printf("  * [%s](#%s)\n", method.Name, method.Name)
-		}
-	}
-
 	for _, g := range groupslice {
 		g := g
 		fmt.Printf("## %s\n", g.GroupName)
@@ -497,29 +331,8 @@ func main() {
 			fmt.Printf("### %s\n", m.Name)
 			fmt.Printf("%s\n\n", m.Comment)
 
-			meth, ok := permStruct.FieldByName(m.Name)
-			if !ok {
-				meth, ok = commonPermStruct.FieldByName(m.Name)
-				if !ok {
-					panic("no perms for method: " + m.Name)
-				}
-			}
-
-			perms := meth.Tag.Get("perm")
-
-			fmt.Printf("Perms: %s\n\n", perms)
-
-			if strings.Count(m.InputExample, "\n") > 0 {
-				fmt.Printf("Inputs:\n```json\n%s\n```\n\n", m.InputExample)
-			} else {
-				fmt.Printf("Inputs: `%s`\n\n", m.InputExample)
-			}
-
-			if strings.Count(m.ResponseExample, "\n") > 0 {
-				fmt.Printf("Response:\n```json\n%s\n```\n\n", m.ResponseExample)
-			} else {
-				fmt.Printf("Response: `%s`\n\n", m.ResponseExample)
-			}
+			fmt.Printf("Inputs: `%s`\n\n", m.InputExample)
+			fmt.Printf("Response: `%s`\n\n", m.ResponseExample)
 		}
 	}
 }

api/test/blockminer.go

@@ -1,61 +0,0 @@
-package test
-
-import (
-	"context"
-	"fmt"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/miner"
-)
-
-type BlockMiner struct {
-	ctx       context.Context
-	t         *testing.T
-	miner     TestStorageNode
-	blocktime time.Duration
-	mine      int64
-	nulls     int64
-	done      chan struct{}
-}
-
-func NewBlockMiner(ctx context.Context, t *testing.T, miner TestStorageNode, blocktime time.Duration) *BlockMiner {
-	return &BlockMiner{
-		ctx:       ctx,
-		t:         t,
-		miner:     miner,
-		blocktime: blocktime,
-		mine:      int64(1),
-		done:      make(chan struct{}),
-	}
-}
-
-func (bm *BlockMiner) MineBlocks() {
-	time.Sleep(time.Second)
-	go func() {
-		defer close(bm.done)
-		for atomic.LoadInt64(&bm.mine) == 1 {
-			select {
-			case <-bm.ctx.Done():
-				return
-			case <-time.After(bm.blocktime):
-			}
-
-			nulls := atomic.SwapInt64(&bm.nulls, 0)
-			if err := bm.miner.MineOne(bm.ctx, miner.MineReq{
-				InjectNulls: abi.ChainEpoch(nulls),
-				Done:        func(bool, abi.ChainEpoch, error) {},
-			}); err != nil {
-				bm.t.Error(err)
-			}
-		}
-	}()
-}
-
-func (bm *BlockMiner) Stop() {
-	atomic.AddInt64(&bm.mine, -1)
-	fmt.Println("shutting down mining")
-	<-bm.done
-}

api/test/ccupgrade.go

@@ -1,127 +0,0 @@
-package test
-
-import (
-	"context"
-	"fmt"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/filecoin-project/go-state-types/abi"
-
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/node/impl"
-)
-
-func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) {
-	for _, height := range []abi.ChainEpoch{
-		2,    // before
-		162,  // while sealing
-		530,  // after upgrade deal
-		5000, // after
-	} {
-		height := height // make linters happy by copying
-		t.Run(fmt.Sprintf("upgrade-%d", height), func(t *testing.T) {
-			testCCUpgrade(t, b, blocktime, height)
-		})
-	}
-}
-
-func testCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, upgradeHeight abi.ChainEpoch) {
-	ctx := context.Background()
-	n, sn := b(t, []FullNodeOpts{FullNodeWithActorsV3At(upgradeHeight)}, OneMiner)
-	client := n[0].FullNode.(*impl.FullNodeAPI)
-	miner := sn[0]
-
-	addrinfo, err := client.NetAddrsListen(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err := miner.NetConnect(ctx, addrinfo); err != nil {
-		t.Fatal(err)
-	}
-	time.Sleep(time.Second)
-
-	mine := int64(1)
-	done := make(chan struct{})
-	go func() {
-		defer close(done)
-		for atomic.LoadInt64(&mine) == 1 {
-			time.Sleep(blocktime)
-			if err := sn[0].MineOne(ctx, MineNext); err != nil {
-				t.Error(err)
-			}
-		}
-	}()
-
-	maddr, err := miner.ActorAddress(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	CC := abi.SectorNumber(GenesisPreseals + 1)
-	Upgraded := CC + 1
-
-	pledgeSectors(t, ctx, miner, 1, 0, nil)
-
-	sl, err := miner.SectorsList(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(sl) != 1 {
-		t.Fatal("expected 1 sector")
-	}
-
-	if sl[0] != CC {
-		t.Fatal("bad")
-	}
-
-	{
-		si, err := client.StateSectorGetInfo(ctx, maddr, CC, types.EmptyTSK)
-		require.NoError(t, err)
-		require.Less(t, 50000, int(si.Expiration))
-	}
-
-	if err := miner.SectorMarkForUpgrade(ctx, sl[0]); err != nil {
-		t.Fatal(err)
-	}
-
-	MakeDeal(t, ctx, 6, client, miner, false, false, 0)
-
-	// Validate upgrade
-
-	{
-		exp, err := client.StateSectorExpiration(ctx, maddr, CC, types.EmptyTSK)
-		require.NoError(t, err)
-		require.NotNil(t, exp)
-		require.Greater(t, 50000, int(exp.OnTime))
-	}
-	{
-		exp, err := client.StateSectorExpiration(ctx, maddr, Upgraded, types.EmptyTSK)
-		require.NoError(t, err)
-		require.Less(t, 50000, int(exp.OnTime))
-	}
-
-	dlInfo, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
-	require.NoError(t, err)
-
-	// Sector should expire.
-	for {
-		// Wait for the sector to expire.
-		status, err := miner.SectorsStatus(ctx, CC, true)
-		require.NoError(t, err)
-		if status.OnTime == 0 && status.Early == 0 {
-			break
-		}
-		t.Log("waiting for sector to expire")
-		// wait one deadline per loop.
-		time.Sleep(time.Duration(dlInfo.WPoStChallengeWindow) * blocktime)
-	}
-
-	fmt.Println("shutting down mining")
-	atomic.AddInt64(&mine, -1)
-	<-done
-}

api/test/deals.go

@@ -12,264 +12,128 @@ import (
 	"time"
 
 	"github.com/ipfs/go-cid"
+
 	files "github.com/ipfs/go-ipfs-files"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/ipld/go-car"
-	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
-	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
-	"github.com/filecoin-project/lotus/chain/types"
-	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
-	"github.com/filecoin-project/lotus/markets/storageadapter"
-	"github.com/filecoin-project/lotus/node"
-	"github.com/filecoin-project/lotus/node/impl"
-	market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
-	ipld "github.com/ipfs/go-ipld-format"
 	dag "github.com/ipfs/go-merkledag"
 	dstest "github.com/ipfs/go-merkledag/test"
 	unixfile "github.com/ipfs/go-unixfs/file"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/node/impl"
+	ipld "github.com/ipfs/go-ipld-format"
 )
 
-func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport, fastRet bool, startEpoch abi.ChainEpoch) {
-	s := setupOneClientOneMiner(t, b, blocktime)
-	defer s.blockMiner.Stop()
-
-	MakeDeal(t, s.ctx, 6, s.client, s.miner, carExport, fastRet, startEpoch)
+func init() {
+	logging.SetAllLoggers(logging.LevelInfo)
+	build.InsecurePoStValidation = true
 }
 
-func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, startEpoch abi.ChainEpoch) {
-	s := setupOneClientOneMiner(t, b, blocktime)
-	defer s.blockMiner.Stop()
+func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport bool) {
+	_ = os.Setenv("BELLMAN_NO_GPU", "1")
 
-	MakeDeal(t, s.ctx, 6, s.client, s.miner, false, false, startEpoch)
-	MakeDeal(t, s.ctx, 7, s.client, s.miner, false, false, startEpoch)
-}
+	ctx := context.Background()
+	n, sn := b(t, 1, oneMiner)
+	client := n[0].FullNode.(*impl.FullNodeAPI)
+	miner := sn[0]
 
-func MakeDeal(t *testing.T, ctx context.Context, rseed int, client api.FullNode, miner TestStorageNode, carExport, fastRet bool, startEpoch abi.ChainEpoch) {
-	res, data, err := CreateClientFile(ctx, client, rseed)
+	addrinfo, err := client.NetAddrsListen(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	fcid := res.Root
-	fmt.Println("FILE CID: ", fcid)
-
-	deal := startDeal(t, ctx, miner, client, fcid, fastRet, startEpoch)
-
-	// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
+	if err := miner.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
 	time.Sleep(time.Second)
-	waitDealSealed(t, ctx, miner, client, deal, false)
 
-	// Retrieval
-	info, err := client.ClientGetDealInfo(ctx, *deal)
-	require.NoError(t, err)
+	mine := true
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for mine {
+			time.Sleep(blocktime)
+			if err := sn[0].MineOne(ctx, func(bool) {}); err != nil {
+				t.Error(err)
+			}
+		}
+	}()
 
-	testRetrieval(t, ctx, client, fcid, &info.PieceCID, carExport, data)
+	makeDeal(t, ctx, 6, client, miner, carExport)
+
+	mine = false
+	fmt.Println("shutting down mining")
+	<-done
 }
 
-func CreateClientFile(ctx context.Context, client api.FullNode, rseed int) (*api.ImportRes, []byte, error) {
+func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
+	_ = os.Setenv("BELLMAN_NO_GPU", "1")
+
+	ctx := context.Background()
+	n, sn := b(t, 1, oneMiner)
+	client := n[0].FullNode.(*impl.FullNodeAPI)
+	miner := sn[0]
+
+	addrinfo, err := client.NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := miner.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Second)
+
+	mine := true
+	done := make(chan struct{})
+
+	go func() {
+		defer close(done)
+		for mine {
+			time.Sleep(blocktime)
+			if err := sn[0].MineOne(ctx, func(bool) {}); err != nil {
+				t.Error(err)
+			}
+		}
+	}()
+
+	makeDeal(t, ctx, 6, client, miner, false)
+	makeDeal(t, ctx, 7, client, miner, false)
+
+	mine = false
+	fmt.Println("shutting down mining")
+	<-done
+}
+
+func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNodeAPI, miner TestStorageNode, carExport bool) {
 	data := make([]byte, 1600)
 	rand.New(rand.NewSource(int64(rseed))).Read(data)
 
-	dir, err := ioutil.TempDir(os.TempDir(), "test-make-deal-")
-	if err != nil {
-		return nil, nil, err
-	}
-
-	path := filepath.Join(dir, "sourcefile.dat")
-	err = ioutil.WriteFile(path, data, 0644)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	res, err := client.ClientImport(ctx, api.FileRef{Path: path})
-	if err != nil {
-		return nil, nil, err
-	}
-	return res, data, nil
-}
-
-func TestPublishDealsBatching(t *testing.T, b APIBuilder, blocktime time.Duration, startEpoch abi.ChainEpoch) {
-	publishPeriod := 10 * time.Second
-	maxDealsPerMsg := uint64(2)
-
-	// Set max deals per publish deals message to 2
-	minerDef := []StorageMiner{{
-		Full: 0,
-		Opts: node.Override(
-			new(*storageadapter.DealPublisher),
-			storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{
-				Period:         publishPeriod,
-				MaxDealsPerMsg: maxDealsPerMsg,
-			})),
-		Preseal: PresealGenesis,
-	}}
-
-	// Create a connect client and miner node
-	n, sn := b(t, OneFull, minerDef)
-	client := n[0].FullNode.(*impl.FullNodeAPI)
-	miner := sn[0]
-	s := connectAndStartMining(t, b, blocktime, client, miner)
-	defer s.blockMiner.Stop()
-
-	// Starts a deal and waits until it's published
-	runDealTillPublish := func(rseed int) {
-		res, _, err := CreateClientFile(s.ctx, s.client, rseed)
-		require.NoError(t, err)
-
-		upds, err := client.ClientGetDealUpdates(s.ctx)
-		require.NoError(t, err)
-
-		startDeal(t, s.ctx, s.miner, s.client, res.Root, false, startEpoch)
-
-		// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
-		time.Sleep(time.Second)
-
-		done := make(chan struct{})
-		go func() {
-			for upd := range upds {
-				if upd.DataRef.Root == res.Root && upd.State == storagemarket.StorageDealAwaitingPreCommit {
-					done <- struct{}{}
-				}
-			}
-		}()
-		<-done
-	}
-
-	// Run three deals in parallel
-	done := make(chan struct{}, maxDealsPerMsg+1)
-	for rseed := 1; rseed <= 3; rseed++ {
-		rseed := rseed
-		go func() {
-			runDealTillPublish(rseed)
-			done <- struct{}{}
-		}()
-	}
-
-	// Wait for two of the deals to be published
-	for i := 0; i < int(maxDealsPerMsg); i++ {
-		<-done
-	}
-
-	// Expect a single PublishStorageDeals message that includes the first two deals
-	msgCids, err := s.client.StateListMessages(s.ctx, &api.MessageMatch{To: market.Address}, types.EmptyTSK, 1)
-	require.NoError(t, err)
-	count := 0
-	for _, msgCid := range msgCids {
-		msg, err := s.client.ChainGetMessage(s.ctx, msgCid)
-		require.NoError(t, err)
-
-		if msg.Method == market.Methods.PublishStorageDeals {
-			count++
-			var pubDealsParams market2.PublishStorageDealsParams
-			err = pubDealsParams.UnmarshalCBOR(bytes.NewReader(msg.Params))
-			require.NoError(t, err)
-			require.Len(t, pubDealsParams.Deals, int(maxDealsPerMsg))
-		}
-	}
-	require.Equal(t, 1, count)
-
-	// The third deal should be published once the publish period expires.
-	// Allow a little padding as it takes a moment for the state change to
-	// be noticed by the client.
-	padding := 10 * time.Second
-	select {
-	case <-time.After(publishPeriod + padding):
-		require.Fail(t, "Expected 3rd deal to be published once publish period elapsed")
-	case <-done: // Success
-	}
-}
-
-func TestFastRetrievalDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, startEpoch abi.ChainEpoch) {
-	s := setupOneClientOneMiner(t, b, blocktime)
-	defer s.blockMiner.Stop()
-
-	data := make([]byte, 1600)
-	rand.New(rand.NewSource(int64(8))).Read(data)
-
 	r := bytes.NewReader(data)
-	fcid, err := s.client.ClientImportLocal(s.ctx, r)
+	fcid, err := client.ClientImportLocal(ctx, r)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	fmt.Println("FILE CID: ", fcid)
 
-	deal := startDeal(t, s.ctx, s.miner, s.client, fcid, true, startEpoch)
+	deal := startDeal(t, ctx, miner, client, fcid)
+
+	// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
+	time.Sleep(time.Second)
+	waitDealSealed(t, ctx, client, deal)
 
-	waitDealPublished(t, s.ctx, s.miner, deal)
-	fmt.Println("deal published, retrieving")
 	// Retrieval
-	info, err := s.client.ClientGetDealInfo(s.ctx, *deal)
-	require.NoError(t, err)
 
-	testRetrieval(t, s.ctx, s.client, fcid, &info.PieceCID, false, data)
+	testRetrieval(t, ctx, err, client, fcid, carExport, data)
 }
 
-func TestSecondDealRetrieval(t *testing.T, b APIBuilder, blocktime time.Duration) {
-	s := setupOneClientOneMiner(t, b, blocktime)
-	defer s.blockMiner.Stop()
-
-	{
-		data1 := make([]byte, 800)
-		rand.New(rand.NewSource(int64(3))).Read(data1)
-		r := bytes.NewReader(data1)
-
-		fcid1, err := s.client.ClientImportLocal(s.ctx, r)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		data2 := make([]byte, 800)
-		rand.New(rand.NewSource(int64(9))).Read(data2)
-		r2 := bytes.NewReader(data2)
-
-		fcid2, err := s.client.ClientImportLocal(s.ctx, r2)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		deal1 := startDeal(t, s.ctx, s.miner, s.client, fcid1, true, 0)
-
-		// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
-		time.Sleep(time.Second)
-		waitDealSealed(t, s.ctx, s.miner, s.client, deal1, true)
-
-		deal2 := startDeal(t, s.ctx, s.miner, s.client, fcid2, true, 0)
-
-		time.Sleep(time.Second)
-		waitDealSealed(t, s.ctx, s.miner, s.client, deal2, false)
-
-		// Retrieval
-		info, err := s.client.ClientGetDealInfo(s.ctx, *deal2)
-		require.NoError(t, err)
-
-		rf, _ := s.miner.SectorsRefs(s.ctx)
-		fmt.Printf("refs: %+v\n", rf)
-
-		testRetrieval(t, s.ctx, s.client, fcid2, &info.PieceCID, false, data2)
-	}
-}
-
-func TestZeroPricePerByteRetrievalDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, startEpoch abi.ChainEpoch) {
-	s := setupOneClientOneMiner(t, b, blocktime)
-	defer s.blockMiner.Stop()
-
-	// Set price-per-byte to zero
-	ask, err := s.miner.MarketGetRetrievalAsk(s.ctx)
-	require.NoError(t, err)
-
-	ask.PricePerByte = abi.NewTokenAmount(0)
-	err = s.miner.MarketSetRetrievalAsk(s.ctx, ask)
-	require.NoError(t, err)
-
-	MakeDeal(t, s.ctx, 6, s.client, s.miner, false, false, startEpoch)
-}
-
-func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client api.FullNode, fcid cid.Cid, fastRet bool, startEpoch abi.ChainEpoch) *cid.Cid {
+func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, fcid cid.Cid) *cid.Cid {
 	maddr, err := miner.ActorAddress(ctx)
 	if err != nil {
 		t.Fatal(err)
@@ -280,16 +144,11 @@ func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client
 		t.Fatal(err)
 	}
 	deal, err := client.ClientStartDeal(ctx, &api.StartDealParams{
-		Data: &storagemarket.DataRef{
-			TransferType: storagemarket.TTGraphsync,
-			Root:         fcid,
-		},
+		Data:              &storagemarket.DataRef{Root: fcid},
 		Wallet:            addr,
 		Miner:             maddr,
 		EpochPrice:        types.NewInt(1000000),
-		DealStartEpoch:    startEpoch,
-		MinBlocksDuration: uint64(build.MinDealDuration),
-		FastRetrieval:     fastRet,
+		MinBlocksDuration: 100,
 	})
 	if err != nil {
 		t.Fatalf("%+v", err)
@@ -297,7 +156,7 @@ func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client
 	return deal
 }
 
-func waitDealSealed(t *testing.T, ctx context.Context, miner TestStorageNode, client api.FullNode, deal *cid.Cid, noseal bool) {
+func waitDealSealed(t *testing.T, ctx context.Context, client *impl.FullNodeAPI, deal *cid.Cid) {
 loop:
 	for {
 		di, err := client.ClientGetDealInfo(ctx, *deal)
@@ -305,11 +164,6 @@ loop:
 			t.Fatal(err)
 		}
 		switch di.State {
-		case storagemarket.StorageDealAwaitingPreCommit, storagemarket.StorageDealSealing:
-			if noseal {
-				return
-			}
-			startSealingWaiting(t, ctx, miner)
 		case storagemarket.StorageDealProposalRejected:
 			t.Fatal("deal rejected")
 		case storagemarket.StorageDealFailing:
@@ -325,53 +179,8 @@ loop:
 	}
 }
 
-func waitDealPublished(t *testing.T, ctx context.Context, miner TestStorageNode, deal *cid.Cid) {
-	subCtx, cancel := context.WithCancel(ctx)
-	defer cancel()
-	updates, err := miner.MarketGetDealUpdates(subCtx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	for {
-		select {
-		case <-ctx.Done():
-			t.Fatal("context timeout")
-		case di := <-updates:
-			if deal.Equals(di.ProposalCid) {
-				switch di.State {
-				case storagemarket.StorageDealProposalRejected:
-					t.Fatal("deal rejected")
-				case storagemarket.StorageDealFailing:
-					t.Fatal("deal failed")
-				case storagemarket.StorageDealError:
-					t.Fatal("deal errored", di.Message)
-				case storagemarket.StorageDealFinalizing, storagemarket.StorageDealAwaitingPreCommit, storagemarket.StorageDealSealing, storagemarket.StorageDealActive:
-					fmt.Println("COMPLETE", di)
-					return
-				}
-				fmt.Println("Deal state: ", storagemarket.DealStates[di.State])
-			}
-		}
-	}
-}
-
-func startSealingWaiting(t *testing.T, ctx context.Context, miner TestStorageNode) {
-	snums, err := miner.SectorsList(ctx)
-	require.NoError(t, err)
-
-	for _, snum := range snums {
-		si, err := miner.SectorsStatus(ctx, snum, false)
-		require.NoError(t, err)
-
-		t.Logf("Sector state: %s", si.State)
-		if si.State == api.SectorState(sealing.WaitDeals) {
-			require.NoError(t, miner.SectorStartSealing(ctx, snum))
-		}
-	}
-}
-
-func testRetrieval(t *testing.T, ctx context.Context, client api.FullNode, fcid cid.Cid, piece *cid.Cid, carExport bool, data []byte) {
-	offers, err := client.ClientFindData(ctx, fcid, piece)
+func testRetrieval(t *testing.T, ctx context.Context, err error, client *impl.FullNodeAPI, fcid cid.Cid, carExport bool, data []byte) {
+	offers, err := client.ClientFindData(ctx, fcid)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -395,14 +204,9 @@ func testRetrieval(t *testing.T, ctx context.Context, client api.FullNode, fcid
 		Path:  filepath.Join(rpath, "ret"),
 		IsCAR: carExport,
 	}
-	updates, err := client.ClientRetrieveWithEvents(ctx, offers[0].Order(caddr), ref)
+	err = client.ClientRetrieve(ctx, offers[0].Order(caddr), ref)
 	if err != nil {
-		t.Fatal(err)
-	}
-	for update := range updates {
-		if update.Err != "" {
-			t.Fatalf("retrieval failed: %s", update.Err)
-		}
+		t.Fatalf("%+v", err)
 	}
 
 	rdata, err := ioutil.ReadFile(filepath.Join(rpath, "ret"))
@@ -448,40 +252,3 @@ func extractCarData(t *testing.T, ctx context.Context, rdata []byte, rpath strin
 	}
 	return rdata
 }
-
-type dealsScaffold struct {
-	ctx        context.Context
-	client     *impl.FullNodeAPI
-	miner      TestStorageNode
-	blockMiner *BlockMiner
-}
-
-func setupOneClientOneMiner(t *testing.T, b APIBuilder, blocktime time.Duration) *dealsScaffold {
-	n, sn := b(t, OneFull, OneMiner)
-	client := n[0].FullNode.(*impl.FullNodeAPI)
-	miner := sn[0]
-	return connectAndStartMining(t, b, blocktime, client, miner)
-}
-
-func connectAndStartMining(t *testing.T, b APIBuilder, blocktime time.Duration, client *impl.FullNodeAPI, miner TestStorageNode) *dealsScaffold {
-	ctx := context.Background()
-	addrinfo, err := client.NetAddrsListen(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err := miner.NetConnect(ctx, addrinfo); err != nil {
-		t.Fatal(err)
-	}
-	time.Sleep(time.Second)
-
-	blockMiner := NewBlockMiner(ctx, t, miner, blocktime)
-	blockMiner.MineBlocks()
-
-	return &dealsScaffold{
-		ctx:        ctx,
-		client:     client,
-		miner:      miner,
-		blockMiner: blockMiner,
-	}
-}

api/test/mining.go

@@ -5,45 +5,43 @@ import (
 	"context"
 	"fmt"
 	"math/rand"
+	"os"
 	"sync/atomic"
 	"testing"
 	"time"
 
 	logging "github.com/ipfs/go-log/v2"
 
-	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/miner"
 	"github.com/filecoin-project/lotus/node/impl"
 )
 
-//nolint:deadcode,varcheck
 var log = logging.Logger("apitest")
 
 func (ts *testSuite) testMining(t *testing.T) {
 	ctx := context.Background()
-	apis, sn := ts.makeNodes(t, OneFull, OneMiner)
+	apis, sn := ts.makeNodes(t, 1, oneMiner)
 	api := apis[0]
 
-	newHeads, err := api.ChainNotify(ctx)
-	require.NoError(t, err)
-	initHead := (<-newHeads)[0]
-	baseHeight := initHead.Val.Height()
-
 	h1, err := api.ChainHead(ctx)
 	require.NoError(t, err)
-	require.Equal(t, int64(h1.Height()), int64(baseHeight))
+	require.Equal(t, abi.ChainEpoch(0), h1.Height())
 
-	MineUntilBlock(ctx, t, apis[0], sn[0], nil)
+	newHeads, err := api.ChainNotify(ctx)
+	require.NoError(t, err)
+	<-newHeads
+
+	err = sn[0].MineOne(ctx, func(bool) {})
 	require.NoError(t, err)
 
 	<-newHeads
 
 	h2, err := api.ChainHead(ctx)
 	require.NoError(t, err)
-	require.Greater(t, int64(h2.Height()), int64(h1.Height()))
+	require.Equal(t, abi.ChainEpoch(1), h2.Height())
 }
 
 func (ts *testSuite) testMiningReal(t *testing.T) {
@@ -53,43 +51,45 @@ func (ts *testSuite) testMiningReal(t *testing.T) {
 	}()
 
 	ctx := context.Background()
-	apis, sn := ts.makeNodes(t, OneFull, OneMiner)
+	apis, sn := ts.makeNodes(t, 1, oneMiner)
 	api := apis[0]
 
-	newHeads, err := api.ChainNotify(ctx)
-	require.NoError(t, err)
-	at := (<-newHeads)[0].Val.Height()
-
 	h1, err := api.ChainHead(ctx)
 	require.NoError(t, err)
-	require.Equal(t, int64(at), int64(h1.Height()))
+	require.Equal(t, abi.ChainEpoch(0), h1.Height())
 
-	MineUntilBlock(ctx, t, apis[0], sn[0], nil)
+	newHeads, err := api.ChainNotify(ctx)
+	require.NoError(t, err)
+	<-newHeads
+
+	err = sn[0].MineOne(ctx, func(bool) {})
 	require.NoError(t, err)
 
 	<-newHeads
 
 	h2, err := api.ChainHead(ctx)
 	require.NoError(t, err)
-	require.Greater(t, int64(h2.Height()), int64(h1.Height()))
+	require.Equal(t, abi.ChainEpoch(1), h2.Height())
 
-	MineUntilBlock(ctx, t, apis[0], sn[0], nil)
+	err = sn[0].MineOne(ctx, func(bool) {})
 	require.NoError(t, err)
 
 	<-newHeads
 
-	h3, err := api.ChainHead(ctx)
+	h2, err = api.ChainHead(ctx)
 	require.NoError(t, err)
-	require.Greater(t, int64(h3.Height()), int64(h2.Height()))
+	require.Equal(t, abi.ChainEpoch(2), h2.Height())
 }
 
 func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExport bool) {
+	_ = os.Setenv("BELLMAN_NO_GPU", "1")
+
 	// test making a deal with a fresh miner, and see if it starts to mine
 
 	ctx := context.Background()
-	n, sn := b(t, OneFull, []StorageMiner{
+	n, sn := b(t, 1, []StorageMiner{
 		{Full: 0, Preseal: PresealGenesis},
-		{Full: 0, Preseal: 0}, // TODO: Add support for miners on non-first full node
+		{Full: 0, Preseal: 0}, // TODO: Add support for storage miners on non-first full node
 	})
 	client := n[0].FullNode.(*impl.FullNodeAPI)
 	provider := sn[1]
@@ -125,30 +125,27 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo
 	done := make(chan struct{})
 	minedTwo := make(chan struct{})
 
-	m2addr, err := sn[1].ActorAddress(context.TODO())
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	go func() {
 		defer close(done)
 
-		complChan := minedTwo
+		prevExpect := 0
 		for atomic.LoadInt32(&mine) != 0 {
-			wait := make(chan int)
-			mdone := func(mined bool, _ abi.ChainEpoch, err error) {
-				n := 0
-				if mined {
-					n = 1
-				}
-				wait <- n
+			wait := make(chan int, 2)
+			mdone := func(mined bool) {
+				go func() {
+					n := 0
+					if mined {
+						n = 1
+					}
+					wait <- n
+				}()
 			}
 
-			if err := sn[0].MineOne(ctx, miner.MineReq{Done: mdone}); err != nil {
+			if err := sn[0].MineOne(ctx, mdone); err != nil {
 				t.Error(err)
 			}
 
-			if err := sn[1].MineOne(ctx, miner.MineReq{Done: mdone}); err != nil {
+			if err := sn[1].MineOne(ctx, mdone); err != nil {
 				t.Error(err)
 			}
 
@@ -156,42 +153,43 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo
 			expect += <-wait
 
 			time.Sleep(blocktime)
-			if expect == 0 {
-				// null block
-				continue
-			}
 
-			var nodeOneMined bool
-			for _, node := range sn {
-				mb, err := node.MiningBase(ctx)
-				if err != nil {
-					t.Error(err)
-					return
-				}
-
-				for _, b := range mb.Blocks() {
-					if b.Miner == m2addr {
-						nodeOneMined = true
-						break
+			for {
+				n := 0
+				for i, node := range sn {
+					mb, err := node.MiningBase(ctx)
+					if err != nil {
+						t.Error(err)
+						return
 					}
+
+					if len(mb.Cids()) != expect {
+						log.Warnf("node %d mining base not complete (%d, want %d)", i, len(mb.Cids()), expect)
+						continue
+					}
+					n++
 				}
-
+				if n == len(sn) {
+					break
+				}
+				time.Sleep(blocktime)
 			}
 
-			if nodeOneMined && complChan != nil {
-				close(complChan)
-				complChan = nil
+			if prevExpect == 2 && expect == 2 && minedTwo != nil {
+				close(minedTwo)
+				minedTwo = nil
 			}
 
+			prevExpect = expect
 		}
 	}()
 
-	deal := startDeal(t, ctx, provider, client, fcid, false, 0)
+	deal := startDeal(t, ctx, provider, client, fcid)
 
 	// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
 	time.Sleep(time.Second)
 
-	waitDealSealed(t, ctx, provider, client, deal, false)
+	waitDealSealed(t, ctx, client, deal)
 
 	<-minedTwo
 

api/test/paych.go

@@ -1,312 +0,0 @@
-package test
-
-import (
-	"context"
-	"fmt"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/ipfs/go-cid"
-
-	"github.com/filecoin-project/go-address"
-	cbor "github.com/ipfs/go-ipld-cbor"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/blockstore"
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/actors/adt"
-	"github.com/filecoin-project/lotus/chain/actors/builtin"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
-	"github.com/filecoin-project/lotus/chain/actors/policy"
-	"github.com/filecoin-project/lotus/chain/events"
-	"github.com/filecoin-project/lotus/chain/events/state"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) {
-	ctx := context.Background()
-	n, sn := b(t, TwoFull, OneMiner)
-
-	paymentCreator := n[0]
-	paymentReceiver := n[1]
-	miner := sn[0]
-
-	// get everyone connected
-	addrs, err := paymentCreator.NetAddrsListen(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err := paymentReceiver.NetConnect(ctx, addrs); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := miner.NetConnect(ctx, addrs); err != nil {
-		t.Fatal(err)
-	}
-
-	// start mining blocks
-	bm := NewBlockMiner(ctx, t, miner, blocktime)
-	bm.MineBlocks()
-
-	// send some funds to register the receiver
-	receiverAddr, err := paymentReceiver.WalletNew(ctx, types.KTSecp256k1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	SendFunds(ctx, t, paymentCreator, receiverAddr, abi.NewTokenAmount(1e18))
-
-	// setup the payment channel
-	createrAddr, err := paymentCreator.WalletDefaultAddress(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	channelAmt := int64(7000)
-	channelInfo, err := paymentCreator.PaychGet(ctx, createrAddr, receiverAddr, abi.NewTokenAmount(channelAmt))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	channel, err := paymentCreator.PaychGetWaitReady(ctx, channelInfo.WaitSentinel)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// allocate three lanes
-	var lanes []uint64
-	for i := 0; i < 3; i++ {
-		lane, err := paymentCreator.PaychAllocateLane(ctx, channel)
-		if err != nil {
-			t.Fatal(err)
-		}
-		lanes = append(lanes, lane)
-	}
-
-	// Make two vouchers each for each lane, then save on the other side
-	// Note that the voucher with a value of 2000 has a higher nonce, so it
-	// supersedes the voucher with a value of 1000
-	for _, lane := range lanes {
-		vouch1, err := paymentCreator.PaychVoucherCreate(ctx, channel, abi.NewTokenAmount(1000), lane)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if vouch1.Voucher == nil {
-			t.Fatal(fmt.Errorf("Not enough funds to create voucher: missing %d", vouch1.Shortfall))
-		}
-		vouch2, err := paymentCreator.PaychVoucherCreate(ctx, channel, abi.NewTokenAmount(2000), lane)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if vouch2.Voucher == nil {
-			t.Fatal(fmt.Errorf("Not enough funds to create voucher: missing %d", vouch2.Shortfall))
-		}
-		delta1, err := paymentReceiver.PaychVoucherAdd(ctx, channel, vouch1.Voucher, nil, abi.NewTokenAmount(1000))
-		if err != nil {
-			t.Fatal(err)
-		}
-		if !delta1.Equals(abi.NewTokenAmount(1000)) {
-			t.Fatal("voucher didn't have the right amount")
-		}
-		delta2, err := paymentReceiver.PaychVoucherAdd(ctx, channel, vouch2.Voucher, nil, abi.NewTokenAmount(1000))
-		if err != nil {
-			t.Fatal(err)
-		}
-		if !delta2.Equals(abi.NewTokenAmount(1000)) {
-			t.Fatal("voucher didn't have the right amount")
-		}
-	}
-
-	// settle the payment channel
-	settleMsgCid, err := paymentCreator.PaychSettle(ctx, channel)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	res := waitForMessage(ctx, t, paymentCreator, settleMsgCid, time.Second*10, "settle")
-	if res.Receipt.ExitCode != 0 {
-		t.Fatal("Unable to settle payment channel")
-	}
-
-	creatorStore := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(paymentCreator)))
-
-	// wait for the receiver to submit their vouchers
-	ev := events.NewEvents(ctx, paymentCreator)
-	preds := state.NewStatePredicates(paymentCreator)
-	finished := make(chan struct{})
-	err = ev.StateChanged(func(ts *types.TipSet) (done bool, more bool, err error) {
-		act, err := paymentCreator.StateGetActor(ctx, channel, ts.Key())
-		if err != nil {
-			return false, false, err
-		}
-		state, err := paych.Load(creatorStore, act)
-		if err != nil {
-			return false, false, err
-		}
-		toSend, err := state.ToSend()
-		if err != nil {
-			return false, false, err
-		}
-		if toSend.GreaterThanEqual(abi.NewTokenAmount(6000)) {
-			return true, false, nil
-		}
-		return false, true, nil
-	}, func(oldTs, newTs *types.TipSet, states events.StateChange, curH abi.ChainEpoch) (more bool, err error) {
-		toSendChange := states.(*state.PayChToSendChange)
-		if toSendChange.NewToSend.GreaterThanEqual(abi.NewTokenAmount(6000)) {
-			close(finished)
-			return false, nil
-		}
-		return true, nil
-	}, func(ctx context.Context, ts *types.TipSet) error {
-		return nil
-	}, int(build.MessageConfidence)+1, build.Finality, func(oldTs, newTs *types.TipSet) (bool, events.StateChange, error) {
-		return preds.OnPaymentChannelActorChanged(channel, preds.OnToSendAmountChanges())(ctx, oldTs.Key(), newTs.Key())
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	select {
-	case <-finished:
-	case <-time.After(time.Second):
-		t.Fatal("Timed out waiting for receiver to submit vouchers")
-	}
-
-	// Create a new voucher now that some vouchers have already been submitted
-	vouchRes, err := paymentCreator.PaychVoucherCreate(ctx, channel, abi.NewTokenAmount(1000), 3)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if vouchRes.Voucher == nil {
-		t.Fatal(fmt.Errorf("Not enough funds to create voucher: missing %d", vouchRes.Shortfall))
-	}
-	vdelta, err := paymentReceiver.PaychVoucherAdd(ctx, channel, vouchRes.Voucher, nil, abi.NewTokenAmount(1000))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !vdelta.Equals(abi.NewTokenAmount(1000)) {
-		t.Fatal("voucher didn't have the right amount")
-	}
-
-	// Create a new voucher whose value would exceed the channel balance
-	excessAmt := abi.NewTokenAmount(1000)
-	vouchRes, err = paymentCreator.PaychVoucherCreate(ctx, channel, excessAmt, 4)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if vouchRes.Voucher != nil {
-		t.Fatal("Expected not to be able to create voucher whose value would exceed channel balance")
-	}
-	if !vouchRes.Shortfall.Equals(excessAmt) {
-		t.Fatal(fmt.Errorf("Expected voucher shortfall of %d, got %d", excessAmt, vouchRes.Shortfall))
-	}
-
-	// Add a voucher whose value would exceed the channel balance
-	vouch := &paych.SignedVoucher{ChannelAddr: channel, Amount: excessAmt, Lane: 4, Nonce: 1}
-	vb, err := vouch.SigningBytes()
-	if err != nil {
-		t.Fatal(err)
-	}
-	sig, err := paymentCreator.WalletSign(ctx, createrAddr, vb)
-	if err != nil {
-		t.Fatal(err)
-	}
-	vouch.Signature = sig
-	_, err = paymentReceiver.PaychVoucherAdd(ctx, channel, vouch, nil, abi.NewTokenAmount(1000))
-	if err == nil {
-		t.Fatal(fmt.Errorf("Expected shortfall error of %d", excessAmt))
-	}
-
-	// wait for the settlement period to pass before collecting
-	waitForBlocks(ctx, t, bm, paymentReceiver, receiverAddr, policy.PaychSettleDelay)
-
-	creatorPreCollectBalance, err := paymentCreator.WalletBalance(ctx, createrAddr)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// collect funds (from receiver, though either party can do it)
-	collectMsg, err := paymentReceiver.PaychCollect(ctx, channel)
-	if err != nil {
-		t.Fatal(err)
-	}
-	res, err = paymentReceiver.StateWaitMsg(ctx, collectMsg, 3)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if res.Receipt.ExitCode != 0 {
-		t.Fatal("unable to collect on payment channel")
-	}
-
-	// Finally, check the balance for the creator
-	currentCreatorBalance, err := paymentCreator.WalletBalance(ctx, createrAddr)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// The highest nonce voucher that the creator sent on each lane is 2000
-	totalVouchers := int64(len(lanes) * 2000)
-
-	// When receiver submits the tokens to the chain, creator should get a
-	// refund on the remaining balance, which is
-	// channel amount - total voucher value
-	expectedRefund := channelAmt - totalVouchers
-	delta := big.Sub(currentCreatorBalance, creatorPreCollectBalance)
-	if !delta.Equals(abi.NewTokenAmount(expectedRefund)) {
-		t.Fatalf("did not send correct funds from creator: expected %d, got %d", expectedRefund, delta)
-	}
-
-	// shut down mining
-	bm.Stop()
-}
-
-func waitForBlocks(ctx context.Context, t *testing.T, bm *BlockMiner, paymentReceiver TestNode, receiverAddr address.Address, count int) {
-	// We need to add null blocks in batches, if we add too many the chain can't sync
-	batchSize := 60
-	for i := 0; i < count; i += batchSize {
-		size := batchSize
-		if i > count {
-			size = count - i
-		}
-
-		// Add a batch of null blocks
-		atomic.StoreInt64(&bm.nulls, int64(size-1))
-
-		// Add a real block
-		m, err := paymentReceiver.MpoolPushMessage(ctx, &types.Message{
-			To:    builtin.BurntFundsActorAddr,
-			From:  receiverAddr,
-			Value: types.NewInt(0),
-		}, nil)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		_, err = paymentReceiver.StateWaitMsg(ctx, m.Cid(), 1)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
-func waitForMessage(ctx context.Context, t *testing.T, paymentCreator TestNode, msgCid cid.Cid, duration time.Duration, desc string) *api.MsgLookup {
-	ctx, cancel := context.WithTimeout(ctx, duration)
-	defer cancel()
-
-	fmt.Println("Waiting for", desc)
-	res, err := paymentCreator.StateWaitMsg(ctx, msgCid, 1)
-	if err != nil {
-		fmt.Println("Error waiting for", desc, err)
-		t.Fatal(err)
-	}
-	if res.Receipt.ExitCode != 0 {
-		t.Fatalf("did not successfully send %s", desc)
-	}
-	fmt.Println("Confirmed", desc)
-	return res
-}

api/test/tape.go

@@ -1,101 +0,0 @@
-package test
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/filecoin-project/go-state-types/network"
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/stmgr"
-	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
-	"github.com/filecoin-project/lotus/node"
-	"github.com/filecoin-project/lotus/node/impl"
-	"github.com/stretchr/testify/require"
-)
-
-func TestTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration) {
-	// The "before" case is disabled, because we need the builder to mock 32 GiB sectors to accurately repro this case
-	// TODO: Make the mock sector size configurable and reenable this
-	//t.Run("before", func(t *testing.T) { testTapeFix(t, b, blocktime, false) })
-	t.Run("after", func(t *testing.T) { testTapeFix(t, b, blocktime, true) })
-}
-func testTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration, after bool) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	upgradeSchedule := stmgr.UpgradeSchedule{{
-		Network:   build.ActorUpgradeNetworkVersion,
-		Height:    1,
-		Migration: stmgr.UpgradeActorsV2,
-	}}
-	if after {
-		upgradeSchedule = append(upgradeSchedule, stmgr.Upgrade{
-			Network: network.Version5,
-			Height:  2,
-		})
-	}
-
-	n, sn := b(t, []FullNodeOpts{{Opts: func(_ []TestNode) node.Option {
-		return node.Override(new(stmgr.UpgradeSchedule), upgradeSchedule)
-	}}}, OneMiner)
-
-	client := n[0].FullNode.(*impl.FullNodeAPI)
-	miner := sn[0]
-
-	addrinfo, err := client.NetAddrsListen(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err := miner.NetConnect(ctx, addrinfo); err != nil {
-		t.Fatal(err)
-	}
-	build.Clock.Sleep(time.Second)
-
-	done := make(chan struct{})
-	go func() {
-		defer close(done)
-		for ctx.Err() == nil {
-			build.Clock.Sleep(blocktime)
-			if err := sn[0].MineOne(ctx, MineNext); err != nil {
-				if ctx.Err() != nil {
-					// context was canceled, ignore the error.
-					return
-				}
-				t.Error(err)
-			}
-		}
-	}()
-	defer func() {
-		cancel()
-		<-done
-	}()
-
-	sid, err := miner.PledgeSector(ctx)
-	require.NoError(t, err)
-
-	fmt.Printf("All sectors is fsm\n")
-
-	// If before, we expect the precommit to fail
-	successState := api.SectorState(sealing.CommitFailed)
-	failureState := api.SectorState(sealing.Proving)
-	if after {
-		// otherwise, it should succeed.
-		successState, failureState = failureState, successState
-	}
-
-	for {
-		st, err := miner.SectorsStatus(ctx, sid.Number, false)
-		require.NoError(t, err)
-		if st.State == successState {
-			break
-		}
-		require.NotEqual(t, failureState, st.State)
-		build.Clock.Sleep(100 * time.Millisecond)
-		fmt.Println("WaitSeal")
-	}
-
-}

api/test/test.go

@@ -2,82 +2,40 @@ package test
 
 import (
 	"context"
-	"fmt"
-	"os"
-	"strings"
 	"testing"
-	"time"
-
-	"github.com/filecoin-project/lotus/chain/stmgr"
-	"github.com/filecoin-project/lotus/chain/types"
-
-	logging "github.com/ipfs/go-log/v2"
-	"github.com/multiformats/go-multiaddr"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/go-state-types/network"
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/miner"
-	"github.com/filecoin-project/lotus/node"
 )
 
-func init() {
-	logging.SetAllLoggers(logging.LevelInfo)
-	err := os.Setenv("BELLMAN_NO_GPU", "1")
-	if err != nil {
-		panic(fmt.Sprintf("failed to set BELLMAN_NO_GPU env variable: %s", err))
-	}
-	build.InsecurePoStValidation = true
-}
-
 type TestNode struct {
 	api.FullNode
-	// ListenAddr is the address on which an API server is listening, if an
-	// API server is created for this Node
-	ListenAddr multiaddr.Multiaddr
 }
 
 type TestStorageNode struct {
 	api.StorageMiner
-	// ListenAddr is the address on which an API server is listening, if an
-	// API server is created for this Node
-	ListenAddr multiaddr.Multiaddr
 
-	MineOne func(context.Context, miner.MineReq) error
-	Stop    func(context.Context) error
+	MineOne func(context.Context, func(bool)) error
 }
 
 var PresealGenesis = -1
 
 const GenesisPreseals = 2
 
-// Options for setting up a mock storage miner
 type StorageMiner struct {
 	Full    int
-	Opts    node.Option
 	Preseal int
 }
 
-type OptionGenerator func([]TestNode) node.Option
-
-// Options for setting up a mock full node
-type FullNodeOpts struct {
-	Lite bool            // run node in "lite" mode
-	Opts OptionGenerator // generate dependency injection options
-}
-
 // APIBuilder is a function which is invoked in test suite to provide
 // test nodes and networks
 //
-// fullOpts array defines options for each full node
 // storage array defines storage nodes, numbers in the array specify full node
 // index the storage node 'belongs' to
-type APIBuilder func(t *testing.T, full []FullNodeOpts, storage []StorageMiner) ([]TestNode, []TestStorageNode)
+type APIBuilder func(t *testing.T, nFull int, storage []StorageMiner) ([]TestNode, []TestStorageNode)
 type testSuite struct {
 	makeNodes APIBuilder
 }
@@ -93,131 +51,25 @@ func TestApis(t *testing.T, b APIBuilder) {
 	t.Run("testConnectTwo", ts.testConnectTwo)
 	t.Run("testMining", ts.testMining)
 	t.Run("testMiningReal", ts.testMiningReal)
-	t.Run("testSearchMsg", ts.testSearchMsg)
 }
 
-func DefaultFullOpts(nFull int) []FullNodeOpts {
-	full := make([]FullNodeOpts, nFull)
-	for i := range full {
-		full[i] = FullNodeOpts{
-			Opts: func(nodes []TestNode) node.Option {
-				return node.Options()
-			},
-		}
-	}
-	return full
-}
-
-var OneMiner = []StorageMiner{{Full: 0, Preseal: PresealGenesis}}
-var OneFull = DefaultFullOpts(1)
-var TwoFull = DefaultFullOpts(2)
-
-var FullNodeWithActorsV3At = func(upgradeHeight abi.ChainEpoch) FullNodeOpts {
-	return FullNodeOpts{
-		Opts: func(nodes []TestNode) node.Option {
-			return node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{
-				// prepare for upgrade.
-				Network:   network.Version9,
-				Height:    1,
-				Migration: stmgr.UpgradeActorsV2,
-			}, {
-				// Skip directly to tape height so precommits work.
-				Network:   network.Version10,
-				Height:    upgradeHeight,
-				Migration: stmgr.UpgradeActorsV3,
-			}})
-		},
-	}
-}
-
-var FullNodeWithSDRAt = func(calico, persian abi.ChainEpoch) FullNodeOpts {
-	return FullNodeOpts{
-		Opts: func(nodes []TestNode) node.Option {
-			return node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{
-				Network:   network.Version6,
-				Height:    1,
-				Migration: stmgr.UpgradeActorsV2,
-			}, {
-				Network:   network.Version7,
-				Height:    calico,
-				Migration: stmgr.UpgradeCalico,
-			}, {
-				Network: network.Version8,
-				Height:  persian,
-			}})
-		},
-	}
-}
-
-var MineNext = miner.MineReq{
-	InjectNulls: 0,
-	Done:        func(bool, abi.ChainEpoch, error) {},
-}
+var oneMiner = []StorageMiner{{Full: 0, Preseal: PresealGenesis}}
 
 func (ts *testSuite) testVersion(t *testing.T) {
-	api.RunningNodeType = api.NodeFull
-
 	ctx := context.Background()
-	apis, _ := ts.makeNodes(t, OneFull, OneMiner)
-	napi := apis[0]
-
-	v, err := napi.Version(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	versions := strings.Split(v.Version, "+")
-	if len(versions) <= 0 {
-		t.Fatal("empty version")
-	}
-	require.Equal(t, versions[0], build.BuildVersion)
-}
-
-func (ts *testSuite) testSearchMsg(t *testing.T) {
-	apis, miners := ts.makeNodes(t, OneFull, OneMiner)
-
+	apis, _ := ts.makeNodes(t, 1, oneMiner)
 	api := apis[0]
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	senderAddr, err := api.WalletDefaultAddress(ctx)
+
+	v, err := api.Version(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}
-
-	msg := &types.Message{
-		From:  senderAddr,
-		To:    senderAddr,
-		Value: big.Zero(),
-	}
-	bm := NewBlockMiner(ctx, t, miners[0], 100*time.Millisecond)
-	bm.MineBlocks()
-	defer bm.Stop()
-
-	sm, err := api.MpoolPushMessage(ctx, msg, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	res, err := api.StateWaitMsg(ctx, sm.Cid(), 1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if res.Receipt.ExitCode != 0 {
-		t.Fatal("did not successfully send message")
-	}
-
-	searchRes, err := api.StateSearchMsg(ctx, sm.Cid())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if searchRes.TipSet != res.TipSet {
-		t.Fatalf("search ts: %s, different from wait ts: %s", searchRes.TipSet, res.TipSet)
-	}
-
+	require.Equal(t, v.Version, build.BuildVersion)
 }
 
 func (ts *testSuite) testID(t *testing.T) {
 	ctx := context.Background()
-	apis, _ := ts.makeNodes(t, OneFull, OneMiner)
+	apis, _ := ts.makeNodes(t, 1, oneMiner)
 	api := apis[0]
 
 	id, err := api.ID(ctx)
@@ -229,7 +81,7 @@ func (ts *testSuite) testID(t *testing.T) {
 
 func (ts *testSuite) testConnectTwo(t *testing.T) {
 	ctx := context.Background()
-	apis, _ := ts.makeNodes(t, TwoFull, OneMiner)
+	apis, _ := ts.makeNodes(t, 2, oneMiner)
 
 	p, err := apis[0].NetPeers(ctx)
 	if err != nil {

api/test/util.go

@@ -1,86 +0,0 @@
-package test
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"github.com/filecoin-project/go-state-types/abi"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/miner"
-)
-
-func SendFunds(ctx context.Context, t *testing.T, sender TestNode, addr address.Address, amount abi.TokenAmount) {
-	senderAddr, err := sender.WalletDefaultAddress(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	msg := &types.Message{
-		From:  senderAddr,
-		To:    addr,
-		Value: amount,
-	}
-
-	sm, err := sender.MpoolPushMessage(ctx, msg, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	res, err := sender.StateWaitMsg(ctx, sm.Cid(), 1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if res.Receipt.ExitCode != 0 {
-		t.Fatal("did not successfully send money")
-	}
-}
-
-func MineUntilBlock(ctx context.Context, t *testing.T, fn TestNode, sn TestStorageNode, cb func(abi.ChainEpoch)) {
-	for i := 0; i < 1000; i++ {
-		var success bool
-		var err error
-		var epoch abi.ChainEpoch
-		wait := make(chan struct{})
-		mineErr := sn.MineOne(ctx, miner.MineReq{
-			Done: func(win bool, ep abi.ChainEpoch, e error) {
-				success = win
-				err = e
-				epoch = ep
-				wait <- struct{}{}
-			},
-		})
-		if mineErr != nil {
-			t.Fatal(mineErr)
-		}
-		<-wait
-		if err != nil {
-			t.Fatal(err)
-		}
-		if success {
-			// Wait until it shows up on the given full nodes ChainHead
-			nloops := 50
-			for i := 0; i < nloops; i++ {
-				ts, err := fn.ChainHead(ctx)
-				if err != nil {
-					t.Fatal(err)
-				}
-				if ts.Height() == epoch {
-					break
-				}
-				if i == nloops-1 {
-					t.Fatal("block never managed to sync to node")
-				}
-				time.Sleep(time.Millisecond * 10)
-			}
-
-			if cb != nil {
-				cb(epoch)
-			}
-			return
-		}
-		t.Log("did not mine block, trying again", i)
-	}
-	t.Fatal("failed to mine 1000 times in a row...")
-}

api/types.go

@@ -2,15 +2,8 @@ package api
 
 import (
 	"encoding/json"
-	"fmt"
-	"time"
-
-	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/ipfs/go-cid"
 
 	"github.com/libp2p/go-libp2p-core/peer"
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	ma "github.com/multiformats/go-multiaddr"
 )
 
@@ -44,74 +37,5 @@ type ObjStat struct {
 
 type PubsubScore struct {
 	ID    peer.ID
-	Score *pubsub.PeerScoreSnapshot
-}
-
-type MessageSendSpec struct {
-	MaxFee abi.TokenAmount
-}
-
-type DataTransferChannel struct {
-	TransferID  datatransfer.TransferID
-	Status      datatransfer.Status
-	BaseCID     cid.Cid
-	IsInitiator bool
-	IsSender    bool
-	Voucher     string
-	Message     string
-	OtherPeer   peer.ID
-	Transferred uint64
-}
-
-// NewDataTransferChannel constructs an API DataTransferChannel type from full channel state snapshot and a host id
-func NewDataTransferChannel(hostID peer.ID, channelState datatransfer.ChannelState) DataTransferChannel {
-	channel := DataTransferChannel{
-		TransferID: channelState.TransferID(),
-		Status:     channelState.Status(),
-		BaseCID:    channelState.BaseCID(),
-		IsSender:   channelState.Sender() == hostID,
-		Message:    channelState.Message(),
-	}
-	stringer, ok := channelState.Voucher().(fmt.Stringer)
-	if ok {
-		channel.Voucher = stringer.String()
-	} else {
-		voucherJSON, err := json.Marshal(channelState.Voucher())
-		if err != nil {
-			channel.Voucher = fmt.Errorf("Voucher Serialization: %w", err).Error()
-		} else {
-			channel.Voucher = string(voucherJSON)
-		}
-	}
-	if channel.IsSender {
-		channel.IsInitiator = !channelState.IsPull()
-		channel.Transferred = channelState.Sent()
-		channel.OtherPeer = channelState.Recipient()
-	} else {
-		channel.IsInitiator = channelState.IsPull()
-		channel.Transferred = channelState.Received()
-		channel.OtherPeer = channelState.Sender()
-	}
-	return channel
-}
-
-type NetBlockList struct {
-	Peers     []peer.ID
-	IPAddrs   []string
-	IPSubnets []string
-}
-
-type ExtendedPeerInfo struct {
-	ID          peer.ID
-	Agent       string
-	Addrs       []string
-	Protocols   []string
-	ConnMgrMeta *ConnMgrInfo
-}
-
-type ConnMgrInfo struct {
-	FirstSeen time.Time
-	Value     int
-	Tags      map[string]int
-	Conns     map[string]time.Time
+	Score float64
 }

api/utils.go

@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/specs-actors/actors/crypto"
 )
 
 type SignFunc = func(context.Context, []byte) (*crypto.Signature, error)

api/version.go

@@ -1,71 +0,0 @@
-package api
-
-import (
-	"fmt"
-
-	xerrors "golang.org/x/xerrors"
-)
-
-type Version uint32
-
-func newVer(major, minor, patch uint8) Version {
-	return Version(uint32(major)<<16 | uint32(minor)<<8 | uint32(patch))
-}
-
-// Ints returns (major, minor, patch) versions
-func (ve Version) Ints() (uint32, uint32, uint32) {
-	v := uint32(ve)
-	return (v & majorOnlyMask) >> 16, (v & minorOnlyMask) >> 8, v & patchOnlyMask
-}
-
-func (ve Version) String() string {
-	vmj, vmi, vp := ve.Ints()
-	return fmt.Sprintf("%d.%d.%d", vmj, vmi, vp)
-}
-
-func (ve Version) EqMajorMinor(v2 Version) bool {
-	return ve&minorMask == v2&minorMask
-}
-
-type NodeType int
-
-const (
-	NodeUnknown NodeType = iota
-
-	NodeFull
-	NodeMiner
-	NodeWorker
-)
-
-var RunningNodeType NodeType
-
-func VersionForType(nodeType NodeType) (Version, error) {
-	switch nodeType {
-	case NodeFull:
-		return FullAPIVersion, nil
-	case NodeMiner:
-		return MinerAPIVersion, nil
-	case NodeWorker:
-		return WorkerAPIVersion, nil
-	default:
-		return Version(0), xerrors.Errorf("unknown node type %d", nodeType)
-	}
-}
-
-// semver versions of the rpc api exposed
-var (
-	FullAPIVersion   = newVer(1, 1, 0)
-	MinerAPIVersion  = newVer(1, 0, 1)
-	WorkerAPIVersion = newVer(1, 0, 0)
-)
-
-//nolint:varcheck,deadcode
-const (
-	majorMask = 0xff0000
-	minorMask = 0xffff00
-	patchMask = 0xffffff
-
-	majorOnlyMask = 0xff0000
-	minorOnlyMask = 0x00ff00
-	patchOnlyMask = 0x0000ff
-)

blockstore/api.go

@@ -1,66 +0,0 @@
-package blockstore
-
-import (
-	"context"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	"golang.org/x/xerrors"
-)
-
-type ChainIO interface {
-	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
-	ChainHasObj(context.Context, cid.Cid) (bool, error)
-}
-
-type apiBlockstore struct {
-	api ChainIO
-}
-
-// This blockstore is adapted in the constructor.
-var _ BasicBlockstore = (*apiBlockstore)(nil)
-
-func NewAPIBlockstore(cio ChainIO) Blockstore {
-	bs := &apiBlockstore{api: cio}
-	return Adapt(bs) // return an adapted blockstore.
-}
-
-func (a *apiBlockstore) DeleteBlock(cid.Cid) error {
-	return xerrors.New("not supported")
-}
-
-func (a *apiBlockstore) Has(c cid.Cid) (bool, error) {
-	return a.api.ChainHasObj(context.TODO(), c)
-}
-
-func (a *apiBlockstore) Get(c cid.Cid) (blocks.Block, error) {
-	bb, err := a.api.ChainReadObj(context.TODO(), c)
-	if err != nil {
-		return nil, err
-	}
-	return blocks.NewBlockWithCid(bb, c)
-}
-
-func (a *apiBlockstore) GetSize(c cid.Cid) (int, error) {
-	bb, err := a.api.ChainReadObj(context.TODO(), c)
-	if err != nil {
-		return 0, err
-	}
-	return len(bb), nil
-}
-
-func (a *apiBlockstore) Put(blocks.Block) error {
-	return xerrors.New("not supported")
-}
-
-func (a *apiBlockstore) PutMany([]blocks.Block) error {
-	return xerrors.New("not supported")
-}
-
-func (a *apiBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return nil, xerrors.New("not supported")
-}
-
-func (a *apiBlockstore) HashOnRead(enabled bool) {
-	return
-}

blockstore/badger/blockstore.go

@@ -1,481 +0,0 @@
-package badgerbs
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"sync/atomic"
-
-	"github.com/dgraph-io/badger/v2"
-	"github.com/dgraph-io/badger/v2/options"
-	"github.com/multiformats/go-base32"
-	"go.uber.org/zap"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	logger "github.com/ipfs/go-log/v2"
-	pool "github.com/libp2p/go-buffer-pool"
-
-	"github.com/filecoin-project/lotus/blockstore"
-)
-
-var (
-	// KeyPool is the buffer pool we use to compute storage keys.
-	KeyPool *pool.BufferPool = pool.GlobalPool
-)
-
-var (
-	// ErrBlockstoreClosed is returned from blockstore operations after
-	// the blockstore has been closed.
-	ErrBlockstoreClosed = fmt.Errorf("badger blockstore closed")
-
-	log = logger.Logger("badgerbs")
-)
-
-// aliases to mask badger dependencies.
-const (
-	// FileIO is equivalent to badger/options.FileIO.
-	FileIO = options.FileIO
-	// MemoryMap is equivalent to badger/options.MemoryMap.
-	MemoryMap = options.MemoryMap
-	// LoadToRAM is equivalent to badger/options.LoadToRAM.
-	LoadToRAM = options.LoadToRAM
-)
-
-// Options embeds the badger options themselves, and augments them with
-// blockstore-specific options.
-type Options struct {
-	badger.Options
-
-	// Prefix is an optional prefix to prepend to keys. Default: "".
-	Prefix string
-}
-
-func DefaultOptions(path string) Options {
-	return Options{
-		Options: badger.DefaultOptions(path),
-		Prefix:  "",
-	}
-}
-
-// badgerLogger is a local wrapper for go-log to make the interface
-// compatible with badger.Logger (namely, aliasing Warnf to Warningf)
-type badgerLogger struct {
-	*zap.SugaredLogger // skips 1 caller to get useful line info, skipping over badger.Options.
-
-	skip2 *zap.SugaredLogger // skips 2 callers, just like above + this logger.
-}
-
-// Warningf is required by the badger logger APIs.
-func (b *badgerLogger) Warningf(format string, args ...interface{}) {
-	b.skip2.Warnf(format, args...)
-}
-
-const (
-	stateOpen int64 = iota
-	stateClosing
-	stateClosed
-)
-
-// Blockstore is a badger-backed IPLD blockstore.
-//
-// NOTE: once Close() is called, methods will try their best to return
-// ErrBlockstoreClosed. This will guaranteed to happen for all subsequent
-// operation calls after Close() has returned, but it may not happen for
-// operations in progress. Those are likely to fail with a different error.
-type Blockstore struct {
-	DB *badger.DB
-
-	// state is guarded by atomic.
-	state int64
-
-	prefixing bool
-	prefix    []byte
-	prefixLen int
-}
-
-var _ blockstore.Blockstore = (*Blockstore)(nil)
-var _ blockstore.Viewer = (*Blockstore)(nil)
-var _ io.Closer = (*Blockstore)(nil)
-
-// Open creates a new badger-backed blockstore, with the supplied options.
-func Open(opts Options) (*Blockstore, error) {
-	opts.Logger = &badgerLogger{
-		SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(),
-		skip2:         log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(),
-	}
-
-	db, err := badger.Open(opts.Options)
-	if err != nil {
-		return nil, fmt.Errorf("failed to open badger blockstore: %w", err)
-	}
-
-	bs := &Blockstore{DB: db}
-	if p := opts.Prefix; p != "" {
-		bs.prefixing = true
-		bs.prefix = []byte(p)
-		bs.prefixLen = len(bs.prefix)
-	}
-
-	return bs, nil
-}
-
-// Close closes the store. If the store has already been closed, this noops and
-// returns an error, even if the first closure resulted in error.
-func (b *Blockstore) Close() error {
-	if !atomic.CompareAndSwapInt64(&b.state, stateOpen, stateClosing) {
-		return nil
-	}
-
-	defer atomic.StoreInt64(&b.state, stateClosed)
-	return b.DB.Close()
-}
-
-// CollectGarbage runs garbage collection on the value log
-func (b *Blockstore) CollectGarbage() error {
-	if atomic.LoadInt64(&b.state) != stateOpen {
-		return ErrBlockstoreClosed
-	}
-
-	var err error
-	for err == nil {
-		err = b.DB.RunValueLogGC(0.125)
-	}
-
-	if err == badger.ErrNoRewrite {
-		// not really an error in this case
-		return nil
-	}
-
-	return err
-}
-
-// View implements blockstore.Viewer, which leverages zero-copy read-only
-// access to values.
-func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
-	if atomic.LoadInt64(&b.state) != stateOpen {
-		return ErrBlockstoreClosed
-	}
-
-	k, pooled := b.PooledStorageKey(cid)
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	return b.DB.View(func(txn *badger.Txn) error {
-		switch item, err := txn.Get(k); err {
-		case nil:
-			return item.Value(fn)
-		case badger.ErrKeyNotFound:
-			return blockstore.ErrNotFound
-		default:
-			return fmt.Errorf("failed to view block from badger blockstore: %w", err)
-		}
-	})
-}
-
-// Has implements Blockstore.Has.
-func (b *Blockstore) Has(cid cid.Cid) (bool, error) {
-	if atomic.LoadInt64(&b.state) != stateOpen {
-		return false, ErrBlockstoreClosed
-	}
-
-	k, pooled := b.PooledStorageKey(cid)
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	err := b.DB.View(func(txn *badger.Txn) error {
-		_, err := txn.Get(k)
-		return err
-	})
-
-	switch err {
-	case badger.ErrKeyNotFound:
-		return false, nil
-	case nil:
-		return true, nil
-	default:
-		return false, fmt.Errorf("failed to check if block exists in badger blockstore: %w", err)
-	}
-}
-
-// Get implements Blockstore.Get.
-func (b *Blockstore) Get(cid cid.Cid) (blocks.Block, error) {
-	if !cid.Defined() {
-		return nil, blockstore.ErrNotFound
-	}
-
-	if atomic.LoadInt64(&b.state) != stateOpen {
-		return nil, ErrBlockstoreClosed
-	}
-
-	k, pooled := b.PooledStorageKey(cid)
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	var val []byte
-	err := b.DB.View(func(txn *badger.Txn) error {
-		switch item, err := txn.Get(k); err {
-		case nil:
-			val, err = item.ValueCopy(nil)
-			return err
-		case badger.ErrKeyNotFound:
-			return blockstore.ErrNotFound
-		default:
-			return fmt.Errorf("failed to get block from badger blockstore: %w", err)
-		}
-	})
-	if err != nil {
-		return nil, err
-	}
-	return blocks.NewBlockWithCid(val, cid)
-}
-
-// GetSize implements Blockstore.GetSize.
-func (b *Blockstore) GetSize(cid cid.Cid) (int, error) {
-	if atomic.LoadInt64(&b.state) != stateOpen {
-		return -1, ErrBlockstoreClosed
-	}
-
-	k, pooled := b.PooledStorageKey(cid)
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	var size int
-	err := b.DB.View(func(txn *badger.Txn) error {
-		switch item, err := txn.Get(k); err {
-		case nil:
-			size = int(item.ValueSize())
-		case badger.ErrKeyNotFound:
-			return blockstore.ErrNotFound
-		default:
-			return fmt.Errorf("failed to get block size from badger blockstore: %w", err)
-		}
-		return nil
-	})
-	if err != nil {
-		size = -1
-	}
-	return size, err
-}
-
-// Put implements Blockstore.Put.
-func (b *Blockstore) Put(block blocks.Block) error {
-	if atomic.LoadInt64(&b.state) != stateOpen {
-		return ErrBlockstoreClosed
-	}
-
-	k, pooled := b.PooledStorageKey(block.Cid())
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	err := b.DB.Update(func(txn *badger.Txn) error {
-		return txn.Set(k, block.RawData())
-	})
-	if err != nil {
-		err = fmt.Errorf("failed to put block in badger blockstore: %w", err)
-	}
-	return err
-}
-
-// PutMany implements Blockstore.PutMany.
-func (b *Blockstore) PutMany(blocks []blocks.Block) error {
-	if atomic.LoadInt64(&b.state) != stateOpen {
-		return ErrBlockstoreClosed
-	}
-
-	batch := b.DB.NewWriteBatch()
-	defer batch.Cancel()
-
-	// toReturn tracks the byte slices to return to the pool, if we're using key
-	// prefixing. we can't return each slice to the pool after each Set, because
-	// badger holds on to the slice.
-	var toReturn [][]byte
-	if b.prefixing {
-		toReturn = make([][]byte, 0, len(blocks))
-		defer func() {
-			for _, b := range toReturn {
-				KeyPool.Put(b)
-			}
-		}()
-	}
-
-	for _, block := range blocks {
-		k, pooled := b.PooledStorageKey(block.Cid())
-		if pooled {
-			toReturn = append(toReturn, k)
-		}
-		if err := batch.Set(k, block.RawData()); err != nil {
-			return err
-		}
-	}
-
-	err := batch.Flush()
-	if err != nil {
-		err = fmt.Errorf("failed to put blocks in badger blockstore: %w", err)
-	}
-	return err
-}
-
-// DeleteBlock implements Blockstore.DeleteBlock.
-func (b *Blockstore) DeleteBlock(cid cid.Cid) error {
-	if atomic.LoadInt64(&b.state) != stateOpen {
-		return ErrBlockstoreClosed
-	}
-
-	k, pooled := b.PooledStorageKey(cid)
-	if pooled {
-		defer KeyPool.Put(k)
-	}
-
-	return b.DB.Update(func(txn *badger.Txn) error {
-		return txn.Delete(k)
-	})
-}
-
-func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
-	if atomic.LoadInt64(&b.state) != stateOpen {
-		return ErrBlockstoreClosed
-	}
-
-	batch := b.DB.NewWriteBatch()
-	defer batch.Cancel()
-
-	// toReturn tracks the byte slices to return to the pool, if we're using key
-	// prefixing. we can't return each slice to the pool after each Set, because
-	// badger holds on to the slice.
-	var toReturn [][]byte
-	if b.prefixing {
-		toReturn = make([][]byte, 0, len(cids))
-		defer func() {
-			for _, b := range toReturn {
-				KeyPool.Put(b)
-			}
-		}()
-	}
-
-	for _, cid := range cids {
-		k, pooled := b.PooledStorageKey(cid)
-		if pooled {
-			toReturn = append(toReturn, k)
-		}
-		if err := batch.Delete(k); err != nil {
-			return err
-		}
-	}
-
-	err := batch.Flush()
-	if err != nil {
-		err = fmt.Errorf("failed to delete blocks from badger blockstore: %w", err)
-	}
-	return err
-}
-
-// AllKeysChan implements Blockstore.AllKeysChan.
-func (b *Blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	if atomic.LoadInt64(&b.state) != stateOpen {
-		return nil, ErrBlockstoreClosed
-	}
-
-	txn := b.DB.NewTransaction(false)
-	opts := badger.IteratorOptions{PrefetchSize: 100}
-	if b.prefixing {
-		opts.Prefix = b.prefix
-	}
-	iter := txn.NewIterator(opts)
-
-	ch := make(chan cid.Cid)
-	go func() {
-		defer close(ch)
-		defer iter.Close()
-
-		// NewCidV1 makes a copy of the multihash buffer, so we can reuse it to
-		// contain allocs.
-		var buf []byte
-		for iter.Rewind(); iter.Valid(); iter.Next() {
-			if ctx.Err() != nil {
-				return // context has fired.
-			}
-			if atomic.LoadInt64(&b.state) != stateOpen {
-				// open iterators will run even after the database is closed...
-				return // closing, yield.
-			}
-			k := iter.Item().Key()
-			if b.prefixing {
-				k = k[b.prefixLen:]
-			}
-
-			if reqlen := base32.RawStdEncoding.DecodedLen(len(k)); len(buf) < reqlen {
-				buf = make([]byte, reqlen)
-			}
-			if n, err := base32.RawStdEncoding.Decode(buf, k); err == nil {
-				select {
-				case ch <- cid.NewCidV1(cid.Raw, buf[:n]):
-				case <-ctx.Done():
-					return
-				}
-			} else {
-				log.Warnf("failed to decode key %s in badger AllKeysChan; err: %s", k, err)
-			}
-		}
-	}()
-
-	return ch, nil
-}
-
-// HashOnRead implements Blockstore.HashOnRead. It is not supported by this
-// blockstore.
-func (b *Blockstore) HashOnRead(_ bool) {
-	log.Warnf("called HashOnRead on badger blockstore; function not supported; ignoring")
-}
-
-// PooledStorageKey returns the storage key under which this CID is stored.
-//
-// The key is: prefix + base32_no_padding(cid.Hash)
-//
-// This method may return pooled byte slice, which MUST be returned to the
-// KeyPool if pooled=true, or a leak will occur.
-func (b *Blockstore) PooledStorageKey(cid cid.Cid) (key []byte, pooled bool) {
-	h := cid.Hash()
-	size := base32.RawStdEncoding.EncodedLen(len(h))
-	if !b.prefixing { // optimize for branch prediction.
-		k := pool.Get(size)
-		base32.RawStdEncoding.Encode(k, h)
-		return k, true // slicing upto length unnecessary; the pool has already done this.
-	}
-
-	size += b.prefixLen
-	k := pool.Get(size)
-	copy(k, b.prefix)
-	base32.RawStdEncoding.Encode(k[b.prefixLen:], h)
-	return k, true // slicing upto length unnecessary; the pool has already done this.
-}
-
-// Storage acts like PooledStorageKey, but attempts to write the storage key
-// into the provided slice. If the slice capacity is insufficient, it allocates
-// a new byte slice with enough capacity to accommodate the result. This method
-// returns the resulting slice.
-func (b *Blockstore) StorageKey(dst []byte, cid cid.Cid) []byte {
-	h := cid.Hash()
-	reqsize := base32.RawStdEncoding.EncodedLen(len(h)) + b.prefixLen
-	if reqsize > cap(dst) {
-		// passed slice is smaller than required size; create new.
-		dst = make([]byte, reqsize)
-	} else if reqsize > len(dst) {
-		// passed slice has enough capacity, but its length is
-		// restricted, expand.
-		dst = dst[:cap(dst)]
-	}
-
-	if b.prefixing { // optimize for branch prediction.
-		copy(dst, b.prefix)
-		base32.RawStdEncoding.Encode(dst[b.prefixLen:], h)
-	} else {
-		base32.RawStdEncoding.Encode(dst, h)
-	}
-	return dst[:reqsize]
-}

blockstore/badger/blockstore_test.go

@@ -1,91 +0,0 @@
-package badgerbs
-
-import (
-	"io/ioutil"
-	"os"
-	"testing"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/stretchr/testify/require"
-
-	"github.com/filecoin-project/lotus/blockstore"
-)
-
-func TestBadgerBlockstore(t *testing.T) {
-	(&Suite{
-		NewBlockstore:  newBlockstore(DefaultOptions),
-		OpenBlockstore: openBlockstore(DefaultOptions),
-	}).RunTests(t, "non_prefixed")
-
-	prefixed := func(path string) Options {
-		opts := DefaultOptions(path)
-		opts.Prefix = "/prefixed/"
-		return opts
-	}
-
-	(&Suite{
-		NewBlockstore:  newBlockstore(prefixed),
-		OpenBlockstore: openBlockstore(prefixed),
-	}).RunTests(t, "prefixed")
-}
-
-func TestStorageKey(t *testing.T) {
-	bs, _ := newBlockstore(DefaultOptions)(t)
-	bbs := bs.(*Blockstore)
-	defer bbs.Close() //nolint:errcheck
-
-	cid1 := blocks.NewBlock([]byte("some data")).Cid()
-	cid2 := blocks.NewBlock([]byte("more data")).Cid()
-	cid3 := blocks.NewBlock([]byte("a little more data")).Cid()
-	require.NotEqual(t, cid1, cid2) // sanity check
-	require.NotEqual(t, cid2, cid3) // sanity check
-
-	// nil slice; let StorageKey allocate for us.
-	k1 := bbs.StorageKey(nil, cid1)
-	require.Len(t, k1, 55)
-	require.True(t, cap(k1) == len(k1))
-
-	// k1's backing array is reused.
-	k2 := bbs.StorageKey(k1, cid2)
-	require.Len(t, k2, 55)
-	require.True(t, cap(k2) == len(k1))
-
-	// bring k2 to len=0, and verify that its backing array gets reused
-	// (i.e. k1 and k2 are overwritten)
-	k3 := bbs.StorageKey(k2[:0], cid3)
-	require.Len(t, k3, 55)
-	require.True(t, cap(k3) == len(k3))
-
-	// backing array of k1 and k2 has been modified, i.e. memory is shared.
-	require.Equal(t, k3, k1)
-	require.Equal(t, k3, k2)
-}
-
-func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
-	return func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
-		tb.Helper()
-
-		path, err := ioutil.TempDir("", "")
-		if err != nil {
-			tb.Fatal(err)
-		}
-
-		db, err := Open(optsSupplier(path))
-		if err != nil {
-			tb.Fatal(err)
-		}
-
-		tb.Cleanup(func() {
-			_ = os.RemoveAll(path)
-		})
-
-		return db, path
-	}
-}
-
-func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) {
-	return func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) {
-		tb.Helper()
-		return Open(optsSupplier(path))
-	}
-}

blockstore/badger/blockstore_test_suite.go

@@ -1,313 +0,0 @@
-package badgerbs
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"reflect"
-	"strings"
-	"testing"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	u "github.com/ipfs/go-ipfs-util"
-
-	"github.com/filecoin-project/lotus/blockstore"
-
-	"github.com/stretchr/testify/require"
-)
-
-// TODO: move this to go-ipfs-blockstore.
-type Suite struct {
-	NewBlockstore  func(tb testing.TB) (bs blockstore.BasicBlockstore, path string)
-	OpenBlockstore func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error)
-}
-
-func (s *Suite) RunTests(t *testing.T, prefix string) {
-	v := reflect.TypeOf(s)
-	f := func(t *testing.T) {
-		for i := 0; i < v.NumMethod(); i++ {
-			if m := v.Method(i); strings.HasPrefix(m.Name, "Test") {
-				f := m.Func.Interface().(func(*Suite, *testing.T))
-				t.Run(m.Name, func(t *testing.T) {
-					f(s, t)
-				})
-			}
-		}
-	}
-
-	if prefix == "" {
-		f(t)
-	} else {
-		t.Run(prefix, f)
-	}
-}
-
-func (s *Suite) TestGetWhenKeyNotPresent(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	c := cid.NewCidV0(u.Hash([]byte("stuff")))
-	bl, err := bs.Get(c)
-	require.Nil(t, bl)
-	require.Equal(t, blockstore.ErrNotFound, err)
-}
-
-func (s *Suite) TestGetWhenKeyIsNil(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	_, err := bs.Get(cid.Undef)
-	require.Equal(t, blockstore.ErrNotFound, err)
-}
-
-func (s *Suite) TestPutThenGetBlock(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	orig := blocks.NewBlock([]byte("some data"))
-
-	err := bs.Put(orig)
-	require.NoError(t, err)
-
-	fetched, err := bs.Get(orig.Cid())
-	require.NoError(t, err)
-	require.Equal(t, orig.RawData(), fetched.RawData())
-}
-
-func (s *Suite) TestHas(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	orig := blocks.NewBlock([]byte("some data"))
-
-	err := bs.Put(orig)
-	require.NoError(t, err)
-
-	ok, err := bs.Has(orig.Cid())
-	require.NoError(t, err)
-	require.True(t, ok)
-
-	ok, err = bs.Has(blocks.NewBlock([]byte("another thing")).Cid())
-	require.NoError(t, err)
-	require.False(t, ok)
-}
-
-func (s *Suite) TestCidv0v1(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	orig := blocks.NewBlock([]byte("some data"))
-
-	err := bs.Put(orig)
-	require.NoError(t, err)
-
-	fetched, err := bs.Get(cid.NewCidV1(cid.DagProtobuf, orig.Cid().Hash()))
-	require.NoError(t, err)
-	require.Equal(t, orig.RawData(), fetched.RawData())
-}
-
-func (s *Suite) TestPutThenGetSizeBlock(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	block := blocks.NewBlock([]byte("some data"))
-	missingBlock := blocks.NewBlock([]byte("missingBlock"))
-	emptyBlock := blocks.NewBlock([]byte{})
-
-	err := bs.Put(block)
-	require.NoError(t, err)
-
-	blockSize, err := bs.GetSize(block.Cid())
-	require.NoError(t, err)
-	require.Len(t, block.RawData(), blockSize)
-
-	err = bs.Put(emptyBlock)
-	require.NoError(t, err)
-
-	emptySize, err := bs.GetSize(emptyBlock.Cid())
-	require.NoError(t, err)
-	require.Zero(t, emptySize)
-
-	missingSize, err := bs.GetSize(missingBlock.Cid())
-	require.Equal(t, blockstore.ErrNotFound, err)
-	require.Equal(t, -1, missingSize)
-}
-
-func (s *Suite) TestAllKeysSimple(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	keys := insertBlocks(t, bs, 100)
-
-	ctx := context.Background()
-	ch, err := bs.AllKeysChan(ctx)
-	require.NoError(t, err)
-	actual := collect(ch)
-
-	require.ElementsMatch(t, keys, actual)
-}
-
-func (s *Suite) TestAllKeysRespectsContext(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	_ = insertBlocks(t, bs, 100)
-
-	ctx, cancel := context.WithCancel(context.Background())
-	ch, err := bs.AllKeysChan(ctx)
-	require.NoError(t, err)
-
-	// consume 2, then cancel context.
-	v, ok := <-ch
-	require.NotEqual(t, cid.Undef, v)
-	require.True(t, ok)
-
-	v, ok = <-ch
-	require.NotEqual(t, cid.Undef, v)
-	require.True(t, ok)
-
-	cancel()
-	// pull one value out to avoid race
-	_, _ = <-ch
-
-	v, ok = <-ch
-	require.Equal(t, cid.Undef, v)
-	require.False(t, ok)
-}
-
-func (s *Suite) TestDoubleClose(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	c, ok := bs.(io.Closer)
-	if !ok {
-		t.SkipNow()
-	}
-	require.NoError(t, c.Close())
-	require.NoError(t, c.Close())
-}
-
-func (s *Suite) TestReopenPutGet(t *testing.T) {
-	bs, path := s.NewBlockstore(t)
-	c, ok := bs.(io.Closer)
-	if !ok {
-		t.SkipNow()
-	}
-
-	orig := blocks.NewBlock([]byte("some data"))
-	err := bs.Put(orig)
-	require.NoError(t, err)
-
-	err = c.Close()
-	require.NoError(t, err)
-
-	bs, err = s.OpenBlockstore(t, path)
-	require.NoError(t, err)
-
-	fetched, err := bs.Get(orig.Cid())
-	require.NoError(t, err)
-	require.Equal(t, orig.RawData(), fetched.RawData())
-
-	err = bs.(io.Closer).Close()
-	require.NoError(t, err)
-}
-
-func (s *Suite) TestPutMany(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	blks := []blocks.Block{
-		blocks.NewBlock([]byte("foo1")),
-		blocks.NewBlock([]byte("foo2")),
-		blocks.NewBlock([]byte("foo3")),
-	}
-	err := bs.PutMany(blks)
-	require.NoError(t, err)
-
-	for _, blk := range blks {
-		fetched, err := bs.Get(blk.Cid())
-		require.NoError(t, err)
-		require.Equal(t, blk.RawData(), fetched.RawData())
-
-		ok, err := bs.Has(blk.Cid())
-		require.NoError(t, err)
-		require.True(t, ok)
-	}
-
-	ch, err := bs.AllKeysChan(context.Background())
-	require.NoError(t, err)
-
-	cids := collect(ch)
-	require.Len(t, cids, 3)
-}
-
-func (s *Suite) TestDelete(t *testing.T) {
-	bs, _ := s.NewBlockstore(t)
-	if c, ok := bs.(io.Closer); ok {
-		defer func() { require.NoError(t, c.Close()) }()
-	}
-
-	blks := []blocks.Block{
-		blocks.NewBlock([]byte("foo1")),
-		blocks.NewBlock([]byte("foo2")),
-		blocks.NewBlock([]byte("foo3")),
-	}
-	err := bs.PutMany(blks)
-	require.NoError(t, err)
-
-	err = bs.DeleteBlock(blks[1].Cid())
-	require.NoError(t, err)
-
-	ch, err := bs.AllKeysChan(context.Background())
-	require.NoError(t, err)
-
-	cids := collect(ch)
-	require.Len(t, cids, 2)
-	require.ElementsMatch(t, cids, []cid.Cid{
-		cid.NewCidV1(cid.Raw, blks[0].Cid().Hash()),
-		cid.NewCidV1(cid.Raw, blks[2].Cid().Hash()),
-	})
-
-	has, err := bs.Has(blks[1].Cid())
-	require.NoError(t, err)
-	require.False(t, has)
-
-}
-
-func insertBlocks(t *testing.T, bs blockstore.BasicBlockstore, count int) []cid.Cid {
-	keys := make([]cid.Cid, count)
-	for i := 0; i < count; i++ {
-		block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
-		err := bs.Put(block)
-		require.NoError(t, err)
-		// NewBlock assigns a CIDv0; we convert it to CIDv1 because that's what
-		// the store returns.
-		keys[i] = cid.NewCidV1(cid.Raw, block.Multihash())
-	}
-	return keys
-}
-
-func collect(ch <-chan cid.Cid) []cid.Cid {
-	var keys []cid.Cid
-	for k := range ch {
-		keys = append(keys, k)
-	}
-	return keys
-}

blockstore/blockstore.go

@@ -1,95 +0,0 @@
-package blockstore
-
-import (
-	cid "github.com/ipfs/go-cid"
-	ds "github.com/ipfs/go-datastore"
-	logging "github.com/ipfs/go-log/v2"
-
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
-)
-
-var log = logging.Logger("blockstore")
-
-var ErrNotFound = blockstore.ErrNotFound
-
-// Blockstore is the blockstore interface used by Lotus. It is the union
-// of the basic go-ipfs blockstore, with other capabilities required by Lotus,
-// e.g. View or Sync.
-type Blockstore interface {
-	blockstore.Blockstore
-	blockstore.Viewer
-	BatchDeleter
-}
-
-// BasicBlockstore is an alias to the original IPFS Blockstore.
-type BasicBlockstore = blockstore.Blockstore
-
-type Viewer = blockstore.Viewer
-
-type BatchDeleter interface {
-	DeleteMany(cids []cid.Cid) error
-}
-
-// WrapIDStore wraps the underlying blockstore in an "identity" blockstore.
-// The ID store filters out all puts for blocks with CIDs using the "identity"
-// hash function. It also extracts inlined blocks from CIDs using the identity
-// hash function and returns them on get/has, ignoring the contents of the
-// blockstore.
-func WrapIDStore(bstore blockstore.Blockstore) Blockstore {
-	if is, ok := bstore.(*idstore); ok {
-		// already wrapped
-		return is
-	}
-
-	if bs, ok := bstore.(Blockstore); ok {
-		// we need to wrap our own because we don't want to neuter the DeleteMany method
-		// the underlying blockstore has implemented an (efficient) DeleteMany
-		return NewIDStore(bs)
-	}
-
-	// The underlying blockstore does not implement DeleteMany, so we need to shim it.
-	// This is less efficient as it'll iterate and perform single deletes.
-	return NewIDStore(Adapt(bstore))
-}
-
-// FromDatastore creates a new blockstore backed by the given datastore.
-func FromDatastore(dstore ds.Batching) Blockstore {
-	return WrapIDStore(blockstore.NewBlockstore(dstore))
-}
-
-type adaptedBlockstore struct {
-	blockstore.Blockstore
-}
-
-var _ Blockstore = (*adaptedBlockstore)(nil)
-
-func (a *adaptedBlockstore) View(cid cid.Cid, callback func([]byte) error) error {
-	blk, err := a.Get(cid)
-	if err != nil {
-		return err
-	}
-	return callback(blk.RawData())
-}
-
-func (a *adaptedBlockstore) DeleteMany(cids []cid.Cid) error {
-	for _, cid := range cids {
-		err := a.DeleteBlock(cid)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// Adapt adapts a standard blockstore to a Lotus blockstore by
-// enriching it with the extra methods that Lotus requires (e.g. View, Sync).
-//
-// View proxies over to Get and calls the callback with the value supplied by Get.
-// Sync noops.
-func Adapt(bs blockstore.Blockstore) Blockstore {
-	if ret, ok := bs.(Blockstore); ok {
-		return ret
-	}
-	return &adaptedBlockstore{bs}
-}

blockstore/buffered.go

@@ -1,174 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"os"
-
-	block "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-)
-
-// buflog is a logger for the buffered blockstore. It is subscoped from the
-// blockstore logger.
-var buflog = log.Named("buf")
-
-type BufferedBlockstore struct {
-	read  Blockstore
-	write Blockstore
-}
-
-func NewBuffered(base Blockstore) *BufferedBlockstore {
-	var buf Blockstore
-	if os.Getenv("LOTUS_DISABLE_VM_BUF") == "iknowitsabadidea" {
-		buflog.Warn("VM BLOCKSTORE BUFFERING IS DISABLED")
-		buf = base
-	} else {
-		buf = NewMemory()
-	}
-
-	bs := &BufferedBlockstore{
-		read:  base,
-		write: buf,
-	}
-	return bs
-}
-
-func NewTieredBstore(r Blockstore, w Blockstore) *BufferedBlockstore {
-	return &BufferedBlockstore{
-		read:  r,
-		write: w,
-	}
-}
-
-var (
-	_ Blockstore = (*BufferedBlockstore)(nil)
-	_ Viewer     = (*BufferedBlockstore)(nil)
-)
-
-func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	a, err := bs.read.AllKeysChan(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	b, err := bs.write.AllKeysChan(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	out := make(chan cid.Cid)
-	go func() {
-		defer close(out)
-		for a != nil || b != nil {
-			select {
-			case val, ok := <-a:
-				if !ok {
-					a = nil
-				} else {
-					select {
-					case out <- val:
-					case <-ctx.Done():
-						return
-					}
-				}
-			case val, ok := <-b:
-				if !ok {
-					b = nil
-				} else {
-					select {
-					case out <- val:
-					case <-ctx.Done():
-						return
-					}
-				}
-			}
-		}
-	}()
-
-	return out, nil
-}
-
-func (bs *BufferedBlockstore) DeleteBlock(c cid.Cid) error {
-	if err := bs.read.DeleteBlock(c); err != nil {
-		return err
-	}
-
-	return bs.write.DeleteBlock(c)
-}
-
-func (bs *BufferedBlockstore) DeleteMany(cids []cid.Cid) error {
-	if err := bs.read.DeleteMany(cids); err != nil {
-		return err
-	}
-
-	return bs.write.DeleteMany(cids)
-}
-
-func (bs *BufferedBlockstore) View(c cid.Cid, callback func([]byte) error) error {
-	// both stores are viewable.
-	if err := bs.write.View(c, callback); err == ErrNotFound {
-		// not found in write blockstore; fall through.
-	} else {
-		return err // propagate errors, or nil, i.e. found.
-	}
-	return bs.read.View(c, callback)
-}
-
-func (bs *BufferedBlockstore) Get(c cid.Cid) (block.Block, error) {
-	if out, err := bs.write.Get(c); err != nil {
-		if err != ErrNotFound {
-			return nil, err
-		}
-	} else {
-		return out, nil
-	}
-
-	return bs.read.Get(c)
-}
-
-func (bs *BufferedBlockstore) GetSize(c cid.Cid) (int, error) {
-	s, err := bs.read.GetSize(c)
-	if err == ErrNotFound || s == 0 {
-		return bs.write.GetSize(c)
-	}
-
-	return s, err
-}
-
-func (bs *BufferedBlockstore) Put(blk block.Block) error {
-	has, err := bs.read.Has(blk.Cid()) // TODO: consider dropping this check
-	if err != nil {
-		return err
-	}
-
-	if has {
-		return nil
-	}
-
-	return bs.write.Put(blk)
-}
-
-func (bs *BufferedBlockstore) Has(c cid.Cid) (bool, error) {
-	has, err := bs.write.Has(c)
-	if err != nil {
-		return false, err
-	}
-	if has {
-		return true, nil
-	}
-
-	return bs.read.Has(c)
-}
-
-func (bs *BufferedBlockstore) HashOnRead(hor bool) {
-	bs.read.HashOnRead(hor)
-	bs.write.HashOnRead(hor)
-}
-
-func (bs *BufferedBlockstore) PutMany(blks []block.Block) error {
-	return bs.write.PutMany(blks)
-}
-
-func (bs *BufferedBlockstore) Read() Blockstore {
-	return bs.read
-}

blockstore/cached.go

@@ -1,25 +0,0 @@
-package blockstore
-
-import (
-	"context"
-
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
-)
-
-type CacheOpts = blockstore.CacheOpts
-
-func DefaultCacheOpts() CacheOpts {
-	return CacheOpts{
-		HasBloomFilterSize:   0,
-		HasBloomFilterHashes: 0,
-		HasARCCacheSize:      512 << 10,
-	}
-}
-
-func CachedBlockstore(ctx context.Context, bs Blockstore, opts CacheOpts) (Blockstore, error) {
-	cached, err := blockstore.CachedBlockstore(ctx, bs, opts)
-	if err != nil {
-		return nil, err
-	}
-	return WrapIDStore(cached), nil
-}

blockstore/doc.go

@@ -1,9 +0,0 @@
-// Package blockstore and subpackages contain most of the blockstore
-// implementations used by Lotus.
-//
-// Blockstores not ultimately constructed out of the building blocks in this
-// package may not work properly.
-//
-// This package re-exports parts of the go-ipfs-blockstore package such that
-// no other package needs to import it directly, for ergonomics and traceability.
-package blockstore

blockstore/fallback.go

@@ -1,106 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"sync"
-	"time"
-
-	"golang.org/x/xerrors"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-)
-
-// UnwrapFallbackStore takes a blockstore, and returns the underlying blockstore
-// if it was a FallbackStore. Otherwise, it just returns the supplied store
-// unmodified.
-func UnwrapFallbackStore(bs Blockstore) (Blockstore, bool) {
-	if fbs, ok := bs.(*FallbackStore); ok {
-		return fbs.Blockstore, true
-	}
-	return bs, false
-}
-
-// FallbackStore is a read-through store that queries another (potentially
-// remote) source if the block is not found locally. If the block is found
-// during the fallback, it stores it in the local store.
-type FallbackStore struct {
-	Blockstore
-
-	lk sync.RWMutex
-	// missFn is the function that will be invoked on a local miss to pull the
-	// block from elsewhere.
-	missFn func(context.Context, cid.Cid) (blocks.Block, error)
-}
-
-var _ Blockstore = (*FallbackStore)(nil)
-
-func (fbs *FallbackStore) SetFallback(missFn func(context.Context, cid.Cid) (blocks.Block, error)) {
-	fbs.lk.Lock()
-	defer fbs.lk.Unlock()
-
-	fbs.missFn = missFn
-}
-
-func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
-	log.Warnf("fallbackstore: block not found locally, fetching from the network; cid: %s", c)
-	fbs.lk.RLock()
-	defer fbs.lk.RUnlock()
-
-	if fbs.missFn == nil {
-		// FallbackStore wasn't configured yet (chainstore/bitswap aren't up yet)
-		// Wait for a bit and retry
-		fbs.lk.RUnlock()
-		time.Sleep(5 * time.Second)
-		fbs.lk.RLock()
-
-		if fbs.missFn == nil {
-			log.Errorw("fallbackstore: missFn not configured yet")
-			return nil, ErrNotFound
-		}
-	}
-
-	ctx, cancel := context.WithTimeout(context.TODO(), 120*time.Second)
-	defer cancel()
-
-	b, err := fbs.missFn(ctx, c)
-	if err != nil {
-		return nil, err
-	}
-
-	// chain bitswap puts blocks in temp blockstore which is cleaned up
-	// every few min (to drop any messages we fetched but don't want)
-	// in this case we want to keep this block around
-	if err := fbs.Put(b); err != nil {
-		return nil, xerrors.Errorf("persisting fallback-fetched block: %w", err)
-	}
-	return b, nil
-}
-
-func (fbs *FallbackStore) Get(c cid.Cid) (blocks.Block, error) {
-	b, err := fbs.Blockstore.Get(c)
-	switch err {
-	case nil:
-		return b, nil
-	case ErrNotFound:
-		return fbs.getFallback(c)
-	default:
-		return b, err
-	}
-}
-
-func (fbs *FallbackStore) GetSize(c cid.Cid) (int, error) {
-	sz, err := fbs.Blockstore.GetSize(c)
-	switch err {
-	case nil:
-		return sz, nil
-	case ErrNotFound:
-		b, err := fbs.getFallback(c)
-		if err != nil {
-			return 0, err
-		}
-		return len(b.RawData()), nil
-	default:
-		return sz, err
-	}
-}

blockstore/idstore.go

@@ -1,174 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"io"
-
-	"golang.org/x/xerrors"
-
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-	mh "github.com/multiformats/go-multihash"
-)
-
-var _ Blockstore = (*idstore)(nil)
-
-type idstore struct {
-	bs Blockstore
-}
-
-func NewIDStore(bs Blockstore) Blockstore {
-	return &idstore{bs: bs}
-}
-
-func decodeCid(cid cid.Cid) (inline bool, data []byte, err error) {
-	if cid.Prefix().MhType != mh.IDENTITY {
-		return false, nil, nil
-	}
-
-	dmh, err := mh.Decode(cid.Hash())
-	if err != nil {
-		return false, nil, err
-	}
-
-	if dmh.Code == mh.IDENTITY {
-		return true, dmh.Digest, nil
-	}
-
-	return false, nil, err
-}
-
-func (b *idstore) Has(cid cid.Cid) (bool, error) {
-	inline, _, err := decodeCid(cid)
-	if err != nil {
-		return false, xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return true, nil
-	}
-
-	return b.bs.Has(cid)
-}
-
-func (b *idstore) Get(cid cid.Cid) (blocks.Block, error) {
-	inline, data, err := decodeCid(cid)
-	if err != nil {
-		return nil, xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return blocks.NewBlockWithCid(data, cid)
-	}
-
-	return b.bs.Get(cid)
-}
-
-func (b *idstore) GetSize(cid cid.Cid) (int, error) {
-	inline, data, err := decodeCid(cid)
-	if err != nil {
-		return 0, xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return len(data), err
-	}
-
-	return b.bs.GetSize(cid)
-}
-
-func (b *idstore) View(cid cid.Cid, cb func([]byte) error) error {
-	inline, data, err := decodeCid(cid)
-	if err != nil {
-		return xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return cb(data)
-	}
-
-	return b.bs.View(cid, cb)
-}
-
-func (b *idstore) Put(blk blocks.Block) error {
-	inline, _, err := decodeCid(blk.Cid())
-	if err != nil {
-		return xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return nil
-	}
-
-	return b.bs.Put(blk)
-}
-
-func (b *idstore) PutMany(blks []blocks.Block) error {
-	toPut := make([]blocks.Block, 0, len(blks))
-	for _, blk := range blks {
-		inline, _, err := decodeCid(blk.Cid())
-		if err != nil {
-			return xerrors.Errorf("error decoding Cid: %w", err)
-		}
-
-		if inline {
-			continue
-		}
-		toPut = append(toPut, blk)
-	}
-
-	if len(toPut) > 0 {
-		return b.bs.PutMany(toPut)
-	}
-
-	return nil
-}
-
-func (b *idstore) DeleteBlock(cid cid.Cid) error {
-	inline, _, err := decodeCid(cid)
-	if err != nil {
-		return xerrors.Errorf("error decoding Cid: %w", err)
-	}
-
-	if inline {
-		return nil
-	}
-
-	return b.bs.DeleteBlock(cid)
-}
-
-func (b *idstore) DeleteMany(cids []cid.Cid) error {
-	toDelete := make([]cid.Cid, 0, len(cids))
-	for _, cid := range cids {
-		inline, _, err := decodeCid(cid)
-		if err != nil {
-			return xerrors.Errorf("error decoding Cid: %w", err)
-		}
-
-		if inline {
-			continue
-		}
-		toDelete = append(toDelete, cid)
-	}
-
-	if len(toDelete) > 0 {
-		return b.bs.DeleteMany(toDelete)
-	}
-
-	return nil
-}
-
-func (b *idstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return b.bs.AllKeysChan(ctx)
-}
-
-func (b *idstore) HashOnRead(enabled bool) {
-	b.bs.HashOnRead(enabled)
-}
-
-func (b *idstore) Close() error {
-	if c, ok := b.bs.(io.Closer); ok {
-		return c.Close()
-	}
-	return nil
-}

blockstore/ipfs.go

@@ -1,155 +0,0 @@
-package blockstore
-
-import (
-	"bytes"
-	"context"
-	"io/ioutil"
-
-	"golang.org/x/xerrors"
-
-	"github.com/multiformats/go-multiaddr"
-	"github.com/multiformats/go-multihash"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	httpapi "github.com/ipfs/go-ipfs-http-client"
-	iface "github.com/ipfs/interface-go-ipfs-core"
-	"github.com/ipfs/interface-go-ipfs-core/options"
-	"github.com/ipfs/interface-go-ipfs-core/path"
-)
-
-type IPFSBlockstore struct {
-	ctx             context.Context
-	api, offlineAPI iface.CoreAPI
-}
-
-var _ BasicBlockstore = (*IPFSBlockstore)(nil)
-
-func NewLocalIPFSBlockstore(ctx context.Context, onlineMode bool) (Blockstore, error) {
-	localApi, err := httpapi.NewLocalApi()
-	if err != nil {
-		return nil, xerrors.Errorf("getting local ipfs api: %w", err)
-	}
-	api, err := localApi.WithOptions(options.Api.Offline(!onlineMode))
-	if err != nil {
-		return nil, xerrors.Errorf("setting offline mode: %s", err)
-	}
-
-	offlineAPI := api
-	if onlineMode {
-		offlineAPI, err = localApi.WithOptions(options.Api.Offline(true))
-		if err != nil {
-			return nil, xerrors.Errorf("applying offline mode: %s", err)
-		}
-	}
-
-	bs := &IPFSBlockstore{
-		ctx:        ctx,
-		api:        api,
-		offlineAPI: offlineAPI,
-	}
-
-	return Adapt(bs), nil
-}
-
-func NewRemoteIPFSBlockstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineMode bool) (Blockstore, error) {
-	httpApi, err := httpapi.NewApi(maddr)
-	if err != nil {
-		return nil, xerrors.Errorf("setting remote ipfs api: %w", err)
-	}
-	api, err := httpApi.WithOptions(options.Api.Offline(!onlineMode))
-	if err != nil {
-		return nil, xerrors.Errorf("applying offline mode: %s", err)
-	}
-
-	offlineAPI := api
-	if onlineMode {
-		offlineAPI, err = httpApi.WithOptions(options.Api.Offline(true))
-		if err != nil {
-			return nil, xerrors.Errorf("applying offline mode: %s", err)
-		}
-	}
-
-	bs := &IPFSBlockstore{
-		ctx:        ctx,
-		api:        api,
-		offlineAPI: offlineAPI,
-	}
-
-	return Adapt(bs), nil
-}
-
-func (i *IPFSBlockstore) DeleteBlock(cid cid.Cid) error {
-	return xerrors.Errorf("not supported")
-}
-
-func (i *IPFSBlockstore) Has(cid cid.Cid) (bool, error) {
-	_, err := i.offlineAPI.Block().Stat(i.ctx, path.IpldPath(cid))
-	if err != nil {
-		// The underlying client is running in Offline mode.
-		// Stat() will fail with an err if the block isn't in the
-		// blockstore. If that's the case, return false without
-		// an error since that's the original intention of this method.
-		if err.Error() == "blockservice: key not found" {
-			return false, nil
-		}
-		return false, xerrors.Errorf("getting ipfs block: %w", err)
-	}
-
-	return true, nil
-}
-
-func (i *IPFSBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
-	rd, err := i.api.Block().Get(i.ctx, path.IpldPath(cid))
-	if err != nil {
-		return nil, xerrors.Errorf("getting ipfs block: %w", err)
-	}
-
-	data, err := ioutil.ReadAll(rd)
-	if err != nil {
-		return nil, err
-	}
-
-	return blocks.NewBlockWithCid(data, cid)
-}
-
-func (i *IPFSBlockstore) GetSize(cid cid.Cid) (int, error) {
-	st, err := i.api.Block().Stat(i.ctx, path.IpldPath(cid))
-	if err != nil {
-		return 0, xerrors.Errorf("getting ipfs block: %w", err)
-	}
-
-	return st.Size(), nil
-}
-
-func (i *IPFSBlockstore) Put(block blocks.Block) error {
-	mhd, err := multihash.Decode(block.Cid().Hash())
-	if err != nil {
-		return err
-	}
-
-	_, err = i.api.Block().Put(i.ctx, bytes.NewReader(block.RawData()),
-		options.Block.Hash(mhd.Code, mhd.Length),
-		options.Block.Format(cid.CodecToStr[block.Cid().Type()]))
-	return err
-}
-
-func (i *IPFSBlockstore) PutMany(blocks []blocks.Block) error {
-	// TODO: could be done in parallel
-
-	for _, block := range blocks {
-		if err := i.Put(block); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (i *IPFSBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return nil, xerrors.Errorf("not supported")
-}
-
-func (i *IPFSBlockstore) HashOnRead(enabled bool) {
-	return // TODO: We could technically support this, but..
-}

blockstore/mem.go

@@ -1,102 +0,0 @@
-package blockstore
-
-import (
-	"context"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-)
-
-// NewMemory returns a temporary memory-backed blockstore.
-func NewMemory() MemBlockstore {
-	return make(MemBlockstore)
-}
-
-// MemBlockstore is a terminal blockstore that keeps blocks in memory.
-type MemBlockstore map[cid.Cid]blocks.Block
-
-func (m MemBlockstore) DeleteBlock(k cid.Cid) error {
-	delete(m, k)
-	return nil
-}
-
-func (m MemBlockstore) DeleteMany(ks []cid.Cid) error {
-	for _, k := range ks {
-		delete(m, k)
-	}
-	return nil
-}
-
-func (m MemBlockstore) Has(k cid.Cid) (bool, error) {
-	_, ok := m[k]
-	return ok, nil
-}
-
-func (m MemBlockstore) View(k cid.Cid, callback func([]byte) error) error {
-	b, ok := m[k]
-	if !ok {
-		return ErrNotFound
-	}
-	return callback(b.RawData())
-}
-
-func (m MemBlockstore) Get(k cid.Cid) (blocks.Block, error) {
-	b, ok := m[k]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	return b, nil
-}
-
-// GetSize returns the CIDs mapped BlockSize
-func (m MemBlockstore) GetSize(k cid.Cid) (int, error) {
-	b, ok := m[k]
-	if !ok {
-		return 0, ErrNotFound
-	}
-	return len(b.RawData()), nil
-}
-
-// Put puts a given block to the underlying datastore
-func (m MemBlockstore) Put(b blocks.Block) error {
-	// Convert to a basic block for safety, but try to reuse the existing
-	// block if it's already a basic block.
-	k := b.Cid()
-	if _, ok := b.(*blocks.BasicBlock); !ok {
-		// If we already have the block, abort.
-		if _, ok := m[k]; ok {
-			return nil
-		}
-		// the error is only for debugging.
-		b, _ = blocks.NewBlockWithCid(b.RawData(), b.Cid())
-	}
-	m[b.Cid()] = b
-	return nil
-}
-
-// PutMany puts a slice of blocks at the same time using batching
-// capabilities of the underlying datastore whenever possible.
-func (m MemBlockstore) PutMany(bs []blocks.Block) error {
-	for _, b := range bs {
-		_ = m.Put(b) // can't fail
-	}
-	return nil
-}
-
-// AllKeysChan returns a channel from which
-// the CIDs in the Blockstore can be read. It should respect
-// the given context, closing the channel if it becomes Done.
-func (m MemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	ch := make(chan cid.Cid, len(m))
-	for k := range m {
-		ch <- k
-	}
-	close(ch)
-	return ch, nil
-}
-
-// HashOnRead specifies if every read block should be
-// rehashed to make sure it matches its CID.
-func (m MemBlockstore) HashOnRead(enabled bool) {
-	// no-op
-}

blockstore/metrics.go

@@ -1,154 +0,0 @@
-package blockstore
-
-import (
-	"time"
-
-	"go.opencensus.io/stats"
-	"go.opencensus.io/stats/view"
-	"go.opencensus.io/tag"
-)
-
-//
-// Currently unused, but kept in repo in case we introduce one of the candidate
-// cache implementations (Freecache, Ristretto), both of which report these
-// metrics.
-//
-
-// CacheMetricsEmitInterval is the interval at which metrics are emitted onto
-// OpenCensus.
-var CacheMetricsEmitInterval = 5 * time.Second
-
-var (
-	CacheName, _ = tag.NewKey("cache_name")
-)
-
-// CacheMeasures groups all metrics emitted by the blockstore caches.
-var CacheMeasures = struct {
-	HitRatio       *stats.Float64Measure
-	Hits           *stats.Int64Measure
-	Misses         *stats.Int64Measure
-	Entries        *stats.Int64Measure
-	QueriesServed  *stats.Int64Measure
-	Adds           *stats.Int64Measure
-	Updates        *stats.Int64Measure
-	Evictions      *stats.Int64Measure
-	CostAdded      *stats.Int64Measure
-	CostEvicted    *stats.Int64Measure
-	SetsDropped    *stats.Int64Measure
-	SetsRejected   *stats.Int64Measure
-	QueriesDropped *stats.Int64Measure
-}{
-	HitRatio:       stats.Float64("blockstore/cache/hit_ratio", "Hit ratio of blockstore cache", stats.UnitDimensionless),
-	Hits:           stats.Int64("blockstore/cache/hits", "Total number of hits at blockstore cache", stats.UnitDimensionless),
-	Misses:         stats.Int64("blockstore/cache/misses", "Total number of misses at blockstore cache", stats.UnitDimensionless),
-	Entries:        stats.Int64("blockstore/cache/entry_count", "Total number of entries currently in the blockstore cache", stats.UnitDimensionless),
-	QueriesServed:  stats.Int64("blockstore/cache/queries_served", "Total number of queries served by the blockstore cache", stats.UnitDimensionless),
-	Adds:           stats.Int64("blockstore/cache/adds", "Total number of adds to blockstore cache", stats.UnitDimensionless),
-	Updates:        stats.Int64("blockstore/cache/updates", "Total number of updates in blockstore cache", stats.UnitDimensionless),
-	Evictions:      stats.Int64("blockstore/cache/evictions", "Total number of evictions from blockstore cache", stats.UnitDimensionless),
-	CostAdded:      stats.Int64("blockstore/cache/cost_added", "Total cost (byte size) of entries added into blockstore cache", stats.UnitBytes),
-	CostEvicted:    stats.Int64("blockstore/cache/cost_evicted", "Total cost (byte size) of entries evicted by blockstore cache", stats.UnitBytes),
-	SetsDropped:    stats.Int64("blockstore/cache/sets_dropped", "Total number of sets dropped by blockstore cache", stats.UnitDimensionless),
-	SetsRejected:   stats.Int64("blockstore/cache/sets_rejected", "Total number of sets rejected by blockstore cache", stats.UnitDimensionless),
-	QueriesDropped: stats.Int64("blockstore/cache/queries_dropped", "Total number of queries dropped by blockstore cache", stats.UnitDimensionless),
-}
-
-// CacheViews groups all cache-related default views.
-var CacheViews = struct {
-	HitRatio       *view.View
-	Hits           *view.View
-	Misses         *view.View
-	Entries        *view.View
-	QueriesServed  *view.View
-	Adds           *view.View
-	Updates        *view.View
-	Evictions      *view.View
-	CostAdded      *view.View
-	CostEvicted    *view.View
-	SetsDropped    *view.View
-	SetsRejected   *view.View
-	QueriesDropped *view.View
-}{
-	HitRatio: &view.View{
-		Measure:     CacheMeasures.HitRatio,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Hits: &view.View{
-		Measure:     CacheMeasures.Hits,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Misses: &view.View{
-		Measure:     CacheMeasures.Misses,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Entries: &view.View{
-		Measure:     CacheMeasures.Entries,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	QueriesServed: &view.View{
-		Measure:     CacheMeasures.QueriesServed,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Adds: &view.View{
-		Measure:     CacheMeasures.Adds,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Updates: &view.View{
-		Measure:     CacheMeasures.Updates,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	Evictions: &view.View{
-		Measure:     CacheMeasures.Evictions,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	CostAdded: &view.View{
-		Measure:     CacheMeasures.CostAdded,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	CostEvicted: &view.View{
-		Measure:     CacheMeasures.CostEvicted,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	SetsDropped: &view.View{
-		Measure:     CacheMeasures.SetsDropped,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	SetsRejected: &view.View{
-		Measure:     CacheMeasures.SetsRejected,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-	QueriesDropped: &view.View{
-		Measure:     CacheMeasures.QueriesDropped,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{CacheName},
-	},
-}
-
-// DefaultViews exports all default views for this package.
-var DefaultViews = []*view.View{
-	CacheViews.HitRatio,
-	CacheViews.Hits,
-	CacheViews.Misses,
-	CacheViews.Entries,
-	CacheViews.QueriesServed,
-	CacheViews.Adds,
-	CacheViews.Updates,
-	CacheViews.Evictions,
-	CacheViews.CostAdded,
-	CacheViews.CostEvicted,
-	CacheViews.SetsDropped,
-	CacheViews.SetsRejected,
-	CacheViews.QueriesDropped,
-}

blockstore/splitstore/markset.go

@@ -1,38 +0,0 @@
-package splitstore
-
-import (
-	"path/filepath"
-
-	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
-)
-
-// MarkSet is a utility to keep track of seen CID, and later query for them.
-//
-// * If the expected dataset is large, it can be backed by a datastore (e.g. bbolt).
-// * If a probabilistic result is acceptable, it can be backed by a bloom filter (default).
-type MarkSet interface {
-	Mark(cid.Cid) error
-	Has(cid.Cid) (bool, error)
-	Close() error
-}
-
-// markBytes is deliberately a non-nil empty byte slice for serialization.
-var markBytes = []byte{}
-
-type MarkSetEnv interface {
-	Create(name string, sizeHint int64) (MarkSet, error)
-	Close() error
-}
-
-func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) {
-	switch mtype {
-	case "", "bloom":
-		return NewBloomMarkSetEnv()
-	case "bolt":
-		return NewBoltMarkSetEnv(filepath.Join(path, "markset.bolt"))
-	default:
-		return nil, xerrors.Errorf("unknown mark set type %s", mtype)
-	}
-}

blockstore/splitstore/markset_bloom.go

@@ -1,77 +0,0 @@
-package splitstore
-
-import (
-	"crypto/rand"
-	"crypto/sha256"
-
-	"golang.org/x/xerrors"
-
-	bbloom "github.com/ipfs/bbloom"
-	cid "github.com/ipfs/go-cid"
-)
-
-const (
-	BloomFilterMinSize     = 10_000_000
-	BloomFilterProbability = 0.01
-)
-
-type BloomMarkSetEnv struct{}
-
-var _ MarkSetEnv = (*BloomMarkSetEnv)(nil)
-
-type BloomMarkSet struct {
-	salt []byte
-	bf   *bbloom.Bloom
-}
-
-var _ MarkSet = (*BloomMarkSet)(nil)
-
-func NewBloomMarkSetEnv() (*BloomMarkSetEnv, error) {
-	return &BloomMarkSetEnv{}, nil
-}
-
-func (e *BloomMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
-	size := int64(BloomFilterMinSize)
-	for size < sizeHint {
-		size += BloomFilterMinSize
-	}
-
-	salt := make([]byte, 4)
-	_, err := rand.Read(salt)
-	if err != nil {
-		return nil, xerrors.Errorf("error reading salt: %w", err)
-	}
-
-	bf, err := bbloom.New(float64(size), BloomFilterProbability)
-	if err != nil {
-		return nil, xerrors.Errorf("error creating bloom filter: %w", err)
-	}
-
-	return &BloomMarkSet{salt: salt, bf: bf}, nil
-}
-
-func (e *BloomMarkSetEnv) Close() error {
-	return nil
-}
-
-func (s *BloomMarkSet) saltedKey(cid cid.Cid) []byte {
-	hash := cid.Hash()
-	key := make([]byte, len(s.salt)+len(hash))
-	n := copy(key, s.salt)
-	copy(key[n:], hash)
-	rehash := sha256.Sum256(key)
-	return rehash[:]
-}
-
-func (s *BloomMarkSet) Mark(cid cid.Cid) error {
-	s.bf.Add(s.saltedKey(cid))
-	return nil
-}
-
-func (s *BloomMarkSet) Has(cid cid.Cid) (bool, error) {
-	return s.bf.Has(s.saltedKey(cid)), nil
-}
-
-func (s *BloomMarkSet) Close() error {
-	return nil
-}

blockstore/splitstore/markset_bolt.go

@@ -1,81 +0,0 @@
-package splitstore
-
-import (
-	"time"
-
-	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
-	bolt "go.etcd.io/bbolt"
-)
-
-type BoltMarkSetEnv struct {
-	db *bolt.DB
-}
-
-var _ MarkSetEnv = (*BoltMarkSetEnv)(nil)
-
-type BoltMarkSet struct {
-	db       *bolt.DB
-	bucketId []byte
-}
-
-var _ MarkSet = (*BoltMarkSet)(nil)
-
-func NewBoltMarkSetEnv(path string) (*BoltMarkSetEnv, error) {
-	db, err := bolt.Open(path, 0644,
-		&bolt.Options{
-			Timeout: 1 * time.Second,
-			NoSync:  true,
-		})
-	if err != nil {
-		return nil, err
-	}
-
-	return &BoltMarkSetEnv{db: db}, nil
-}
-
-func (e *BoltMarkSetEnv) Create(name string, hint int64) (MarkSet, error) {
-	bucketId := []byte(name)
-	err := e.db.Update(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists(bucketId)
-		if err != nil {
-			return xerrors.Errorf("error creating bolt db bucket %s: %w", name, err)
-		}
-		return nil
-	})
-
-	if err != nil {
-		return nil, err
-	}
-
-	return &BoltMarkSet{db: e.db, bucketId: bucketId}, nil
-}
-
-func (e *BoltMarkSetEnv) Close() error {
-	return e.db.Close()
-}
-
-func (s *BoltMarkSet) Mark(cid cid.Cid) error {
-	return s.db.Update(func(tx *bolt.Tx) error {
-		b := tx.Bucket(s.bucketId)
-		return b.Put(cid.Hash(), markBytes)
-	})
-}
-
-func (s *BoltMarkSet) Has(cid cid.Cid) (result bool, err error) {
-	err = s.db.View(func(tx *bolt.Tx) error {
-		b := tx.Bucket(s.bucketId)
-		v := b.Get(cid.Hash())
-		result = v != nil
-		return nil
-	})
-
-	return result, err
-}
-
-func (s *BoltMarkSet) Close() error {
-	return s.db.Update(func(tx *bolt.Tx) error {
-		return tx.DeleteBucket(s.bucketId)
-	})
-}

blockstore/splitstore/markset_test.go

@@ -1,138 +0,0 @@
-package splitstore
-
-import (
-	"io/ioutil"
-	"testing"
-
-	cid "github.com/ipfs/go-cid"
-	"github.com/multiformats/go-multihash"
-)
-
-func TestBoltMarkSet(t *testing.T) {
-	testMarkSet(t, "bolt")
-}
-
-func TestBloomMarkSet(t *testing.T) {
-	testMarkSet(t, "bloom")
-}
-
-func testMarkSet(t *testing.T, lsType string) {
-	t.Helper()
-
-	path, err := ioutil.TempDir("", "sweep-test.*")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	env, err := OpenMarkSetEnv(path, lsType)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer env.Close() //nolint:errcheck
-
-	hotSet, err := env.Create("hot", 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	coldSet, err := env.Create("cold", 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	makeCid := func(key string) cid.Cid {
-		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		return cid.NewCidV1(cid.Raw, h)
-	}
-
-	mustHave := func(s MarkSet, cid cid.Cid) {
-		has, err := s.Has(cid)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if !has {
-			t.Fatal("mark not found")
-		}
-	}
-
-	mustNotHave := func(s MarkSet, cid cid.Cid) {
-		has, err := s.Has(cid)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if has {
-			t.Fatal("unexpected mark")
-		}
-	}
-
-	k1 := makeCid("a")
-	k2 := makeCid("b")
-	k3 := makeCid("c")
-	k4 := makeCid("d")
-
-	hotSet.Mark(k1)  //nolint
-	hotSet.Mark(k2)  //nolint
-	coldSet.Mark(k3) //nolint
-
-	mustHave(hotSet, k1)
-	mustHave(hotSet, k2)
-	mustNotHave(hotSet, k3)
-	mustNotHave(hotSet, k4)
-
-	mustNotHave(coldSet, k1)
-	mustNotHave(coldSet, k2)
-	mustHave(coldSet, k3)
-	mustNotHave(coldSet, k4)
-
-	// close them and reopen to redo the dance
-
-	err = hotSet.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = coldSet.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	hotSet, err = env.Create("hot", 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	coldSet, err = env.Create("cold", 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	hotSet.Mark(k3)  //nolint
-	hotSet.Mark(k4)  //nolint
-	coldSet.Mark(k1) //nolint
-
-	mustNotHave(hotSet, k1)
-	mustNotHave(hotSet, k2)
-	mustHave(hotSet, k3)
-	mustHave(hotSet, k4)
-
-	mustHave(coldSet, k1)
-	mustNotHave(coldSet, k2)
-	mustNotHave(coldSet, k3)
-	mustNotHave(coldSet, k4)
-
-	err = hotSet.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = coldSet.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-}

blockstore/splitstore/splitstore_test.go

@@ -1,255 +0,0 @@
-package splitstore
-
-import (
-	"context"
-	"fmt"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/blockstore"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/chain/types/mock"
-
-	cid "github.com/ipfs/go-cid"
-	datastore "github.com/ipfs/go-datastore"
-	dssync "github.com/ipfs/go-datastore/sync"
-	logging "github.com/ipfs/go-log/v2"
-)
-
-func init() {
-	CompactionThreshold = 5
-	CompactionCold = 1
-	CompactionBoundary = 2
-	logging.SetLogLevel("splitstore", "DEBUG")
-}
-
-func testSplitStore(t *testing.T, cfg *Config) {
-	chain := &mockChain{}
-	// genesis
-	genBlock := mock.MkBlock(nil, 0, 0)
-	genTs := mock.TipSet(genBlock)
-	chain.push(genTs)
-
-	// the myriads of stores
-	ds := dssync.MutexWrap(datastore.NewMapDatastore())
-	hot := blockstore.NewMemorySync()
-	cold := blockstore.NewMemorySync()
-
-	// put the genesis block to cold store
-	blk, err := genBlock.ToStorageBlock()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = cold.Put(blk)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// open the splitstore
-	ss, err := Open("", ds, hot, cold, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer ss.Close() //nolint
-
-	err = ss.Start(chain)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// make some tipsets, but not enough to cause compaction
-	mkBlock := func(curTs *types.TipSet, i int) *types.TipSet {
-		blk := mock.MkBlock(curTs, uint64(i), uint64(i))
-		sblk, err := blk.ToStorageBlock()
-		if err != nil {
-			t.Fatal(err)
-		}
-		err = ss.Put(sblk)
-		if err != nil {
-			t.Fatal(err)
-		}
-		ts := mock.TipSet(blk)
-		chain.push(ts)
-
-		return ts
-	}
-
-	mkGarbageBlock := func(curTs *types.TipSet, i int) {
-		blk := mock.MkBlock(curTs, uint64(i), uint64(i))
-		sblk, err := blk.ToStorageBlock()
-		if err != nil {
-			t.Fatal(err)
-		}
-		err = ss.Put(sblk)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	waitForCompaction := func() {
-		for atomic.LoadInt32(&ss.compacting) == 1 {
-			time.Sleep(100 * time.Millisecond)
-		}
-	}
-
-	curTs := genTs
-	for i := 1; i < 5; i++ {
-		curTs = mkBlock(curTs, i)
-		waitForCompaction()
-	}
-
-	mkGarbageBlock(genTs, 1)
-
-	// count objects in the cold and hot stores
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	countBlocks := func(bs blockstore.Blockstore) int {
-		count := 0
-		ch, err := bs.AllKeysChan(ctx)
-		if err != nil {
-			t.Fatal(err)
-		}
-		for range ch {
-			count++
-		}
-		return count
-	}
-
-	coldCnt := countBlocks(cold)
-	hotCnt := countBlocks(hot)
-
-	if coldCnt != 1 {
-		t.Errorf("expected %d blocks, but got %d", 1, coldCnt)
-	}
-
-	if hotCnt != 5 {
-		t.Errorf("expected %d blocks, but got %d", 5, hotCnt)
-	}
-
-	// trigger a compaction
-	for i := 5; i < 10; i++ {
-		curTs = mkBlock(curTs, i)
-		waitForCompaction()
-	}
-
-	coldCnt = countBlocks(cold)
-	hotCnt = countBlocks(hot)
-
-	if !cfg.EnableFullCompaction {
-		if coldCnt != 5 {
-			t.Errorf("expected %d cold blocks, but got %d", 5, coldCnt)
-		}
-
-		if hotCnt != 5 {
-			t.Errorf("expected %d hot blocks, but got %d", 5, hotCnt)
-		}
-	}
-
-	if cfg.EnableFullCompaction && !cfg.EnableGC {
-		if coldCnt != 3 {
-			t.Errorf("expected %d cold blocks, but got %d", 3, coldCnt)
-		}
-
-		if hotCnt != 7 {
-			t.Errorf("expected %d hot blocks, but got %d", 7, hotCnt)
-		}
-	}
-
-	if cfg.EnableFullCompaction && cfg.EnableGC {
-		if coldCnt != 2 {
-			t.Errorf("expected %d cold blocks, but got %d", 2, coldCnt)
-		}
-
-		if hotCnt != 7 {
-			t.Errorf("expected %d hot blocks, but got %d", 7, hotCnt)
-		}
-	}
-}
-
-func TestSplitStoreSimpleCompaction(t *testing.T) {
-	testSplitStore(t, &Config{TrackingStoreType: "mem"})
-}
-
-func TestSplitStoreFullCompactionWithoutGC(t *testing.T) {
-	testSplitStore(t, &Config{
-		TrackingStoreType:    "mem",
-		EnableFullCompaction: true,
-	})
-}
-
-func TestSplitStoreFullCompactionWithGC(t *testing.T) {
-	testSplitStore(t, &Config{
-		TrackingStoreType:    "mem",
-		EnableFullCompaction: true,
-		EnableGC:             true,
-	})
-}
-
-type mockChain struct {
-	sync.Mutex
-	tipsets  []*types.TipSet
-	listener func(revert []*types.TipSet, apply []*types.TipSet) error
-}
-
-func (c *mockChain) push(ts *types.TipSet) {
-	c.Lock()
-	c.tipsets = append(c.tipsets, ts)
-	c.Unlock()
-
-	if c.listener != nil {
-		err := c.listener(nil, []*types.TipSet{ts})
-		if err != nil {
-			log.Errorf("mockchain: error dispatching listener: %s", err)
-		}
-	}
-}
-
-func (c *mockChain) GetTipsetByHeight(_ context.Context, epoch abi.ChainEpoch, _ *types.TipSet, _ bool) (*types.TipSet, error) {
-	c.Lock()
-	defer c.Unlock()
-
-	iEpoch := int(epoch)
-	if iEpoch > len(c.tipsets) {
-		return nil, fmt.Errorf("bad epoch %d", epoch)
-	}
-
-	return c.tipsets[iEpoch-1], nil
-}
-
-func (c *mockChain) GetHeaviestTipSet() *types.TipSet {
-	c.Lock()
-	defer c.Unlock()
-
-	return c.tipsets[len(c.tipsets)-1]
-}
-
-func (c *mockChain) SubscribeHeadChanges(change func(revert []*types.TipSet, apply []*types.TipSet) error) {
-	c.listener = change
-}
-
-func (c *mockChain) WalkSnapshot(_ context.Context, ts *types.TipSet, epochs abi.ChainEpoch, _ bool, _ bool, f func(cid.Cid) error) error {
-	c.Lock()
-	defer c.Unlock()
-
-	start := int(ts.Height()) - 1
-	end := start - int(epochs)
-	if end < 0 {
-		end = -1
-	}
-	for i := start; i > end; i-- {
-		ts := c.tipsets[i]
-		for _, cid := range ts.Cids() {
-			err := f(cid)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}

blockstore/splitstore/tracking.go

@@ -1,109 +0,0 @@
-package splitstore
-
-import (
-	"path/filepath"
-	"sync"
-
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	cid "github.com/ipfs/go-cid"
-)
-
-// TrackingStore is a persistent store that tracks blocks that are added
-// to the hotstore, tracking the epoch at which they are written.
-type TrackingStore interface {
-	Put(cid.Cid, abi.ChainEpoch) error
-	PutBatch([]cid.Cid, abi.ChainEpoch) error
-	Get(cid.Cid) (abi.ChainEpoch, error)
-	Delete(cid.Cid) error
-	DeleteBatch([]cid.Cid) error
-	ForEach(func(cid.Cid, abi.ChainEpoch) error) error
-	Sync() error
-	Close() error
-}
-
-// OpenTrackingStore opens a tracking store of the specified type in the
-// specified path.
-func OpenTrackingStore(path string, ttype string) (TrackingStore, error) {
-	switch ttype {
-	case "", "bolt":
-		return OpenBoltTrackingStore(filepath.Join(path, "tracker.bolt"))
-	case "mem":
-		return NewMemTrackingStore(), nil
-	default:
-		return nil, xerrors.Errorf("unknown tracking store type %s", ttype)
-	}
-}
-
-// NewMemTrackingStore creates an in-memory tracking store.
-// This is only useful for test or situations where you don't want to open the
-// real tracking store (eg concurrent read only access on a node's datastore)
-func NewMemTrackingStore() *MemTrackingStore {
-	return &MemTrackingStore{tab: make(map[cid.Cid]abi.ChainEpoch)}
-}
-
-// MemTrackingStore is a simple in-memory tracking store
-type MemTrackingStore struct {
-	sync.Mutex
-	tab map[cid.Cid]abi.ChainEpoch
-}
-
-var _ TrackingStore = (*MemTrackingStore)(nil)
-
-func (s *MemTrackingStore) Put(cid cid.Cid, epoch abi.ChainEpoch) error {
-	s.Lock()
-	defer s.Unlock()
-	s.tab[cid] = epoch
-	return nil
-}
-
-func (s *MemTrackingStore) PutBatch(cids []cid.Cid, epoch abi.ChainEpoch) error {
-	s.Lock()
-	defer s.Unlock()
-	for _, cid := range cids {
-		s.tab[cid] = epoch
-	}
-	return nil
-}
-
-func (s *MemTrackingStore) Get(cid cid.Cid) (abi.ChainEpoch, error) {
-	s.Lock()
-	defer s.Unlock()
-	epoch, ok := s.tab[cid]
-	if ok {
-		return epoch, nil
-	}
-	return 0, xerrors.Errorf("missing tracking epoch for %s", cid)
-}
-
-func (s *MemTrackingStore) Delete(cid cid.Cid) error {
-	s.Lock()
-	defer s.Unlock()
-	delete(s.tab, cid)
-	return nil
-}
-
-func (s *MemTrackingStore) DeleteBatch(cids []cid.Cid) error {
-	s.Lock()
-	defer s.Unlock()
-	for _, cid := range cids {
-		delete(s.tab, cid)
-	}
-	return nil
-}
-
-func (s *MemTrackingStore) ForEach(f func(cid.Cid, abi.ChainEpoch) error) error {
-	s.Lock()
-	defer s.Unlock()
-	for cid, epoch := range s.tab {
-		err := f(cid, epoch)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (s *MemTrackingStore) Sync() error  { return nil }
-func (s *MemTrackingStore) Close() error { return nil }

blockstore/splitstore/tracking_bolt.go

@@ -1,120 +0,0 @@
-package splitstore
-
-import (
-	"time"
-
-	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
-	bolt "go.etcd.io/bbolt"
-
-	"github.com/filecoin-project/go-state-types/abi"
-)
-
-type BoltTrackingStore struct {
-	db       *bolt.DB
-	bucketId []byte
-}
-
-var _ TrackingStore = (*BoltTrackingStore)(nil)
-
-func OpenBoltTrackingStore(path string) (*BoltTrackingStore, error) {
-	opts := &bolt.Options{
-		Timeout: 1 * time.Second,
-		NoSync:  true,
-	}
-	db, err := bolt.Open(path, 0644, opts)
-	if err != nil {
-		return nil, err
-	}
-
-	bucketId := []byte("tracker")
-	err = db.Update(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists(bucketId)
-		if err != nil {
-			return xerrors.Errorf("error creating bolt db bucket %s: %w", string(bucketId), err)
-		}
-		return nil
-	})
-
-	if err != nil {
-		_ = db.Close()
-		return nil, err
-	}
-
-	return &BoltTrackingStore{db: db, bucketId: bucketId}, nil
-}
-
-func (s *BoltTrackingStore) Put(cid cid.Cid, epoch abi.ChainEpoch) error {
-	val := epochToBytes(epoch)
-	return s.db.Batch(func(tx *bolt.Tx) error {
-		b := tx.Bucket(s.bucketId)
-		return b.Put(cid.Hash(), val)
-	})
-}
-
-func (s *BoltTrackingStore) PutBatch(cids []cid.Cid, epoch abi.ChainEpoch) error {
-	val := epochToBytes(epoch)
-	return s.db.Batch(func(tx *bolt.Tx) error {
-		b := tx.Bucket(s.bucketId)
-		for _, cid := range cids {
-			err := b.Put(cid.Hash(), val)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	})
-}
-
-func (s *BoltTrackingStore) Get(cid cid.Cid) (epoch abi.ChainEpoch, err error) {
-	err = s.db.View(func(tx *bolt.Tx) error {
-		b := tx.Bucket(s.bucketId)
-		val := b.Get(cid.Hash())
-		if val == nil {
-			return xerrors.Errorf("missing tracking epoch for %s", cid)
-		}
-		epoch = bytesToEpoch(val)
-		return nil
-	})
-	return epoch, err
-}
-
-func (s *BoltTrackingStore) Delete(cid cid.Cid) error {
-	return s.db.Batch(func(tx *bolt.Tx) error {
-		b := tx.Bucket(s.bucketId)
-		return b.Delete(cid.Hash())
-	})
-}
-
-func (s *BoltTrackingStore) DeleteBatch(cids []cid.Cid) error {
-	return s.db.Batch(func(tx *bolt.Tx) error {
-		b := tx.Bucket(s.bucketId)
-		for _, cid := range cids {
-			err := b.Delete(cid.Hash())
-			if err != nil {
-				return xerrors.Errorf("error deleting %s", cid)
-			}
-		}
-		return nil
-	})
-}
-
-func (s *BoltTrackingStore) ForEach(f func(cid.Cid, abi.ChainEpoch) error) error {
-	return s.db.View(func(tx *bolt.Tx) error {
-		b := tx.Bucket(s.bucketId)
-		return b.ForEach(func(k, v []byte) error {
-			cid := cid.NewCidV1(cid.Raw, k)
-			epoch := bytesToEpoch(v)
-			return f(cid, epoch)
-		})
-	})
-}
-
-func (s *BoltTrackingStore) Sync() error {
-	return s.db.Sync()
-}
-
-func (s *BoltTrackingStore) Close() error {
-	return s.db.Close()
-}

blockstore/splitstore/tracking_test.go

@@ -1,130 +0,0 @@
-package splitstore
-
-import (
-	"io/ioutil"
-	"testing"
-
-	cid "github.com/ipfs/go-cid"
-	"github.com/multiformats/go-multihash"
-
-	"github.com/filecoin-project/go-state-types/abi"
-)
-
-func TestBoltTrackingStore(t *testing.T) {
-	testTrackingStore(t, "bolt")
-}
-
-func testTrackingStore(t *testing.T, tsType string) {
-	t.Helper()
-
-	makeCid := func(key string) cid.Cid {
-		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		return cid.NewCidV1(cid.Raw, h)
-	}
-
-	mustHave := func(s TrackingStore, cid cid.Cid, epoch abi.ChainEpoch) {
-		val, err := s.Get(cid)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if val != epoch {
-			t.Fatal("epoch mismatch")
-		}
-	}
-
-	mustNotHave := func(s TrackingStore, cid cid.Cid) {
-		_, err := s.Get(cid)
-		if err == nil {
-			t.Fatal("expected error")
-		}
-	}
-
-	path, err := ioutil.TempDir("", "snoop-test.*")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	s, err := OpenTrackingStore(path, tsType)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	k1 := makeCid("a")
-	k2 := makeCid("b")
-	k3 := makeCid("c")
-	k4 := makeCid("d")
-
-	s.Put(k1, 1) //nolint
-	s.Put(k2, 2) //nolint
-	s.Put(k3, 3) //nolint
-	s.Put(k4, 4) //nolint
-
-	mustHave(s, k1, 1)
-	mustHave(s, k2, 2)
-	mustHave(s, k3, 3)
-	mustHave(s, k4, 4)
-
-	s.Delete(k1) // nolint
-	s.Delete(k2) // nolint
-
-	mustNotHave(s, k1)
-	mustNotHave(s, k2)
-	mustHave(s, k3, 3)
-	mustHave(s, k4, 4)
-
-	s.PutBatch([]cid.Cid{k1}, 1) //nolint
-	s.PutBatch([]cid.Cid{k2}, 2) //nolint
-
-	mustHave(s, k1, 1)
-	mustHave(s, k2, 2)
-	mustHave(s, k3, 3)
-	mustHave(s, k4, 4)
-
-	allKeys := map[string]struct{}{
-		k1.String(): {},
-		k2.String(): {},
-		k3.String(): {},
-		k4.String(): {},
-	}
-
-	err = s.ForEach(func(k cid.Cid, _ abi.ChainEpoch) error {
-		_, ok := allKeys[k.String()]
-		if !ok {
-			t.Fatal("unexpected key")
-		}
-
-		delete(allKeys, k.String())
-		return nil
-	})
-
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if len(allKeys) != 0 {
-		t.Fatal("not all keys were returned")
-	}
-
-	// no close and reopen and ensure the keys still exist
-	err = s.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	s, err = OpenTrackingStore(path, tsType)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	mustHave(s, k1, 1)
-	mustHave(s, k2, 2)
-	mustHave(s, k3, 3)
-	mustHave(s, k4, 4)
-
-	s.Close() //nolint:errcheck
-}

blockstore/sync.go

@@ -1,81 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"sync"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-)
-
-// NewMemorySync returns a thread-safe in-memory blockstore.
-func NewMemorySync() *SyncBlockstore {
-	return &SyncBlockstore{bs: make(MemBlockstore)}
-}
-
-// SyncBlockstore is a terminal blockstore that is a synchronized version
-// of MemBlockstore.
-type SyncBlockstore struct {
-	mu sync.RWMutex
-	bs MemBlockstore // specifically use a memStore to save indirection overhead.
-}
-
-func (m *SyncBlockstore) DeleteBlock(k cid.Cid) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return m.bs.DeleteBlock(k)
-}
-
-func (m *SyncBlockstore) DeleteMany(ks []cid.Cid) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return m.bs.DeleteMany(ks)
-}
-
-func (m *SyncBlockstore) Has(k cid.Cid) (bool, error) {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	return m.bs.Has(k)
-}
-
-func (m *SyncBlockstore) View(k cid.Cid, callback func([]byte) error) error {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-
-	return m.bs.View(k, callback)
-}
-
-func (m *SyncBlockstore) Get(k cid.Cid) (blocks.Block, error) {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	return m.bs.Get(k)
-}
-
-func (m *SyncBlockstore) GetSize(k cid.Cid) (int, error) {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	return m.bs.GetSize(k)
-}
-
-func (m *SyncBlockstore) Put(b blocks.Block) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return m.bs.Put(b)
-}
-
-func (m *SyncBlockstore) PutMany(bs []blocks.Block) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return m.bs.PutMany(bs)
-}
-
-func (m *SyncBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	// this blockstore implementation doesn't do any async work.
-	return m.bs.AllKeysChan(ctx)
-}
-
-func (m *SyncBlockstore) HashOnRead(enabled bool) {
-	// noop
-}

blockstore/timed.go

@@ -1,178 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"fmt"
-	"sync"
-	"time"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	"github.com/raulk/clock"
-	"go.uber.org/multierr"
-)
-
-// TimedCacheBlockstore is a blockstore that keeps blocks for at least the
-// specified caching interval before discarding them. Garbage collection must
-// be started and stopped by calling Start/Stop.
-//
-// Under the covers, it's implemented with an active and an inactive blockstore
-// that are rotated every cache time interval. This means all blocks will be
-// stored at most 2x the cache interval.
-//
-// Create a new instance by calling the NewTimedCacheBlockstore constructor.
-type TimedCacheBlockstore struct {
-	mu               sync.RWMutex
-	active, inactive MemBlockstore
-	clock            clock.Clock
-	interval         time.Duration
-	closeCh          chan struct{}
-	doneRotatingCh   chan struct{}
-}
-
-func NewTimedCacheBlockstore(interval time.Duration) *TimedCacheBlockstore {
-	b := &TimedCacheBlockstore{
-		active:   NewMemory(),
-		inactive: NewMemory(),
-		interval: interval,
-		clock:    clock.New(),
-	}
-	return b
-}
-
-func (t *TimedCacheBlockstore) Start(_ context.Context) error {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	if t.closeCh != nil {
-		return fmt.Errorf("already started")
-	}
-	t.closeCh = make(chan struct{})
-	go func() {
-		ticker := t.clock.Ticker(t.interval)
-		defer ticker.Stop()
-		for {
-			select {
-			case <-ticker.C:
-				t.rotate()
-				if t.doneRotatingCh != nil {
-					t.doneRotatingCh <- struct{}{}
-				}
-			case <-t.closeCh:
-				return
-			}
-		}
-	}()
-	return nil
-}
-
-func (t *TimedCacheBlockstore) Stop(_ context.Context) error {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	if t.closeCh == nil {
-		return fmt.Errorf("not started")
-	}
-	select {
-	case <-t.closeCh:
-		// already closed
-	default:
-		close(t.closeCh)
-	}
-	return nil
-}
-
-func (t *TimedCacheBlockstore) rotate() {
-	newBs := NewMemory()
-
-	t.mu.Lock()
-	t.inactive, t.active = t.active, newBs
-	t.mu.Unlock()
-}
-
-func (t *TimedCacheBlockstore) Put(b blocks.Block) error {
-	// Don't check the inactive set here. We want to keep this block for at
-	// least one interval.
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	return t.active.Put(b)
-}
-
-func (t *TimedCacheBlockstore) PutMany(bs []blocks.Block) error {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	return t.active.PutMany(bs)
-}
-
-func (t *TimedCacheBlockstore) View(k cid.Cid, callback func([]byte) error) error {
-	t.mu.RLock()
-	defer t.mu.RUnlock()
-	err := t.active.View(k, callback)
-	if err == ErrNotFound {
-		err = t.inactive.View(k, callback)
-	}
-	return err
-}
-
-func (t *TimedCacheBlockstore) Get(k cid.Cid) (blocks.Block, error) {
-	t.mu.RLock()
-	defer t.mu.RUnlock()
-	b, err := t.active.Get(k)
-	if err == ErrNotFound {
-		b, err = t.inactive.Get(k)
-	}
-	return b, err
-}
-
-func (t *TimedCacheBlockstore) GetSize(k cid.Cid) (int, error) {
-	t.mu.RLock()
-	defer t.mu.RUnlock()
-	size, err := t.active.GetSize(k)
-	if err == ErrNotFound {
-		size, err = t.inactive.GetSize(k)
-	}
-	return size, err
-}
-
-func (t *TimedCacheBlockstore) Has(k cid.Cid) (bool, error) {
-	t.mu.RLock()
-	defer t.mu.RUnlock()
-	if has, err := t.active.Has(k); err != nil {
-		return false, err
-	} else if has {
-		return true, nil
-	}
-	return t.inactive.Has(k)
-}
-
-func (t *TimedCacheBlockstore) HashOnRead(_ bool) {
-	// no-op
-}
-
-func (t *TimedCacheBlockstore) DeleteBlock(k cid.Cid) error {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	return multierr.Combine(t.active.DeleteBlock(k), t.inactive.DeleteBlock(k))
-}
-
-func (t *TimedCacheBlockstore) DeleteMany(ks []cid.Cid) error {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	return multierr.Combine(t.active.DeleteMany(ks), t.inactive.DeleteMany(ks))
-}
-
-func (t *TimedCacheBlockstore) AllKeysChan(_ context.Context) (<-chan cid.Cid, error) {
-	t.mu.RLock()
-	defer t.mu.RUnlock()
-
-	ch := make(chan cid.Cid, len(t.active)+len(t.inactive))
-	for c := range t.active {
-		ch <- c
-	}
-	for c := range t.inactive {
-		if _, ok := t.active[c]; ok {
-			continue
-		}
-		ch <- c
-	}
-	close(ch)
-	return ch, nil
-}

blockstore/timed_test.go

@@ -1,85 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"github.com/raulk/clock"
-	"github.com/stretchr/testify/require"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-)
-
-func TestTimedCacheBlockstoreSimple(t *testing.T) {
-	tc := NewTimedCacheBlockstore(10 * time.Millisecond)
-	mClock := clock.NewMock()
-	mClock.Set(time.Now())
-	tc.clock = mClock
-	tc.doneRotatingCh = make(chan struct{})
-
-	_ = tc.Start(context.Background())
-	mClock.Add(1) // IDK why it is needed but it makes it work
-
-	defer func() {
-		_ = tc.Stop(context.Background())
-	}()
-
-	b1 := blocks.NewBlock([]byte("foo"))
-	require.NoError(t, tc.Put(b1))
-
-	b2 := blocks.NewBlock([]byte("bar"))
-	require.NoError(t, tc.Put(b2))
-
-	b3 := blocks.NewBlock([]byte("baz"))
-
-	b1out, err := tc.Get(b1.Cid())
-	require.NoError(t, err)
-	require.Equal(t, b1.RawData(), b1out.RawData())
-
-	has, err := tc.Has(b1.Cid())
-	require.NoError(t, err)
-	require.True(t, has)
-
-	mClock.Add(10 * time.Millisecond)
-	<-tc.doneRotatingCh
-
-	// We should still have everything.
-	has, err = tc.Has(b1.Cid())
-	require.NoError(t, err)
-	require.True(t, has)
-
-	has, err = tc.Has(b2.Cid())
-	require.NoError(t, err)
-	require.True(t, has)
-
-	// extend b2, add b3.
-	require.NoError(t, tc.Put(b2))
-	require.NoError(t, tc.Put(b3))
-
-	// all keys once.
-	allKeys, err := tc.AllKeysChan(context.Background())
-	var ks []cid.Cid
-	for k := range allKeys {
-		ks = append(ks, k)
-	}
-	require.NoError(t, err)
-	require.ElementsMatch(t, ks, []cid.Cid{b1.Cid(), b2.Cid(), b3.Cid()})
-
-	mClock.Add(10 * time.Millisecond)
-	<-tc.doneRotatingCh
-	// should still have b2, and b3, but not b1
-
-	has, err = tc.Has(b1.Cid())
-	require.NoError(t, err)
-	require.False(t, has)
-
-	has, err = tc.Has(b2.Cid())
-	require.NoError(t, err)
-	require.True(t, has)
-
-	has, err = tc.Has(b3.Cid())
-	require.NoError(t, err)
-	require.True(t, has)
-}

blockstore/union.go

@@ -1,119 +0,0 @@
-package blockstore
-
-import (
-	"context"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-)
-
-type unionBlockstore []Blockstore
-
-// Union returns an unioned blockstore.
-//
-// * Reads return from the first blockstore that has the value, querying in the
-//   supplied order.
-// * Writes (puts and deltes) are broadcast to all stores.
-//
-func Union(stores ...Blockstore) Blockstore {
-	return unionBlockstore(stores)
-}
-
-func (m unionBlockstore) Has(cid cid.Cid) (has bool, err error) {
-	for _, bs := range m {
-		if has, err = bs.Has(cid); has || err != nil {
-			break
-		}
-	}
-	return has, err
-}
-
-func (m unionBlockstore) Get(cid cid.Cid) (blk blocks.Block, err error) {
-	for _, bs := range m {
-		if blk, err = bs.Get(cid); err == nil || err != ErrNotFound {
-			break
-		}
-	}
-	return blk, err
-}
-
-func (m unionBlockstore) View(cid cid.Cid, callback func([]byte) error) (err error) {
-	for _, bs := range m {
-		if err = bs.View(cid, callback); err == nil || err != ErrNotFound {
-			break
-		}
-	}
-	return err
-}
-
-func (m unionBlockstore) GetSize(cid cid.Cid) (size int, err error) {
-	for _, bs := range m {
-		if size, err = bs.GetSize(cid); err == nil || err != ErrNotFound {
-			break
-		}
-	}
-	return size, err
-}
-
-func (m unionBlockstore) Put(block blocks.Block) (err error) {
-	for _, bs := range m {
-		if err = bs.Put(block); err != nil {
-			break
-		}
-	}
-	return err
-}
-
-func (m unionBlockstore) PutMany(blks []blocks.Block) (err error) {
-	for _, bs := range m {
-		if err = bs.PutMany(blks); err != nil {
-			break
-		}
-	}
-	return err
-}
-
-func (m unionBlockstore) DeleteBlock(cid cid.Cid) (err error) {
-	for _, bs := range m {
-		if err = bs.DeleteBlock(cid); err != nil {
-			break
-		}
-	}
-	return err
-}
-
-func (m unionBlockstore) DeleteMany(cids []cid.Cid) (err error) {
-	for _, bs := range m {
-		if err = bs.DeleteMany(cids); err != nil {
-			break
-		}
-	}
-	return err
-}
-
-func (m unionBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	// this does not deduplicate; this interface needs to be revisited.
-	outCh := make(chan cid.Cid)
-
-	go func() {
-		defer close(outCh)
-
-		for _, bs := range m {
-			ch, err := bs.AllKeysChan(ctx)
-			if err != nil {
-				return
-			}
-			for cid := range ch {
-				outCh <- cid
-			}
-		}
-	}()
-
-	return outCh, nil
-}
-
-func (m unionBlockstore) HashOnRead(enabled bool) {
-	for _, bs := range m {
-		bs.HashOnRead(enabled)
-	}
-}

blockstore/union_test.go

@@ -1,102 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"testing"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/stretchr/testify/require"
-)
-
-var (
-	b0 = blocks.NewBlock([]byte("abc"))
-	b1 = blocks.NewBlock([]byte("foo"))
-	b2 = blocks.NewBlock([]byte("bar"))
-)
-
-func TestUnionBlockstore_Get(t *testing.T) {
-	m1 := NewMemory()
-	m2 := NewMemory()
-
-	_ = m1.Put(b1)
-	_ = m2.Put(b2)
-
-	u := Union(m1, m2)
-
-	v1, err := u.Get(b1.Cid())
-	require.NoError(t, err)
-	require.Equal(t, b1.RawData(), v1.RawData())
-
-	v2, err := u.Get(b2.Cid())
-	require.NoError(t, err)
-	require.Equal(t, b2.RawData(), v2.RawData())
-}
-
-func TestUnionBlockstore_Put_PutMany_Delete_AllKeysChan(t *testing.T) {
-	m1 := NewMemory()
-	m2 := NewMemory()
-
-	u := Union(m1, m2)
-
-	err := u.Put(b0)
-	require.NoError(t, err)
-
-	var has bool
-
-	// write was broadcasted to all stores.
-	has, _ = m1.Has(b0.Cid())
-	require.True(t, has)
-
-	has, _ = m2.Has(b0.Cid())
-	require.True(t, has)
-
-	has, _ = u.Has(b0.Cid())
-	require.True(t, has)
-
-	// put many.
-	err = u.PutMany([]blocks.Block{b1, b2})
-	require.NoError(t, err)
-
-	// write was broadcasted to all stores.
-	has, _ = m1.Has(b1.Cid())
-	require.True(t, has)
-
-	has, _ = m1.Has(b2.Cid())
-	require.True(t, has)
-
-	has, _ = m2.Has(b1.Cid())
-	require.True(t, has)
-
-	has, _ = m2.Has(b2.Cid())
-	require.True(t, has)
-
-	// also in the union store.
-	has, _ = u.Has(b1.Cid())
-	require.True(t, has)
-
-	has, _ = u.Has(b2.Cid())
-	require.True(t, has)
-
-	// deleted from all stores.
-	err = u.DeleteBlock(b1.Cid())
-	require.NoError(t, err)
-
-	has, _ = u.Has(b1.Cid())
-	require.False(t, has)
-
-	has, _ = m1.Has(b1.Cid())
-	require.False(t, has)
-
-	has, _ = m2.Has(b1.Cid())
-	require.False(t, has)
-
-	// check that AllKeysChan returns b0 and b2, twice (once per backing store)
-	ch, err := u.AllKeysChan(context.Background())
-	require.NoError(t, err)
-
-	var i int
-	for range ch {
-		i++
-	}
-	require.Equal(t, 4, i)
-}

build/bootstrap.go

@@ -2,29 +2,44 @@ package build
 
 import (
 	"context"
+	"os"
 	"strings"
 
 	"github.com/filecoin-project/lotus/lib/addrutil"
+	"golang.org/x/xerrors"
 
 	rice "github.com/GeertJohan/go.rice"
 	"github.com/libp2p/go-libp2p-core/peer"
 )
 
 func BuiltinBootstrap() ([]peer.AddrInfo, error) {
-	if DisableBuiltinAssets {
-		return nil, nil
-	}
+	var out []peer.AddrInfo
 
 	b := rice.MustFindBox("bootstrap")
-
-	if BootstrappersFile != "" {
-		spi := b.MustString(BootstrappersFile)
-		if spi == "" {
-			return nil, nil
+	err := b.Walk("", func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return xerrors.Errorf("failed to walk box: %w", err)
 		}
 
-		return addrutil.ParseAddresses(context.TODO(), strings.Split(strings.TrimSpace(spi), "\n"))
+		if !strings.HasSuffix(path, ".pi") {
+			return nil
+		}
+		spi := b.MustString(path)
+		if spi == "" {
+			return nil
+		}
+		pi, err := addrutil.ParseAddresses(context.TODO(), strings.Split(strings.TrimSpace(spi), "\n"))
+		out = append(out, pi...)
+		return err
+	})
+	return out, err
+}
+
+func DrandBootstrap() ([]peer.AddrInfo, error) {
+	addrs := []string{
+		"/dnsaddr/pl-eu.testnet.drand.sh/",
+		"/dnsaddr/pl-us.testnet.drand.sh/",
+		"/dnsaddr/pl-sin.testnet.drand.sh/",
 	}
-
-	return nil, nil
+	return addrutil.ParseAddresses(context.TODO(), addrs)
 }

build/bootstrap/bootstrappers.pi

@@ -0,0 +1,6 @@
+/dns4/t01000.miner.interopnet.kittyhawk.wtf/tcp/1347/p2p/12D3KooWLVgeFhXA7ztuSS1uvk4NVzZwRZLxg8PBy1mJuHfc8J6E
+/ip4/34.217.110.132/tcp/1347/p2p/12D3KooWLVgeFhXA7ztuSS1uvk4NVzZwRZLxg8PBy1mJuHfc8J6E
+/dns4/peer0.interopnet.kittyhawk.wtf/tcp/1347/p2p/12D3KooWBkXWpT1hLHXz7VzR8ZHCJ42P6c9Nb9532ATSguBpLTKJ
+/ip4/54.187.182.170/tcp/1347/p2p/12D3KooWBkXWpT1hLHXz7VzR8ZHCJ42P6c9Nb9532ATSguBpLTKJ
+/dns4/peer1.interopnet.kittyhawk.wtf/tcp/1347/p2p/12D3KooWJk5hQHNdCsidtX4jhKQMBXKJdo2kjhrNHUZBPmCZKwKt
+/ip4/52.24.84.39/tcp/1347/p2p/12D3KooWJk5hQHNdCsidtX4jhKQMBXKJdo2kjhrNHUZBPmCZKwKt

build/bootstrap/butterflynet.pi

@@ -1,2 +0,0 @@
-/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWQafkXgEWDgcVhZvF6KMhiC8ktdxjvmdQN8RarRXe9jCc
-/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWE7UmZ4DLk9WBdEJUSwuSCPiSqjoCv3wPeoe8Tq3yMa77

build/bootstrap/calibnet.pi

@@ -1,4 +0,0 @@
-/dns4/bootstrap-0.calibration.fildev.network/tcp/1347/p2p/12D3KooWRLZAseMo9h7fRD6ojn6YYDXHsBSavX5YmjBZ9ngtAEec
-/dns4/bootstrap-1.calibration.fildev.network/tcp/1347/p2p/12D3KooWJFtDXgZEQMEkjJPSrbfdvh2xfjVKrXeNFG1t8ioJXAzv
-/dns4/bootstrap-2.calibration.fildev.network/tcp/1347/p2p/12D3KooWP1uB9Lo7yCA3S17TD4Y5wStP5Nk7Vqh53m8GsFjkyujD
-/dns4/bootstrap-3.calibration.fildev.network/tcp/1347/p2p/12D3KooWLrPM4WPK1YRGPCUwndWcDX8GCYgms3DiuofUmxwvhMCn

build/bootstrap/mainnet.pi

@@ -1,15 +0,0 @@
-/dns4/bootstrap-0.mainnet.filops.net/tcp/1347/p2p/12D3KooWCVe8MmsEMes2FzgTpt9fXtmCY7wrq91GRiaC8PHSCCBj
-/dns4/bootstrap-1.mainnet.filops.net/tcp/1347/p2p/12D3KooWCwevHg1yLCvktf2nvLu7L9894mcrJR4MsBCcm4syShVc
-/dns4/bootstrap-2.mainnet.filops.net/tcp/1347/p2p/12D3KooWEWVwHGn2yR36gKLozmb4YjDJGerotAPGxmdWZx2nxMC4
-/dns4/bootstrap-3.mainnet.filops.net/tcp/1347/p2p/12D3KooWKhgq8c7NQ9iGjbyK7v7phXvG6492HQfiDaGHLHLQjk7R
-/dns4/bootstrap-4.mainnet.filops.net/tcp/1347/p2p/12D3KooWL6PsFNPhYftrJzGgF5U18hFoaVhfGk7xwzD8yVrHJ3Uc
-/dns4/bootstrap-5.mainnet.filops.net/tcp/1347/p2p/12D3KooWLFynvDQiUpXoHroV1YxKHhPJgysQGH2k3ZGwtWzR4dFH
-/dns4/bootstrap-6.mainnet.filops.net/tcp/1347/p2p/12D3KooWP5MwCiqdMETF9ub1P3MbCvQCcfconnYHbWg6sUJcDRQQ
-/dns4/bootstrap-7.mainnet.filops.net/tcp/1347/p2p/12D3KooWRs3aY1p3juFjPy8gPN95PEQChm2QKGUCAdcDCC4EBMKf
-/dns4/bootstrap-8.mainnet.filops.net/tcp/1347/p2p/12D3KooWScFR7385LTyR4zU1bYdzSiiAb5rnNABfVahPvVSzyTkR
-/dns4/lotus-bootstrap.forceup.cn/tcp/41778/p2p/12D3KooWFQsv3nRMUevZNWWsY1Wu6NUzUbawnWU5NcRhgKuJA37C
-/dns4/bootstrap-0.starpool.in/tcp/12757/p2p/12D3KooWGHpBMeZbestVEWkfdnC9u7p6uFHXL1n7m1ZBqsEmiUzz
-/dns4/bootstrap-1.starpool.in/tcp/12757/p2p/12D3KooWQZrGH1PxSNZPum99M1zNvjNFM33d1AAu5DcvdHptuU7u
-/dns4/node.glif.io/tcp/1235/p2p/12D3KooWBF8cpp65hp2u9LK5mh19x67ftAam84z9LsfaquTDSBpt
-/dns4/bootstrap-0.ipfsmain.cn/tcp/34721/p2p/12D3KooWQnwEGNqcM2nAcPtRR9rAX8Hrg4k9kJLCHoTR5chJfz6d
-/dns4/bootstrap-1.ipfsmain.cn/tcp/34723/p2p/12D3KooWMKxMkD5DMpSWsW7dBddKxKT7L2GgbNuckz9otxvkvByP

build/bootstrap/nerpanet.pi

@@ -1,4 +0,0 @@
-/dns4/bootstrap-0.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWNfuGjtzWTVz8eJGZ2C3aJg2xLqorhsagu4LTWw6CwpK9
-/dns4/bootstrap-1.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWDfsxYk7dC6NNsHqZqqyMJCzkjZuXhjsmqBk3TUCBZLga
-/dns4/bootstrap-2.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWRZAGHmCCaa2gkYmnC4Q2TEwHGFSh6Fh1FFJ7RSXak5yN
-/dns4/bootstrap-3.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWBFxEigSKLvxJVdw3JziC9ePHHnyAn5LifWSqg2kttcth

build/clock.go

@@ -1,10 +0,0 @@
-package build
-
-import "github.com/raulk/clock"
-
-// Clock is the global clock for the system. In standard builds,
-// we use a real-time clock, which maps to the `time` package.
-//
-// Tests that need control of time can replace this variable with
-// clock.NewMock(). Always use real time for socket/stream deadlines.
-var Clock = clock.New()

build/drand.go

@@ -1,74 +0,0 @@
-package build
-
-import (
-	"sort"
-
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-)
-
-type DrandEnum int
-
-func DrandConfigSchedule() dtypes.DrandSchedule {
-	out := dtypes.DrandSchedule{}
-	for start, config := range DrandSchedule {
-		out = append(out, dtypes.DrandPoint{Start: start, Config: DrandConfigs[config]})
-	}
-
-	sort.Slice(out, func(i, j int) bool {
-		return out[i].Start < out[j].Start
-	})
-
-	return out
-}
-
-const (
-	DrandMainnet DrandEnum = iota + 1
-	DrandTestnet
-	DrandDevnet
-	DrandLocalnet
-	DrandIncentinet
-)
-
-var DrandConfigs = map[DrandEnum]dtypes.DrandConfig{
-	DrandMainnet: {
-		Servers: []string{
-			"https://api.drand.sh",
-			"https://api2.drand.sh",
-			"https://api3.drand.sh",
-			"https://drand.cloudflare.com",
-		},
-		Relays: []string{
-			"/dnsaddr/api.drand.sh/",
-			"/dnsaddr/api2.drand.sh/",
-			"/dnsaddr/api3.drand.sh/",
-		},
-		ChainInfoJSON: `{"public_key":"868f005eb8e6e4ca0a47c8a77ceaa5309a47978a7c71bc5cce96366b5d7a569937c529eeda66c7293784a9402801af31","period":30,"genesis_time":1595431050,"hash":"8990e7a9aaed2ffed73dbd7092123d6f289930540d7651336225dc172e51b2ce","groupHash":"176f93498eac9ca337150b46d21dd58673ea4e3581185f869672e59fa4cb390a"}`,
-	},
-	DrandTestnet: {
-		Servers: []string{
-			"https://pl-eu.testnet.drand.sh",
-			"https://pl-us.testnet.drand.sh",
-			"https://pl-sin.testnet.drand.sh",
-		},
-		Relays: []string{
-			"/dnsaddr/pl-eu.testnet.drand.sh/",
-			"/dnsaddr/pl-us.testnet.drand.sh/",
-			"/dnsaddr/pl-sin.testnet.drand.sh/",
-		},
-		ChainInfoJSON: `{"public_key":"922a2e93828ff83345bae533f5172669a26c02dc76d6bf59c80892e12ab1455c229211886f35bb56af6d5bea981024df","period":25,"genesis_time":1590445175,"hash":"84b2234fb34e835dccd048255d7ad3194b81af7d978c3bf157e3469592ae4e02","groupHash":"4dd408e5fdff9323c76a9b6f087ba8fdc5a6da907bd9217d9d10f2287d081957"}`,
-	},
-	DrandDevnet: {
-		Servers: []string{
-			"https://dev1.drand.sh",
-			"https://dev2.drand.sh",
-		},
-		Relays: []string{
-			"/dnsaddr/dev1.drand.sh/",
-			"/dnsaddr/dev2.drand.sh/",
-		},
-		ChainInfoJSON: `{"public_key":"8cda589f88914aa728fd183f383980b35789ce81b274e5daee1f338b77d02566ef4d3fb0098af1f844f10f9c803c1827","period":25,"genesis_time":1595348225,"hash":"e73b7dc3c4f6a236378220c0dd6aa110eb16eed26c11259606e07ee122838d4f","groupHash":"567d4785122a5a3e75a9bc9911d7ea807dd85ff76b78dc4ff06b075712898607"}`,
-	},
-	DrandIncentinet: {
-		ChainInfoJSON: `{"public_key":"8cad0c72c606ab27d36ee06de1d5b2db1faf92e447025ca37575ab3a8aac2eaae83192f846fc9e158bc738423753d000","period":30,"genesis_time":1595873820,"hash":"80c8b872c714f4c00fdd3daa465d5514049f457f01f85a4caf68cdcd394ba039","groupHash":"d9406aaed487f7af71851b4399448e311f2328923d454e971536c05398ce2d9b"}`,
-	},
-}

build/flags.go

@@ -1,15 +0,0 @@
-package build
-
-// DisableBuiltinAssets disables the resolution of go.rice boxes that store
-// built-in assets, such as proof parameters, bootstrap peers, genesis blocks,
-// etc.
-//
-// When this value is set to true, it is expected that the user will
-// provide any such configurations through the Lotus API itself.
-//
-// This is useful when you're using Lotus as a library, such as to orchestrate
-// test scenarios, or for other purposes where you don't need to use the
-// defaults shipped with the binary.
-//
-// For this flag to be effective, it must be enabled _before_ instantiating Lotus.
-var DisableBuiltinAssets = false

build/genesis.go

@@ -14,7 +14,7 @@ func MaybeGenesis() []byte {
 		log.Warnf("loading built-in genesis: %s", err)
 		return nil
 	}
-	genBytes, err := builtinGen.Bytes(GenesisFile)
+	genBytes, err := builtinGen.Bytes("devnet.car")
 	if err != nil {
 		log.Warnf("loading built-in genesis: %s", err)
 	}

build/isnearupgrade.go

@@ -1,9 +0,0 @@
-package build
-
-import (
-	"github.com/filecoin-project/go-state-types/abi"
-)
-
-func IsNearUpgrade(epoch, upgradeEpoch abi.ChainEpoch) bool {
-	return epoch > upgradeEpoch-Finality && epoch < upgradeEpoch+Finality
-}

build/params_2k.go

@@ -3,47 +3,27 @@
 package build
 
 import (
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/chain/actors/policy"
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
+	"github.com/filecoin-project/specs-actors/actors/builtin/power"
+	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
 )
 
-const BootstrappersFile = ""
-const GenesisFile = ""
-
-const UpgradeBreezeHeight = -1
-const BreezeGasTampingDuration = 0
-
-const UpgradeSmokeHeight = -1
-const UpgradeIgnitionHeight = -2
-const UpgradeRefuelHeight = -3
-const UpgradeTapeHeight = -4
-
-const UpgradeActorsV2Height = 10
-const UpgradeLiftoffHeight = -5
-
-const UpgradeKumquatHeight = 15
-const UpgradeCalicoHeight = 20
-const UpgradePersianHeight = 25
-const UpgradeOrangeHeight = 27
-const UpgradeClausHeight = 30
-
-const UpgradeActorsV3Height = 35
-
-var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
-	0: DrandMainnet,
-}
-
 func init() {
-	policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1)
-	policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048))
-	policy.SetMinVerifiedDealSize(abi.NewStoragePower(256))
+	power.ConsensusMinerMinPower = big.NewInt(2048)
+	miner.SupportedProofTypes = map[abi.RegisteredProof]struct{}{
+		abi.RegisteredProof_StackedDRG2KiBSeal: {},
+	}
+	verifreg.MinVerifiedDealSize = big.NewInt(256)
 
 	BuildType |= Build2k
 }
 
-const BlockDelaySecs = uint64(4)
+// Seconds
+const BlockDelay = 2
 
-const PropagationDelaySecs = uint64(1)
+const PropagationDelay = 3
 
 // SlashablePowerDelay is the number of epochs after ElectionPeriodStart, after
 // which the miner is slashed
@@ -53,5 +33,3 @@ const SlashablePowerDelay = 20
 
 // Epochs
 const InteractivePoRepConfidence = 6
-
-const BootstrapPeerThreshold = 1

build/params_butterfly.go

@@ -1,52 +0,0 @@
-// +build butterflynet
-
-package build
-
-import (
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/chain/actors/policy"
-	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
-)
-
-var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
-	0: DrandMainnet,
-}
-
-const BootstrappersFile = "butterflynet.pi"
-const GenesisFile = "butterflynet.car"
-
-const UpgradeBreezeHeight = -1
-const BreezeGasTampingDuration = 120
-const UpgradeSmokeHeight = -2
-const UpgradeIgnitionHeight = -3
-const UpgradeRefuelHeight = -4
-
-var UpgradeActorsV2Height = abi.ChainEpoch(30)
-
-const UpgradeTapeHeight = 60
-const UpgradeLiftoffHeight = -5
-const UpgradeKumquatHeight = 90
-const UpgradeCalicoHeight = 120
-const UpgradePersianHeight = 150
-const UpgradeClausHeight = 180
-const UpgradeOrangeHeight = 210
-const UpgradeActorsV3Height = 240
-
-func init() {
-	policy.SetConsensusMinerMinPower(abi.NewStoragePower(2 << 30))
-	policy.SetSupportedProofTypes(
-		abi.RegisteredSealProof_StackedDrg512MiBV1,
-	)
-
-	SetAddressNetwork(address.Testnet)
-
-	Devnet = true
-}
-
-const BlockDelaySecs = uint64(builtin2.EpochDurationSeconds)
-
-const PropagationDelaySecs = uint64(6)
-
-// BootstrapPeerThreshold is the minimum number peers we need to track for a sync worker to start
-const BootstrapPeerThreshold = 2

build/params_calibnet.go

@@ -1,63 +0,0 @@
-// +build calibnet
-
-package build
-
-import (
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/chain/actors/policy"
-	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
-)
-
-var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
-	0: DrandMainnet,
-}
-
-const BootstrappersFile = "calibnet.pi"
-const GenesisFile = "calibnet.car"
-
-const UpgradeBreezeHeight = -1
-const BreezeGasTampingDuration = 120
-
-const UpgradeSmokeHeight = -2
-
-const UpgradeIgnitionHeight = -3
-const UpgradeRefuelHeight = -4
-
-var UpgradeActorsV2Height = abi.ChainEpoch(30)
-
-const UpgradeTapeHeight = 60
-
-const UpgradeLiftoffHeight = -5
-
-const UpgradeKumquatHeight = 90
-
-const UpgradeCalicoHeight = 100
-const UpgradePersianHeight = UpgradeCalicoHeight + (builtin2.EpochsInHour * 1)
-
-const UpgradeClausHeight = 250
-
-const UpgradeOrangeHeight = 300
-
-const UpgradeActorsV3Height = 600
-
-func init() {
-	policy.SetConsensusMinerMinPower(abi.NewStoragePower(32 << 30))
-	policy.SetSupportedProofTypes(
-		abi.RegisteredSealProof_StackedDrg32GiBV1,
-		abi.RegisteredSealProof_StackedDrg64GiBV1,
-	)
-
-	SetAddressNetwork(address.Testnet)
-
-	Devnet = true
-
-	BuildType = BuildCalibnet
-}
-
-const BlockDelaySecs = uint64(builtin2.EpochDurationSeconds)
-
-const PropagationDelaySecs = uint64(6)
-
-// BootstrapPeerThreshold is the minimum number peers we need to track for a sync worker to start
-const BootstrapPeerThreshold = 4

build/params_mainnet.go

@@ -1,80 +0,0 @@
-// +build !debug
-// +build !2k
-// +build !testground
-// +build !calibnet
-// +build !nerpanet
-// +build !butterflynet
-
-package build
-
-import (
-	"math"
-	"os"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/chain/actors/policy"
-	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
-)
-
-var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
-	0:                  DrandIncentinet,
-	UpgradeSmokeHeight: DrandMainnet,
-}
-
-const BootstrappersFile = "mainnet.pi"
-const GenesisFile = "mainnet.car"
-
-const UpgradeBreezeHeight = 41280
-
-const BreezeGasTampingDuration = 120
-
-const UpgradeSmokeHeight = 51000
-
-const UpgradeIgnitionHeight = 94000
-const UpgradeRefuelHeight = 130800
-
-const UpgradeActorsV2Height = 138720
-
-const UpgradeTapeHeight = 140760
-
-// This signals our tentative epoch for mainnet launch. Can make it later, but not earlier.
-// Miners, clients, developers, custodians all need time to prepare.
-// We still have upgrades and state changes to do, but can happen after signaling timing here.
-const UpgradeLiftoffHeight = 148888
-
-const UpgradeKumquatHeight = 170000
-
-const UpgradeCalicoHeight = 265200
-const UpgradePersianHeight = UpgradeCalicoHeight + (builtin2.EpochsInHour * 60)
-
-const UpgradeOrangeHeight = 336458
-
-// 2020-12-22T02:00:00Z
-const UpgradeClausHeight = 343200
-
-// 2021-03-04T00:00:30Z
-var UpgradeActorsV3Height = abi.ChainEpoch(550321)
-
-func init() {
-	policy.SetConsensusMinerMinPower(abi.NewStoragePower(10 << 40))
-
-	if os.Getenv("LOTUS_USE_TEST_ADDRESSES") != "1" {
-		SetAddressNetwork(address.Mainnet)
-	}
-
-	if os.Getenv("LOTUS_DISABLE_V3_ACTOR_MIGRATION") == "1" {
-		UpgradeActorsV3Height = math.MaxInt64
-	}
-
-	Devnet = false
-
-	BuildType = BuildMainnet
-}
-
-const BlockDelaySecs = uint64(builtin2.EpochDurationSeconds)
-
-const PropagationDelaySecs = uint64(6)
-
-// BootstrapPeerThreshold is the minimum number peers we need to track for a sync worker to start
-const BootstrapPeerThreshold = 4

build/params_nerpanet.go

@@ -1,72 +0,0 @@
-// +build nerpanet
-
-package build
-
-import (
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/chain/actors/policy"
-
-	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
-)
-
-var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
-	0: DrandMainnet,
-}
-
-const BootstrappersFile = "nerpanet.pi"
-const GenesisFile = "nerpanet.car"
-
-const UpgradeBreezeHeight = -1
-const BreezeGasTampingDuration = 0
-
-const UpgradeSmokeHeight = -1
-
-const UpgradeIgnitionHeight = -2
-const UpgradeRefuelHeight = -3
-
-const UpgradeTapeHeight = -4
-
-const UpgradeLiftoffHeight = -5
-const UpgradeActorsV2Height = 120 // critical: the network can bootstrap from v1 only
-
-const UpgradeKumquatHeight = -6
-
-const UpgradeCalicoHeight = 306000
-const UpgradePersianHeight = UpgradeCalicoHeight + (builtin2.EpochsInHour * 12)
-
-const UpgradeOrangeHeight = 307500
-
-const UpgradeClausHeight = 307600
-
-const UpgradeActorsV3Height = 308000
-
-func init() {
-	// Minimum block production power is set to 4 TiB
-	// Rationale is to discourage small-scale miners from trying to take over the network
-	// One needs to invest in ~2.3x the compute to break consensus, making it not worth it
-	//
-	// DOWNSIDE: the fake-seals need to be kept alive/protected, otherwise network will seize
-	//
-	policy.SetConsensusMinerMinPower(abi.NewStoragePower(4 << 40))
-
-	policy.SetSupportedProofTypes(
-		abi.RegisteredSealProof_StackedDrg512MiBV1,
-		abi.RegisteredSealProof_StackedDrg32GiBV1,
-		abi.RegisteredSealProof_StackedDrg64GiBV1,
-	)
-
-	// Lower the most time-consuming parts of PoRep
-	policy.SetPreCommitChallengeDelay(10)
-
-	// TODO - make this a variable
-	//miner.WPoStChallengeLookback = abi.ChainEpoch(2)
-
-	Devnet = false
-}
-
-const BlockDelaySecs = uint64(builtin2.EpochDurationSeconds)
-
-const PropagationDelaySecs = uint64(6)
-
-// BootstrapPeerThreshold is the minimum number peers we need to track for a sync worker to start
-const BootstrapPeerThreshold = 4

build/params_shared.go

@@ -0,0 +1,124 @@
+package build
+
+import (
+	"math/big"
+	"sort"
+
+	"github.com/libp2p/go-libp2p-core/protocol"
+
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/builtin"
+	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
+
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+)
+
+func DefaultSectorSize() abi.SectorSize {
+	szs := make([]abi.SectorSize, 0, len(miner.SupportedProofTypes))
+	for spt := range miner.SupportedProofTypes {
+		ss, err := spt.SectorSize()
+		if err != nil {
+			panic(err)
+		}
+
+		szs = append(szs, ss)
+	}
+
+	sort.Slice(szs, func(i, j int) bool {
+		return szs[i] < szs[j]
+	})
+
+	return szs[0]
+}
+
+// Core network constants
+
+func BlocksTopic(netName dtypes.NetworkName) string   { return "/fil/blocks/" + string(netName) }
+func MessagesTopic(netName dtypes.NetworkName) string { return "/fil/msgs/" + string(netName) }
+func DhtProtocolName(netName dtypes.NetworkName) protocol.ID {
+	return protocol.ID("/fil/kad/" + string(netName))
+}
+
+// /////
+// Storage
+
+const UnixfsChunkSize uint64 = 1 << 20
+const UnixfsLinksPerLevel = 1024
+
+// /////
+// Consensus / Network
+
+// Seconds
+const AllowableClockDrift = 1
+
+// Epochs
+const ForkLengthThreshold = Finality
+
+// Blocks (e)
+var BlocksPerEpoch = uint64(builtin.ExpectedLeadersPerEpoch)
+
+// Epochs
+const Finality = miner.ChainFinalityish
+const MessageConfidence = 5
+
+// constants for Weight calculation
+// The ratio of weight contributed by short-term vs long-term factors in a given round
+const WRatioNum = int64(1)
+const WRatioDen = 2
+
+// /////
+// Proofs
+
+// Epochs
+const SealRandomnessLookback = Finality
+
+// Epochs
+const SealRandomnessLookbackLimit = SealRandomnessLookback + 2000 // TODO: Get from spec specs-actors
+
+// Maximum lookback that randomness can be sourced from for a seal proof submission
+const MaxSealLookback = SealRandomnessLookbackLimit + 2000 // TODO: Get from specs-actors
+
+// /////
+// Mining
+
+// Epochs
+const TicketRandomnessLookback = 1
+
+const WinningPoStSectorSetLookback = 10
+
+// /////
+// Devnet settings
+
+const TotalFilecoin = 2_000_000_000
+const MiningRewardTotal = 1_400_000_000
+
+const FilecoinPrecision = 1_000_000_000_000_000_000
+
+var InitialRewardBalance *big.Int
+
+// TODO: Move other important consts here
+
+func init() {
+	InitialRewardBalance = big.NewInt(MiningRewardTotal)
+	InitialRewardBalance = InitialRewardBalance.Mul(InitialRewardBalance, big.NewInt(FilecoinPrecision))
+}
+
+// Sync
+const BadBlockCacheSize = 1 << 15
+
+// assuming 4000 messages per round, this lets us not lose any messages across a
+// 10 block reorg.
+const BlsSignatureCacheSize = 40000
+
+// Size of signature verification cache
+// 32k keeps the cache around 10MB in size, max
+const VerifSigCacheSize = 32000
+
+// ///////
+// Limits
+
+// TODO: If this is gonna stay, it should move to specs-actors
+const BlockMessageLimit = 512
+const BlockGasLimit = 100_000_000_000
+
+var DrandChain = `{"public_key":"922a2e93828ff83345bae533f5172669a26c02dc76d6bf59c80892e12ab1455c229211886f35bb56af6d5bea981024df","period":25,"genesis_time":1590445175,"hash":"138a324aa6540f93d0dad002aa89454b1bec2b6e948682cde6bd4db40f4b7c9b"}`

build/params_shared_funcs.go

@@ -1,30 +0,0 @@
-package build
-
-import (
-	"github.com/filecoin-project/go-address"
-
-	"github.com/libp2p/go-libp2p-core/protocol"
-
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-)
-
-// Core network constants
-
-func BlocksTopic(netName dtypes.NetworkName) string   { return "/fil/blocks/" + string(netName) }
-func MessagesTopic(netName dtypes.NetworkName) string { return "/fil/msgs/" + string(netName) }
-func DhtProtocolName(netName dtypes.NetworkName) protocol.ID {
-	return protocol.ID("/fil/kad/" + string(netName))
-}
-
-func SetAddressNetwork(n address.Network) {
-	address.CurrentNetwork = n
-}
-
-func MustParseAddress(addr string) address.Address {
-	ret, err := address.NewFromString(addr)
-	if err != nil {
-		panic(err)
-	}
-
-	return ret
-}

build/params_shared_vals.go

@@ -1,122 +0,0 @@
-// +build !testground
-
-package build
-
-import (
-	"math/big"
-	"os"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/network"
-
-	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
-
-	"github.com/filecoin-project/lotus/chain/actors/policy"
-)
-
-// /////
-// Storage
-
-const UnixfsChunkSize uint64 = 1 << 20
-const UnixfsLinksPerLevel = 1024
-
-// /////
-// Consensus / Network
-
-const AllowableClockDriftSecs = uint64(1)
-const NewestNetworkVersion = network.Version9
-const ActorUpgradeNetworkVersion = network.Version4
-
-// Epochs
-const ForkLengthThreshold = Finality
-
-// Blocks (e)
-var BlocksPerEpoch = uint64(builtin2.ExpectedLeadersPerEpoch)
-
-// Epochs
-const Finality = policy.ChainFinality
-const MessageConfidence = uint64(5)
-
-// constants for Weight calculation
-// The ratio of weight contributed by short-term vs long-term factors in a given round
-const WRatioNum = int64(1)
-const WRatioDen = uint64(2)
-
-// /////
-// Proofs
-
-// Epochs
-// TODO: unused
-const SealRandomnessLookback = policy.SealRandomnessLookback
-
-// /////
-// Mining
-
-// Epochs
-const TicketRandomnessLookback = abi.ChainEpoch(1)
-
-// /////
-// Address
-
-const AddressMainnetEnvVar = "_mainnet_"
-
-// the 'f' prefix doesn't matter
-var ZeroAddress = MustParseAddress("f3yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaby2smx7a")
-
-// /////
-// Devnet settings
-
-var Devnet = true
-
-const FilBase = uint64(2_000_000_000)
-const FilAllocStorageMining = uint64(1_100_000_000)
-
-const FilecoinPrecision = uint64(1_000_000_000_000_000_000)
-const FilReserved = uint64(300_000_000)
-
-var InitialRewardBalance *big.Int
-var InitialFilReserved *big.Int
-
-// TODO: Move other important consts here
-
-func init() {
-	InitialRewardBalance = big.NewInt(int64(FilAllocStorageMining))
-	InitialRewardBalance = InitialRewardBalance.Mul(InitialRewardBalance, big.NewInt(int64(FilecoinPrecision)))
-
-	InitialFilReserved = big.NewInt(int64(FilReserved))
-	InitialFilReserved = InitialFilReserved.Mul(InitialFilReserved, big.NewInt(int64(FilecoinPrecision)))
-
-	if os.Getenv("LOTUS_ADDRESS_TYPE") == AddressMainnetEnvVar {
-		SetAddressNetwork(address.Mainnet)
-	}
-}
-
-// Sync
-const BadBlockCacheSize = 1 << 15
-
-// assuming 4000 messages per round, this lets us not lose any messages across a
-// 10 block reorg.
-const BlsSignatureCacheSize = 40000
-
-// Size of signature verification cache
-// 32k keeps the cache around 10MB in size, max
-const VerifSigCacheSize = 32000
-
-// ///////
-// Limits
-
-// TODO: If this is gonna stay, it should move to specs-actors
-const BlockMessageLimit = 10000
-
-const BlockGasLimit = 10_000_000_000
-const BlockGasTarget = BlockGasLimit / 2
-const BaseFeeMaxChangeDenom = 8 // 12.5%
-const InitialBaseFee = 100e6
-const MinimumBaseFee = 100
-const PackingEfficiencyNum = 4
-const PackingEfficiencyDenom = 5
-
-// Actor consts
-// TODO: Pull from actors when its made not private
-var MinDealDuration = abi.ChainEpoch(180 * builtin2.EpochsInDay)

build/params_testground.go

@@ -1,111 +0,0 @@
-// +build testground
-
-// This file makes hardcoded parameters (const) configurable as vars.
-//
-// Its purpose is to unlock various degrees of flexibility and parametrization
-// when writing Testground plans for Lotus.
-//
-package build
-
-import (
-	"math/big"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/network"
-
-	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
-
-	"github.com/filecoin-project/lotus/chain/actors/policy"
-)
-
-var (
-	UnixfsChunkSize     = uint64(1 << 20)
-	UnixfsLinksPerLevel = 1024
-
-	BlocksPerEpoch        = uint64(builtin2.ExpectedLeadersPerEpoch)
-	BlockMessageLimit     = 512
-	BlockGasLimit         = int64(100_000_000_000)
-	BlockGasTarget        = int64(BlockGasLimit / 2)
-	BaseFeeMaxChangeDenom = int64(8) // 12.5%
-	InitialBaseFee        = int64(100e6)
-	MinimumBaseFee        = int64(100)
-	BlockDelaySecs        = uint64(builtin2.EpochDurationSeconds)
-	PropagationDelaySecs  = uint64(6)
-
-	AllowableClockDriftSecs = uint64(1)
-
-	Finality            = policy.ChainFinality
-	ForkLengthThreshold = Finality
-
-	SlashablePowerDelay        = 20
-	InteractivePoRepConfidence = 6
-
-	MessageConfidence uint64 = 5
-
-	WRatioNum = int64(1)
-	WRatioDen = uint64(2)
-
-	BadBlockCacheSize     = 1 << 15
-	BlsSignatureCacheSize = 40000
-	VerifSigCacheSize     = 32000
-
-	SealRandomnessLookback = policy.SealRandomnessLookback
-
-	TicketRandomnessLookback = abi.ChainEpoch(1)
-
-	FilBase               uint64 = 2_000_000_000
-	FilAllocStorageMining uint64 = 1_400_000_000
-	FilReserved           uint64 = 300_000_000
-
-	FilecoinPrecision uint64 = 1_000_000_000_000_000_000
-
-	InitialRewardBalance = func() *big.Int {
-		v := big.NewInt(int64(FilAllocStorageMining))
-		v = v.Mul(v, big.NewInt(int64(FilecoinPrecision)))
-		return v
-	}()
-
-	InitialFilReserved = func() *big.Int {
-		v := big.NewInt(int64(FilReserved))
-		v = v.Mul(v, big.NewInt(int64(FilecoinPrecision)))
-		return v
-	}()
-
-	// Actor consts
-	// TODO: Pull from actors when its made not private
-	MinDealDuration = abi.ChainEpoch(180 * builtin2.EpochsInDay)
-
-	PackingEfficiencyNum   int64 = 4
-	PackingEfficiencyDenom int64 = 5
-
-	UpgradeBreezeHeight      abi.ChainEpoch = -1
-	BreezeGasTampingDuration abi.ChainEpoch = 0
-
-	UpgradeSmokeHeight    abi.ChainEpoch = -1
-	UpgradeIgnitionHeight abi.ChainEpoch = -2
-	UpgradeRefuelHeight   abi.ChainEpoch = -3
-	UpgradeTapeHeight     abi.ChainEpoch = -4
-	UpgradeActorsV2Height abi.ChainEpoch = 10
-	UpgradeLiftoffHeight  abi.ChainEpoch = -5
-	UpgradeKumquatHeight  abi.ChainEpoch = -6
-	UpgradeCalicoHeight   abi.ChainEpoch = -7
-	UpgradePersianHeight  abi.ChainEpoch = -8
-	UpgradeOrangeHeight   abi.ChainEpoch = -9
-	UpgradeClausHeight    abi.ChainEpoch = -10
-	UpgradeActorsV3Height abi.ChainEpoch = -11
-
-	DrandSchedule = map[abi.ChainEpoch]DrandEnum{
-		0: DrandMainnet,
-	}
-
-	NewestNetworkVersion       = network.Version9
-	ActorUpgradeNetworkVersion = network.Version4
-
-	Devnet      = true
-	ZeroAddress = MustParseAddress("f3yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaby2smx7a")
-
-	BootstrappersFile = ""
-	GenesisFile       = ""
-)
-
-const BootstrapPeerThreshold = 1

build/params_testnet.go

@@ -0,0 +1,26 @@
+// +build !debug
+// +build !2k
+
+package build
+
+import (
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
+	"github.com/filecoin-project/specs-actors/actors/builtin"
+	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
+	"github.com/filecoin-project/specs-actors/actors/builtin/power"
+)
+
+func init() {
+	power.ConsensusMinerMinPower = big.NewInt(1024 << 20)
+	miner.SupportedProofTypes = map[abi.RegisteredProof]struct{}{
+		abi.RegisteredProof_StackedDRG512MiBSeal: {},
+		abi.RegisteredProof_StackedDRG32GiBSeal:  {},
+		abi.RegisteredProof_StackedDRG64GiBSeal:  {},
+	}
+}
+
+// Seconds
+const BlockDelay = builtin.EpochDurationSeconds
+
+const PropagationDelay = 6