Compare commits

..
Author SHA1 Message Date
Travis Person 2534d94b20 build binaries 2021-10-18 19:54:09 +00:00
1096 changed files with 26337 additions and 82755 deletions

No files matched your search

+166 -255
View File
@@ -1,32 +1,23 @@
version: 2.1
orbs:
go: gotest/tools@0.0.13
aws-cli: circleci/aws-cli@1.3.2
packer: salaxander/packer@0.0.3
executors:
golang:
docker:
- image: cimg/go:1.17.9
- image: circleci/golang:1.16.4
resource_class: 2xlarge
ubuntu:
docker:
- image: ubuntu:20.04
packer:
description: |
The HashiCorp provided Packer container
parameters:
packer-version:
type: string
default: "1.8"
docker:
- image: hashicorp/packer:<< parameters.packer-version >>
commands:
install-deps:
steps:
- run: |
sudo apt update
sudo apt install python-is-python3
- go/install-ssh
- go/install: {package: git}
prepare:
parameters:
linux:
@@ -53,13 +44,13 @@ commands:
- restore_cache:
name: Restore parameters cache
keys:
- 'v26-2k-lotus-params'
- 'v25-2k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
- run: ./lotus fetch-params 2048
- save_cache:
name: Save parameters cache
key: 'v26-2k-lotus-params'
key: 'v25-2k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
install_ipfs:
@@ -67,14 +58,14 @@ commands:
- run: |
apt update
apt install -y wget
wget https://github.com/ipfs/go-ipfs/releases/download/v0.12.2/go-ipfs_v0.12.2_linux-amd64.tar.gz
wget https://github.com/ipfs/go-ipfs/releases/download/v0.12.2/go-ipfs_v0.12.2_linux-amd64.tar.gz.sha512
if [ "$(sha512sum go-ipfs_v0.12.2_linux-amd64.tar.gz)" != "$(cat go-ipfs_v0.12.2_linux-amd64.tar.gz.sha512)" ]
wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz
wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512
if [ "$(sha512sum go-ipfs_v0.4.22_linux-amd64.tar.gz)" != "$(cat go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512)" ]
then
echo "ipfs failed checksum check"
exit 1
fi
tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz
tar -xf go-ipfs_v0.4.22_linux-amd64.tar.gz
mv go-ipfs/ipfs /usr/local/bin/ipfs
chmod +x /usr/local/bin/ipfs
git_fetch_all_tags:
@@ -83,25 +74,6 @@ commands:
name: fetch all tags
command: |
git fetch --all
packer_build:
description: "Run a packer build"
parameters:
template:
description: |
The name of the packer template file
type: string
default: packer.json
args:
description: |
Arguments to pass to the packer build command
type: string
default: ""
steps:
- run:
name: "Run a packer build"
command: packer build << parameters.args >> << parameters.template >>
no_output_timeout: 30m
jobs:
mod-tidy-check:
@@ -109,13 +81,9 @@ jobs:
steps:
- install-deps
- prepare
- run: go mod tidy -v
- run:
name: Check git diff
command: |
git --no-pager diff go.mod go.sum
git --no-pager diff --quiet go.mod go.sum
build-linux:
- go/mod-tidy-check
build-all:
executor: golang
steps:
- install-deps
@@ -124,9 +92,6 @@ jobs:
- run: sudo apt-get install npm
- run:
command: make buildall
- run:
name: check tag and version output match
command: ./scripts/version-check.sh ./lotus
- store_artifacts:
path: lotus
- store_artifacts:
@@ -191,12 +156,14 @@ jobs:
command: make deps lotus
no_output_timeout: 30m
- download-params
- go/install-gotestsum:
gobin: $HOME/.local/bin
version: 0.5.2
- run:
name: go test
environment:
TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >>
SKIP_CONFORMANCE: "1"
LOTUS_SRC_DIR: /home/circleci/project
command: |
mkdir -p /tmp/test-reports/<< parameters.suite >>
mkdir -p /tmp/test-artifacts
@@ -216,6 +183,8 @@ jobs:
- when:
condition: << parameters.codecov-upload >>
steps:
- go/install: {package: bash}
- go/install: {package: curl}
- run:
shell: /bin/bash -eo pipefail
command: |
@@ -223,7 +192,7 @@ jobs:
test-conformance:
description: |
Run tests using a corpus of interoperable test vectors for Filecoin
Run tests using a corpus of interoperable test vectors for Filecoin
implementations to test their correctness and compliance with the Filecoin
specifications.
parameters:
@@ -254,6 +223,9 @@ jobs:
cd extern/test-vectors
git fetch
git checkout origin/<< parameters.vectors-branch >>
- go/install-gotestsum:
gobin: $HOME/.local/bin
version: 0.5.2
- run:
name: install statediff globally
command: |
@@ -310,6 +282,21 @@ jobs:
root: "."
paths:
- linux-butterflynet
build-ntwk-nerpa:
description: |
Compile lotus binaries for the nerpa network
parameters:
<<: *test-params
executor: << parameters.executor >>
steps:
- install-deps
- prepare
- run: make nerpanet
- run: mkdir linux-nerpanet && mv lotus lotus-miner lotus-worker linux-nerpanet
- persist_to_workspace:
root: "."
paths:
- linux-nerpanet
build-lotus-soup:
description: |
Compile `lotus-soup` Testground test plan
@@ -343,7 +330,7 @@ jobs:
command: pushd testplans/lotus-soup && mkdir -p $HOME/testground && cp env-ci.toml $HOME/testground/.env.toml && echo 'endpoint="https://ci.testground.ipfs.team"' >> $HOME/testground/.env.toml && echo 'user="circleci"' >> $HOME/testground/.env.toml
- run:
name: "prepare testground home dir and link test plans"
command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup
command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup && ln -s $(pwd)/testplans/graphsync $HOME/testground/plans/graphsync
- run:
name: "go get lotus@master"
command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master
@@ -353,6 +340,10 @@ jobs:
- run:
name: "trigger payment channel stress testplan on taas"
command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
- run:
name: "trigger graphsync testplan on taas"
command: ~/testground-cli run composition -f $HOME/testground/plans/graphsync/_compositions/stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
build-macos:
description: build darwin lotus binary
@@ -366,8 +357,8 @@ jobs:
- run:
name: Install go
command: |
curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \
sudo installer -pkg go1.17.9.darwin-amd64.pkg -target /
curl -O https://dl.google.com/go/go1.16.4.darwin-amd64.pkg && \
sudo installer -pkg go1.16.4.darwin-amd64.pkg -target /
- run:
name: Install pkg-config
command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
@@ -376,6 +367,11 @@ jobs:
name: Install Rust
command: |
curl https://sh.rustup.rs -sSf | sh -s -- -y
- run:
name: Install jq
command: |
sudo curl --location https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 --output /usr/local/bin/jq
sudo chmod +x /usr/local/bin/jq
- run:
name: Install hwloc
command: |
@@ -388,12 +384,10 @@ jobs:
- restore_cache:
name: restore cargo cache
key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
- install-deps
- run:
command: make build
no_output_timeout: 30m
- run:
name: check tag and version output match
command: ./scripts/version-check.sh ./lotus
- store_artifacts:
path: lotus
- store_artifacts:
@@ -414,7 +408,7 @@ jobs:
build-appimage:
machine:
image: ubuntu-2004:202111-02
image: ubuntu-2004:202104-01
steps:
- checkout
- attach_workspace:
@@ -422,16 +416,6 @@ jobs:
- run:
name: install appimage-builder
command: |
# appimage-builder requires /dev/snd to exist. It creates containers during the testing phase
# that pass sound devices from the host to the testing container. (hard coded!)
# https://github.com/AppImageCrafters/appimage-builder/blob/master/appimagebuilder/modules/test/execution_test.py#L54
# Circleci doesn't provide a working sound device; this is enough to fake it.
if [ ! -e /dev/snd ]
then
sudo mkdir /dev/snd
sudo mknod /dev/snd/ControlC0 c 1 2
fi
# docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html
sudo apt update
sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace
@@ -451,6 +435,8 @@ jobs:
command: |
mkdir appimage
mv Lotus-*.AppImage appimage
- store_artifacts:
path: appimage
- persist_to_workspace:
root: "."
paths:
@@ -507,6 +493,9 @@ jobs:
executor:
type: executor
default: golang
golangci-lint-version:
type: string
default: 1.27.0
concurrency:
type: string
default: '2'
@@ -525,10 +514,13 @@ jobs:
- run:
command: make deps
no_output_timeout: 30m
- go/install-golangci-lint:
gobin: $HOME/.local/bin
version: << parameters.golangci-lint-version >>
- run:
name: Lint
command: |
golangci-lint run -v --timeout 2m \
$HOME/.local/bin/golangci-lint run -v --timeout 2m \
--concurrency << parameters.concurrency >> << parameters.args >>
lint-all:
<<: *lint
@@ -536,19 +528,6 @@ jobs:
publish:
description: publish binary artifacts
executor: ubuntu
parameters:
linux:
default: false
description: publish linux binaries?
type: boolean
darwin:
default: false
description: publish darwin binaries?
type: boolean
appimage:
default: false
description: publish appimage binaries?
type: boolean
steps:
- run:
name: Install git jq curl
@@ -559,21 +538,12 @@ jobs:
- install_ipfs
- attach_workspace:
at: "."
- when:
condition: << parameters.linux >>
steps:
- run: ./scripts/build-arch-bundle.sh linux
- run: ./scripts/publish-arch-release.sh linux
- when:
condition: << parameters.darwin>>
steps:
- run: ./scripts/build-arch-bundle.sh darwin
- run: ./scripts/publish-arch-release.sh darwin
- when:
condition: << parameters.appimage >>
steps:
- run: ./scripts/build-appimage-bundle.sh
- run: ./scripts/publish-arch-release.sh appimage
- run:
name: Create bundles
command: ./scripts/build-bundle.sh
- run:
name: Publish release
command: ./scripts/publish-release.sh
publish-snapcraft:
description: build and push snapcraft
@@ -590,6 +560,11 @@ jobs:
- run:
name: install snapcraft
command: sudo snap install snapcraft --classic
- run:
name: create snapcraft config file
command: |
mkdir -p ~/.config/snapcraft
echo "$SNAPCRAFT_LOGIN_FILE" | base64 -d > ~/.config/snapcraft/snapcraft.cfg
- run:
name: build snap
command: snapcraft --use-lxd
@@ -663,11 +638,6 @@ jobs:
default: "latest"
description: A comma-separated string containing docker image tags to build and push (default = latest)
target:
type: string
default: "lotus-all-in-one"
description: Docker target to build
steps:
- run:
name: Confirm that environment variables are set
@@ -707,7 +677,6 @@ jobs:
docker build \
<<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
--target <<parameters.target>> \
-f <<parameters.path>>/<<parameters.dockerfile>> \
$docker_tag_args \
<<parameters.path>>
@@ -720,16 +689,54 @@ jobs:
docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
done
publish-packer-snap:
description: build packer image with snap. mainnet only.
publish-packer-mainnet:
description: build and push AWS IAM and DigitalOcean droplet.
executor:
name: packer
name: packer/default
packer-version: 1.6.6
steps:
- checkout
- attach_workspace:
at: "."
- packer_build:
template: tools/packer/lotus-snap.pkr.hcl
- packer/build:
template: tools/packer/lotus.pkr.hcl
args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
publish-packer-calibrationnet:
description: build and push AWS IAM and DigitalOcean droplet.
executor:
name: packer/default
packer-version: 1.6.6
steps:
- checkout
- attach_workspace:
at: "."
- packer/build:
template: tools/packer/lotus.pkr.hcl
args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
publish-packer-butterflynet:
description: build and push AWS IAM and DigitalOcean droplet.
executor:
name: packer/default
packer-version: 1.6.6
steps:
- checkout
- attach_workspace:
at: "."
- packer/build:
template: tools/packer/lotus.pkr.hcl
args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG"
publish-packer-nerpanet:
description: build and push AWS IAM and DigitalOcean droplet.
executor:
name: packer/default
packer-version: 1.6.6
steps:
- checkout
- attach_workspace:
at: "."
- packer/build:
template: tools/packer/lotus.pkr.hcl
args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG"
publish-dockerhub:
description: publish to dockerhub
machine:
@@ -805,31 +812,11 @@ workflows:
suite: itest-deadlines
target: "./itests/deadlines_test.go"
- test:
name: test-itest-deals_512mb
suite: itest-deals_512mb
target: "./itests/deals_512mb_test.go"
- test:
name: test-itest-deals_anycid
suite: itest-deals_anycid
target: "./itests/deals_anycid_test.go"
- test:
name: test-itest-deals_concurrent
suite: itest-deals_concurrent
target: "./itests/deals_concurrent_test.go"
- test:
name: test-itest-deals_invalid_utf8_label
suite: itest-deals_invalid_utf8_label
target: "./itests/deals_invalid_utf8_label_test.go"
- test:
name: test-itest-deals_max_staging_deals
suite: itest-deals_max_staging_deals
target: "./itests/deals_max_staging_deals_test.go"
- test:
name: test-itest-deals_offline
suite: itest-deals_offline
@@ -840,16 +827,6 @@ workflows:
suite: itest-deals_padding
target: "./itests/deals_padding_test.go"
- test:
name: test-itest-deals_partial_retrieval_dm-level
suite: itest-deals_partial_retrieval_dm-level
target: "./itests/deals_partial_retrieval_dm-level_test.go"
- test:
name: test-itest-deals_partial_retrieval
suite: itest-deals_partial_retrieval
target: "./itests/deals_partial_retrieval_test.go"
- test:
name: test-itest-deals_power
suite: itest-deals_power
@@ -865,11 +842,6 @@ workflows:
suite: itest-deals_publish
target: "./itests/deals_publish_test.go"
- test:
name: test-itest-deals_retry_deal_no_funds
suite: itest-deals_retry_deal_no_funds
target: "./itests/deals_retry_deal_no_funds_test.go"
- test:
name: test-itest-deals
suite: itest-deals
@@ -885,21 +857,6 @@ workflows:
suite: itest-get_messages_in_ts
target: "./itests/get_messages_in_ts_test.go"
- test:
name: test-itest-lite_migration
suite: itest-lite_migration
target: "./itests/lite_migration_test.go"
- test:
name: test-itest-lookup_robust_address
suite: itest-lookup_robust_address
target: "./itests/lookup_robust_address_test.go"
- test:
name: test-itest-mempool
suite: itest-mempool
target: "./itests/mempool_test.go"
- test:
name: test-itest-multisig
suite: itest-multisig
@@ -930,11 +887,6 @@ workflows:
suite: itest-sector_finalize_early
target: "./itests/sector_finalize_early_test.go"
- test:
name: test-itest-sector_make_cc_avail
suite: itest-sector_make_cc_avail
target: "./itests/sector_make_cc_avail_test.go"
- test:
name: test-itest-sector_miner_collateral
suite: itest-sector_miner_collateral
@@ -945,26 +897,11 @@ workflows:
suite: itest-sector_pledge
target: "./itests/sector_pledge_test.go"
- test:
name: test-itest-sector_prefer_no_upgrade
suite: itest-sector_prefer_no_upgrade
target: "./itests/sector_prefer_no_upgrade_test.go"
- test:
name: test-itest-sector_revert_available
suite: itest-sector_revert_available
target: "./itests/sector_revert_available_test.go"
- test:
name: test-itest-sector_terminate
suite: itest-sector_terminate
target: "./itests/sector_terminate_test.go"
- test:
name: test-itest-self_sent_txn
suite: itest-self_sent_txn
target: "./itests/self_sent_txn_test.go"
- test:
name: test-itest-tape
suite: itest-tape
@@ -975,11 +912,6 @@ workflows:
suite: itest-verifreg
target: "./itests/verifreg_test.go"
- test:
name: test-itest-wdpost_config
suite: itest-wdpost_config
target: "./itests/wdpost_config_test.go"
- test:
name: test-itest-wdpost_dispute
suite: itest-wdpost_dispute
@@ -990,16 +922,6 @@ workflows:
suite: itest-wdpost
target: "./itests/wdpost_test.go"
- test:
name: test-itest-wdpost_worker_config
suite: itest-wdpost_worker_config
target: "./itests/wdpost_worker_config_test.go"
- test:
name: test-itest-worker
suite: itest-worker
target: "./itests/worker_test.go"
- test:
name: test-unit-cli
suite: utest-unit-cli
@@ -1019,7 +941,7 @@ workflows:
- test:
go-test-flags: "-run=TestMulticoreSDR"
suite: multicore-sdr-check
target: "./storage/sealer/ffiwrapper"
target: "./extern/sector-storage/ffiwrapper"
proofs-log-test: "1"
- test-conformance:
suite: conformance
@@ -1030,18 +952,14 @@ workflows:
codecov-upload: false
suite: conformance-bleeding-edge
target: "./conformance"
vectors-branch: specs-actors-v7
vectors-branch: master
- trigger-testplans:
filters:
branches:
only:
- master
- build-debug
- build-linux:
filters:
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-all
- build-ntwk-calibration:
filters:
tags:
@@ -1052,48 +970,18 @@ workflows:
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-ntwk-nerpa:
filters:
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-lotus-soup
- build-macos:
filters:
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-appimage:
filters:
branches:
only:
- /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-macos
- build-appimage
- publish:
name: publish-macos
darwin: true
requires:
- build-all
- build-macos
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish:
name: publish-linux
linux: true
requires:
- build-linux
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish:
name: publish-appimage
appimage: true
requires:
- build-appimage
filters:
branches:
@@ -1107,13 +995,46 @@ workflows:
path: .
repo: lotus-dev
tag: '${CIRCLE_SHA1:0:8}'
target: lotus-all-in-one
- build-and-push-image:
dockerfile: Dockerfile.lotus
path: .
repo: lotus-test
tag: '${CIRCLE_SHA1:0:8}'
target: lotus-test
- publish-packer-mainnet:
requires:
- build-all
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish-packer-calibrationnet:
requires:
- build-ntwk-calibration
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish-packer-butterflynet:
requires:
- build-ntwk-butterfly
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish-packer-nerpanet:
requires:
- build-ntwk-nerpa
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish-snapcraft:
name: publish-snapcraft-stable
channel: stable
@@ -1150,13 +1071,3 @@ workflows:
- publish-dockerhub:
name: publish-dockerhub-nightly
tag: nightly
monthly:
triggers:
- schedule:
cron: "0 0 1 * *"
filters:
branches:
only:
- master
jobs:
- publish-packer-snap
+168 -156
View File
@@ -1,32 +1,23 @@
version: 2.1
orbs:
go: gotest/tools@0.0.13
aws-cli: circleci/aws-cli@1.3.2
packer: salaxander/packer@0.0.3
executors:
golang:
docker:
- image: cimg/go:1.17.9
- image: circleci/golang:1.16.4
resource_class: 2xlarge
ubuntu:
docker:
- image: ubuntu:20.04
packer:
description: |
The HashiCorp provided Packer container
parameters:
packer-version:
type: string
default: "1.8"
docker:
- image: hashicorp/packer:<< parameters.packer-version >>
commands:
install-deps:
steps:
- run: |
sudo apt update
sudo apt install python-is-python3
- go/install-ssh
- go/install: {package: git}
prepare:
parameters:
linux:
@@ -53,13 +44,13 @@ commands:
- restore_cache:
name: Restore parameters cache
keys:
- 'v26-2k-lotus-params'
- 'v25-2k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
- run: ./lotus fetch-params 2048
- save_cache:
name: Save parameters cache
key: 'v26-2k-lotus-params'
key: 'v25-2k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
install_ipfs:
@@ -67,14 +58,14 @@ commands:
- run: |
apt update
apt install -y wget
wget https://github.com/ipfs/go-ipfs/releases/download/v0.12.2/go-ipfs_v0.12.2_linux-amd64.tar.gz
wget https://github.com/ipfs/go-ipfs/releases/download/v0.12.2/go-ipfs_v0.12.2_linux-amd64.tar.gz.sha512
if [ "$(sha512sum go-ipfs_v0.12.2_linux-amd64.tar.gz)" != "$(cat go-ipfs_v0.12.2_linux-amd64.tar.gz.sha512)" ]
wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz
wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512
if [ "$(sha512sum go-ipfs_v0.4.22_linux-amd64.tar.gz)" != "$(cat go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512)" ]
then
echo "ipfs failed checksum check"
exit 1
fi
tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz
tar -xf go-ipfs_v0.4.22_linux-amd64.tar.gz
mv go-ipfs/ipfs /usr/local/bin/ipfs
chmod +x /usr/local/bin/ipfs
git_fetch_all_tags:
@@ -83,25 +74,6 @@ commands:
name: fetch all tags
command: |
git fetch --all
packer_build:
description: "Run a packer build"
parameters:
template:
description: |
The name of the packer template file
type: string
default: packer.json
args:
description: |
Arguments to pass to the packer build command
type: string
default: ""
steps:
- run:
name: "Run a packer build"
command: packer build << parameters.args >> << parameters.template >>
no_output_timeout: 30m
jobs:
mod-tidy-check:
@@ -109,13 +81,9 @@ jobs:
steps:
- install-deps
- prepare
- run: go mod tidy -v
- run:
name: Check git diff
command: |
git --no-pager diff go.mod go.sum
git --no-pager diff --quiet go.mod go.sum
build-linux:
- go/mod-tidy-check
build-all:
executor: golang
steps:
- install-deps
@@ -124,9 +92,6 @@ jobs:
- run: sudo apt-get install npm
- run:
command: make buildall
- run:
name: check tag and version output match
command: ./scripts/version-check.sh ./lotus
- store_artifacts:
path: lotus
- store_artifacts:
@@ -191,12 +156,14 @@ jobs:
command: make deps lotus
no_output_timeout: 30m
- download-params
- go/install-gotestsum:
gobin: $HOME/.local/bin
version: 0.5.2
- run:
name: go test
environment:
TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >>
SKIP_CONFORMANCE: "1"
LOTUS_SRC_DIR: /home/circleci/project
command: |
mkdir -p /tmp/test-reports/<< parameters.suite >>
mkdir -p /tmp/test-artifacts
@@ -216,6 +183,8 @@ jobs:
- when:
condition: << parameters.codecov-upload >>
steps:
- go/install: {package: bash}
- go/install: {package: curl}
- run:
shell: /bin/bash -eo pipefail
command: |
@@ -223,7 +192,7 @@ jobs:
test-conformance:
description: |
Run tests using a corpus of interoperable test vectors for Filecoin
Run tests using a corpus of interoperable test vectors for Filecoin
implementations to test their correctness and compliance with the Filecoin
specifications.
parameters:
@@ -254,6 +223,9 @@ jobs:
cd extern/test-vectors
git fetch
git checkout origin/<< parameters.vectors-branch >>
- go/install-gotestsum:
gobin: $HOME/.local/bin
version: 0.5.2
- run:
name: install statediff globally
command: |
@@ -310,6 +282,21 @@ jobs:
root: "."
paths:
- linux-butterflynet
build-ntwk-nerpa:
description: |
Compile lotus binaries for the nerpa network
parameters:
<<: *test-params
executor: << parameters.executor >>
steps:
- install-deps
- prepare
- run: make nerpanet
- run: mkdir linux-nerpanet && mv lotus lotus-miner lotus-worker linux-nerpanet
- persist_to_workspace:
root: "."
paths:
- linux-nerpanet
build-lotus-soup:
description: |
Compile `lotus-soup` Testground test plan
@@ -343,7 +330,7 @@ jobs:
command: pushd testplans/lotus-soup && mkdir -p $HOME/testground && cp env-ci.toml $HOME/testground/.env.toml && echo 'endpoint="https://ci.testground.ipfs.team"' >> $HOME/testground/.env.toml && echo 'user="circleci"' >> $HOME/testground/.env.toml
- run:
name: "prepare testground home dir and link test plans"
command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup
command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup && ln -s $(pwd)/testplans/graphsync $HOME/testground/plans/graphsync
- run:
name: "go get lotus@master"
command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master
@@ -353,11 +340,15 @@ jobs:
- run:
name: "trigger payment channel stress testplan on taas"
command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
- run:
name: "trigger graphsync testplan on taas"
command: ~/testground-cli run composition -f $HOME/testground/plans/graphsync/_compositions/stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
build-macos:
description: build darwin lotus binary
macos:
xcode: "12.5.0"
xcode: "10.0.0"
working_directory: ~/go/src/github.com/filecoin-project/lotus
steps:
- prepare:
@@ -366,8 +357,8 @@ jobs:
- run:
name: Install go
command: |
curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \
sudo installer -pkg go1.17.9.darwin-amd64.pkg -target /
curl -O https://dl.google.com/go/go1.16.4.darwin-amd64.pkg && \
sudo installer -pkg go1.16.4.darwin-amd64.pkg -target /
- run:
name: Install pkg-config
command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
@@ -376,6 +367,11 @@ jobs:
name: Install Rust
command: |
curl https://sh.rustup.rs -sSf | sh -s -- -y
- run:
name: Install jq
command: |
curl --location https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 --output /usr/local/bin/jq
chmod +x /usr/local/bin/jq
- run:
name: Install hwloc
command: |
@@ -388,12 +384,10 @@ jobs:
- restore_cache:
name: restore cargo cache
key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
- install-deps
- run:
command: make build
no_output_timeout: 30m
- run:
name: check tag and version output match
command: ./scripts/version-check.sh ./lotus
- store_artifacts:
path: lotus
- store_artifacts:
@@ -414,7 +408,7 @@ jobs:
build-appimage:
machine:
image: ubuntu-2004:202111-02
image: ubuntu-2004:202104-01
steps:
- checkout
- attach_workspace:
@@ -422,16 +416,6 @@ jobs:
- run:
name: install appimage-builder
command: |
# appimage-builder requires /dev/snd to exist. It creates containers during the testing phase
# that pass sound devices from the host to the testing container. (hard coded!)
# https://github.com/AppImageCrafters/appimage-builder/blob/master/appimagebuilder/modules/test/execution_test.py#L54
# Circleci doesn't provide a working sound device; this is enough to fake it.
if [ ! -e /dev/snd ]
then
sudo mkdir /dev/snd
sudo mknod /dev/snd/ControlC0 c 1 2
fi
# docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html
sudo apt update
sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace
@@ -507,6 +491,9 @@ jobs:
executor:
type: executor
default: golang
golangci-lint-version:
type: string
default: 1.27.0
concurrency:
type: string
default: '2'
@@ -525,10 +512,13 @@ jobs:
- run:
command: make deps
no_output_timeout: 30m
- go/install-golangci-lint:
gobin: $HOME/.local/bin
version: << parameters.golangci-lint-version >>
- run:
name: Lint
command: |
golangci-lint run -v --timeout 2m \
$HOME/.local/bin/golangci-lint run -v --timeout 2m \
--concurrency << parameters.concurrency >> << parameters.args >>
lint-all:
<<: *lint
@@ -536,19 +526,6 @@ jobs:
publish:
description: publish binary artifacts
executor: ubuntu
parameters:
linux:
default: false
description: publish linux binaries?
type: boolean
darwin:
default: false
description: publish darwin binaries?
type: boolean
appimage:
default: false
description: publish appimage binaries?
type: boolean
steps:
- run:
name: Install git jq curl
@@ -559,21 +536,12 @@ jobs:
- install_ipfs
- attach_workspace:
at: "."
- when:
condition: << parameters.linux >>
steps:
- run: ./scripts/build-arch-bundle.sh linux
- run: ./scripts/publish-arch-release.sh linux
- when:
condition: << parameters.darwin>>
steps:
- run: ./scripts/build-arch-bundle.sh darwin
- run: ./scripts/publish-arch-release.sh darwin
- when:
condition: << parameters.appimage >>
steps:
- run: ./scripts/build-appimage-bundle.sh
- run: ./scripts/publish-arch-release.sh appimage
- run:
name: Create bundles
command: ./scripts/build-bundle.sh
- run:
name: Publish release
command: ./scripts/publish-release.sh
publish-snapcraft:
description: build and push snapcraft
@@ -590,6 +558,11 @@ jobs:
- run:
name: install snapcraft
command: sudo snap install snapcraft --classic
- run:
name: create snapcraft config file
command: |
mkdir -p ~/.config/snapcraft
echo "$SNAPCRAFT_LOGIN_FILE" | base64 -d > ~/.config/snapcraft/snapcraft.cfg
- run:
name: build snap
command: snapcraft --use-lxd
@@ -663,11 +636,6 @@ jobs:
default: "latest"
description: A comma-separated string containing docker image tags to build and push (default = latest)
target:
type: string
default: "lotus-all-in-one"
description: Docker target to build
steps:
- run:
name: Confirm that environment variables are set
@@ -707,7 +675,6 @@ jobs:
docker build \
<<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
--target <<parameters.target>> \
-f <<parameters.path>>/<<parameters.dockerfile>> \
$docker_tag_args \
<<parameters.path>>
@@ -720,16 +687,54 @@ jobs:
docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
done
publish-packer-snap:
description: build packer image with snap. mainnet only.
publish-packer-mainnet:
description: build and push AWS IAM and DigitalOcean droplet.
executor:
name: packer
name: packer/default
packer-version: 1.6.6
steps:
- checkout
- attach_workspace:
at: "."
- packer_build:
template: tools/packer/lotus-snap.pkr.hcl
- packer/build:
template: tools/packer/lotus.pkr.hcl
args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
publish-packer-calibrationnet:
description: build and push AWS IAM and DigitalOcean droplet.
executor:
name: packer/default
packer-version: 1.6.6
steps:
- checkout
- attach_workspace:
at: "."
- packer/build:
template: tools/packer/lotus.pkr.hcl
args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
publish-packer-butterflynet:
description: build and push AWS IAM and DigitalOcean droplet.
executor:
name: packer/default
packer-version: 1.6.6
steps:
- checkout
- attach_workspace:
at: "."
- packer/build:
template: tools/packer/lotus.pkr.hcl
args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG"
publish-packer-nerpanet:
description: build and push AWS IAM and DigitalOcean droplet.
executor:
name: packer/default
packer-version: 1.6.6
steps:
- checkout
- attach_workspace:
at: "."
- packer/build:
template: tools/packer/lotus.pkr.hcl
args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG"
publish-dockerhub:
description: publish to dockerhub
machine:
@@ -799,7 +804,7 @@ workflows:
- test:
go-test-flags: "-run=TestMulticoreSDR"
suite: multicore-sdr-check
target: "./storage/sealer/ffiwrapper"
target: "./extern/sector-storage/ffiwrapper"
proofs-log-test: "1"
- test-conformance:
suite: conformance
@@ -810,14 +815,14 @@ workflows:
codecov-upload: false
suite: conformance-bleeding-edge
target: "./conformance"
vectors-branch: specs-actors-v7
vectors-branch: master
- trigger-testplans:
filters:
branches:
only:
- master
- build-debug
- build-linux:
- build-all:
filters:
tags:
only:
@@ -832,48 +837,32 @@ workflows:
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-ntwk-nerpa:
filters:
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-lotus-soup
- build-macos:
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-appimage:
filters:
branches:
only:
- /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish:
name: publish-macos
darwin: true
requires:
- build-all
- build-macos
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish:
name: publish-linux
linux: true
requires:
- build-linux
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish:
name: publish-appimage
appimage: true
requires:
- build-appimage
filters:
branches:
@@ -887,13 +876,46 @@ workflows:
path: .
repo: lotus-dev
tag: '${CIRCLE_SHA1:0:8}'
target: lotus-all-in-one
- build-and-push-image:
dockerfile: Dockerfile.lotus
path: .
repo: lotus-test
tag: '${CIRCLE_SHA1:0:8}'
target: lotus-test
- publish-packer-mainnet:
requires:
- build-all
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish-packer-calibrationnet:
requires:
- build-ntwk-calibration
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish-packer-butterflynet:
requires:
- build-ntwk-butterfly
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish-packer-nerpanet:
requires:
- build-ntwk-nerpa
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish-snapcraft:
name: publish-snapcraft-stable
channel: stable
@@ -930,13 +952,3 @@ workflows:
- publish-dockerhub:
name: publish-dockerhub-nightly
tag: nightly
monthly:
triggers:
- schedule:
cron: "0 0 1 * *"
filters:
branches:
only:
- master
jobs:
- publish-packer-snap
+12 -29
View File
@@ -1,26 +1,9 @@
ignore:
# Auto generated
- "^.*_gen.go$"
- "^.*/mock_full.go$"
# Old actors.
- "^chain/actors/builtin/[^/]*/(message|state|v)[0-4]\\.go$" # We test the latest version only.
# Tests
- "api/test/**"
- "conformance/**"
# Generators
- "gen/**"
- "chain/actors/agen/**"
# Non-critical utilities
- "api/docgen/**"
- "api/docgen-openrpc/**"
coverage:
status:
patch: off
project:
threshold: 1%
tools-and-tests:
target: auto
threshold: 1%
threshold: 0.5%
informational: true
paths:
- "testplans"
@@ -34,28 +17,28 @@ coverage:
- "build"
markets:
target: auto
threshold: 1%
informational: true
threshold: 0.5%
informational: false
paths:
- "markets"
- "paychmgr"
miner:
target: auto
threshold: 1.5%
informational: true
threshold: 0.5%
informational: false
paths:
- "miner"
- "storage"
chain:
target: auto
threshold: 1%
informational: true
threshold: 0.5%
informational: false
paths:
- "chain"
node:
node:
target: auto
threshold: 1%
informational: true
threshold: 0.5%
informational: false
paths:
- "node"
- "blockstore"
@@ -67,8 +50,8 @@ coverage:
- "journal"
cli:
target: auto
threshold: 1%
threshold: 0.5%
informational: true
paths:
- "cli"
- "cmd"
- "cmd"
-21
View File
@@ -1,21 +0,0 @@
## Related Issues
<!-- link all issues that this PR might resolve/fix. If an issue doesn't exist, include a brief motivation for the change being made.-->
## Proposed Changes
<!-- provide a clear list of the changes being made-->
## Additional Info
<!-- callouts, links to documentation, and etc-->
## Checklist
Before you mark the PR ready for review, please make sure that:
- [ ] All commits have a clear commit message.
- [ ] The PR title is in the form of of `<PR type>: <area>: <change being made>`
- example: ` fix: mempool: Introduce a cache for valid signatures`
- `PR type`: _fix_, _feat_, _INTERFACE BREAKING CHANGE_, _CONSENSUS BREAKING_, _build_, _chore_, _ci_, _docs_,_perf_, _refactor_, _revert_, _style_, _test_
- `area`: _api_, _chain_, _state_, _vm_, _data transfer_, _market_, _mempool_, _message_, _block production_, _multisig_, _networking_, _paychan_, _proving_, _sealing_, _wallet_, _deps_
- [ ] This PR has tests for new functionality or change in behaviour
- [ ] If new user-facing features are introduced, clear usage guidelines and / or documentation updates should be included in https://lotus.filecoin.io or [Discussion Tutorials.](https://github.com/filecoin-project/lotus/discussions/categories/tutorials)
- [ ] CI is green
+1 -1
View File
@@ -37,7 +37,7 @@ jobs:
- uses: actions/setup-go@v1
with:
go-version: '1.17.9'
go-version: '1.16.4'
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
-14
View File
@@ -1,14 +0,0 @@
name: sync-master-main
on:
push:
branches:
- master
jobs:
sync:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: update remote branch main
run: |
# overrides the remote branch (origin:github) `main`
git push origin --force master:main
+1 -1
View File
@@ -21,7 +21,7 @@ jobs:
steps:
- uses: actions/checkout@v2
- name: testground run
uses: testground/testground-github-action@v1
uses: coryschwartz/testground-github-action@v1.1
with:
backend_addr: ${{ matrix.backend_addr }}
backend_proto: ${{ matrix.backend_proto }}
-4
View File
@@ -40,7 +40,6 @@ build/paramfetch.sh
/bundle
/darwin
/linux
*.snap
*-fuzz.zip
/chain/types/work_msg/
@@ -48,6 +47,3 @@ bin/ipget
bin/tmp/*
.idea
scratchpad
build/builtin-actors/v*
build/builtin-actors/*.car
+2 -5
View File
@@ -25,7 +25,7 @@ skip-dirs:
issues:
exclude:
- "by other packages, and that stutters; consider calling this"
- "func name will be used as test\\.Test.* by other packages, and that stutters; consider calling this"
- "Potential file inclusion via variable"
- "should have( a package)? comment"
- "Error return value of `logging.SetLogLevel` is not checked"
@@ -37,9 +37,6 @@ issues:
- "string .* has .* occurrences, make it a constant"
- "a blank import should be only in a main or test package, or have a comment justifying it"
- "package comment should be of the form"
- "Potential hardcoded credentials"
- "Use of weak random number generator"
- "xerrors.* is deprecated"
exclude-use-default: false
exclude-rules:
@@ -54,7 +51,7 @@ issues:
- path: build/params_.*\.go
linters:
- golint
- path: api/apistruct/struct.go
linters:
- golint
+7 -5
View File
@@ -49,23 +49,25 @@ AppDir:
fedora:
image: appimagecrafters/tests-env:fedora-30
command: ./AppRun
use_host_x: false
use_host_x: true
debian:
image: appimagecrafters/tests-env:debian-stable
command: ./AppRun
use_host_x: false
use_host_x: true
arch:
image: appimagecrafters/tests-env:archlinux-latest
command: ./AppRun
use_host_x: false
use_host_x: true
centos:
image: appimagecrafters/tests-env:centos-7
command: ./AppRun
use_host_x: false
use_host_x: true
ubuntu:
image: appimagecrafters/tests-env:ubuntu-xenial
command: ./AppRun
use_host_x: false
use_host_x: true
AppImage:
arch: x86_64
update-information: guess
sign-key: None
+155 -1485
View File
File diff suppressed because it is too large. Load diff
+24 -52
View File
@@ -1,4 +1,4 @@
FROM golang:1.17.9-buster AS builder-deps
FROM golang:1.16.4 AS builder-deps
MAINTAINER Lotus Development Team
RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
@@ -28,14 +28,6 @@ WORKDIR /opt/filecoin
RUN make clean deps
FROM builder-local AS builder-test
MAINTAINER Lotus Development Team
WORKDIR /opt/filecoin
RUN make debug
FROM builder-local AS builder
MAINTAINER Lotus Development Team
@@ -44,7 +36,7 @@ WORKDIR /opt/filecoin
ARG RUSTFLAGS=""
ARG GOFLAGS=""
RUN make lotus lotus-miner lotus-worker lotus-shed lotus-wallet lotus-gateway lotus-stats
RUN make lotus lotus-miner lotus-worker lotus-shed lotus-wallet lotus-gateway
FROM ubuntu:20.04 AS base
@@ -61,9 +53,8 @@ COPY --from=builder /usr/lib/x86_64-linux-gnu/libnuma.so.1 /lib/
COPY --from=builder /usr/lib/x86_64-linux-gnu/libhwloc.so.5 /lib/
COPY --from=builder /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /lib/
RUN useradd -r -u 532 -U fc \
&& mkdir -p /etc/OpenCL/vendors \
&& echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
RUN useradd -r -u 532 -U fc
###
FROM base AS lotus
@@ -75,6 +66,8 @@ COPY scripts/docker-lotus-entrypoint.sh /
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
ENV LOTUS_PATH /var/lib/lotus
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
ENV LOTUS_JAEGER_AGENT_PORT 6831
ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
ENV DOCKER_LOTUS_IMPORT_WALLET ""
@@ -99,6 +92,8 @@ MAINTAINER Lotus Development Team
COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/
ENV WALLET_PATH /var/lib/lotus-wallet
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
ENV LOTUS_JAEGER_AGENT_PORT 6831
RUN mkdir /var/lib/lotus-wallet
RUN chown fc: /var/lib/lotus-wallet
@@ -119,6 +114,10 @@ MAINTAINER Lotus Development Team
COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
ENV LOTUS_JAEGER_AGENT_PORT 6831
ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
USER fc
EXPOSE 1234
@@ -136,7 +135,11 @@ COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/
COPY scripts/docker-lotus-miner-entrypoint.sh /
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
ENV LOTUS_MINER_PATH /var/lib/lotus-miner
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
ENV LOTUS_JAEGER_AGENT_PORT 6831
ENV DOCKER_LOTUS_MINER_INIT true
RUN mkdir /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters
RUN chown fc: /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters
@@ -160,7 +163,10 @@ MAINTAINER Lotus Development Team
COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
ENV MINER_API_INFO /ip4/127.0.0.1/tcp/2345/http
ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
ENV LOTUS_JAEGER_AGENT_PORT 6831
RUN mkdir /var/lib/lotus-worker
RUN chown fc: /var/lib/lotus-worker
@@ -180,11 +186,16 @@ CMD ["-help"]
from base as lotus-all-in-one
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
ENV LOTUS_JAEGER_AGENT_PORT 6831
ENV LOTUS_MINER_PATH /var/lib/lotus-miner
ENV LOTUS_PATH /var/lib/lotus
ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
ENV MINER_API_INFO /ip4/127.0.0.1/tcp/2345/http
ENV WALLET_PATH /var/lib/lotus-wallet
ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
ENV DOCKER_LOTUS_MINER_INIT true
COPY --from=builder /opt/filecoin/lotus /usr/local/bin/
COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/
@@ -192,7 +203,6 @@ COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/
COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/
COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/
COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/
COPY --from=builder /opt/filecoin/lotus-stats /usr/local/bin/
RUN mkdir /var/tmp/filecoin-proof-parameters
RUN mkdir /var/lib/lotus
@@ -216,41 +226,3 @@ EXPOSE 1234
EXPOSE 2345
EXPOSE 3456
EXPOSE 1777
###
from base as lotus-test
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
ENV LOTUS_MINER_PATH /var/lib/lotus-miner
ENV LOTUS_PATH /var/lib/lotus
ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
ENV WALLET_PATH /var/lib/lotus-wallet
COPY --from=builder-test /opt/filecoin/lotus /usr/local/bin/
COPY --from=builder-test /opt/filecoin/lotus-miner /usr/local/bin/
COPY --from=builder-test /opt/filecoin/lotus-worker /usr/local/bin/
COPY --from=builder-test /opt/filecoin/lotus-seed /usr/local/bin/
RUN mkdir /var/tmp/filecoin-proof-parameters
RUN mkdir /var/lib/lotus
RUN mkdir /var/lib/lotus-miner
RUN mkdir /var/lib/lotus-worker
RUN mkdir /var/lib/lotus-wallet
RUN chown fc: /var/tmp/filecoin-proof-parameters
RUN chown fc: /var/lib/lotus
RUN chown fc: /var/lib/lotus-miner
RUN chown fc: /var/lib/lotus-worker
RUN chown fc: /var/lib/lotus-wallet
VOLUME /var/tmp/filecoin-proof-parameters
VOLUME /var/lib/lotus
VOLUME /var/lib/lotus-miner
VOLUME /var/lib/lotus-worker
VOLUME /var/lib/lotus-wallet
EXPOSE 1234
EXPOSE 2345
EXPOSE 3456
EXPOSE 1777
+13 -38
View File
@@ -10,7 +10,7 @@ GOCC?=go
GOVERSION:=$(shell $(GOCC) version | tr ' ' '\n' | grep go1 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
ifeq ($(shell expr $(GOVERSION) \< 1016000), 1)
$(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell expr $(GOVERSION) % 1000000 / 1000).$(shell expr $(GOVERSION) % 1000))
$(error Update Golang to version to at least 1.17.9)
$(error Update Golang to version to at least 1.16.0)
endif
# git modules that need to be loaded
@@ -64,7 +64,7 @@ CLEAN+=build/.update-modules
deps: $(BUILD_DEPS)
.PHONY: deps
build-devnets: build lotus-seed lotus-shed lotus-wallet lotus-gateway lotus-fountain lotus-stats
build-devnets: build lotus-seed lotus-shed lotus-wallet lotus-gateway
.PHONY: build-devnets
debug: GOFLAGS+=-tags=debug
@@ -76,6 +76,9 @@ debug: build-devnets
calibnet: GOFLAGS+=-tags=calibnet
calibnet: build-devnets
nerpanet: GOFLAGS+=-tags=nerpanet
nerpanet: build-devnets
butterflynet: GOFLAGS+=-tags=butterflynet
butterflynet: build-devnets
@@ -97,7 +100,7 @@ BINS+=lotus-miner
lotus-worker: $(BUILD_DEPS)
rm -f lotus-worker
$(GOCC) build $(GOFLAGS) -o lotus-worker ./cmd/lotus-worker
$(GOCC) build $(GOFLAGS) -o lotus-worker ./cmd/lotus-seal-worker
.PHONY: lotus-worker
BINS+=lotus-worker
@@ -133,18 +136,6 @@ install-worker:
install-app:
install -C ./$(APP) /usr/local/bin/$(APP)
uninstall: uninstall-daemon uninstall-miner uninstall-worker
.PHONY: uninstall
uninstall-daemon:
rm -f /usr/local/bin/lotus
uninstall-miner:
rm -f /usr/local/bin/lotus-miner
uninstall-worker:
rm -f /usr/local/bin/lotus-worker
# TOOLS
lotus-seed: $(BUILD_DEPS)
@@ -181,7 +172,7 @@ BINS+=lotus-fountain
lotus-bench:
rm -f lotus-bench
$(GOCC) build $(GOFLAGS) -o lotus-bench ./cmd/lotus-bench
$(GOCC) build -o lotus-bench ./cmd/lotus-bench
.PHONY: lotus-bench
BINS+=lotus-bench
@@ -205,7 +196,7 @@ BINS+=lotus-health
lotus-wallet:
rm -f lotus-wallet
$(GOCC) build $(GOFLAGS) -o lotus-wallet ./cmd/lotus-wallet
$(GOCC) build -o lotus-wallet ./cmd/lotus-wallet
.PHONY: lotus-wallet
BINS+=lotus-wallet
@@ -303,16 +294,9 @@ method-gen: api-gen
(cd ./lotuspond/front/src/chain && $(GOCC) run ./methodgen.go)
actors-gen:
$(GOCC) run ./gen/inline-gen . gen/inlinegen-data.json
$(GOCC) run ./chain/actors/agen
$(GOCC) fmt ./...
bundle-gen:
$(GOCC) run ./gen/bundle
$(GOCC) fmt ./build/...
.PHONY: bundle-gen
api-gen:
$(GOCC) run ./gen/api
goimports -w api
@@ -331,7 +315,7 @@ appimage: lotus
cp ./lotus AppDir/usr/bin/
appimage-builder
docsgen: docsgen-md docsgen-openrpc fiximports
docsgen: docsgen-md docsgen-openrpc
docsgen-md-bin: api-gen actors-gen
$(GOCC) build $(GOFLAGS) -o docgen-md ./api/docgen/cmd
@@ -348,7 +332,7 @@ docsgen-md-storage: docsgen-md-bin
docsgen-md-worker: docsgen-md-bin
./docgen-md "api/api_worker.go" "Worker" "api" "./api" > documentation/en/api-v0-methods-worker.md
docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker docsgen-openrpc-gateway
docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker
docsgen-openrpc-full: docsgen-openrpc-bin
./docgen-openrpc "api/api_full.go" "FullNode" "api" "./api" -gzip > build/openrpc/full.json.gz
@@ -356,20 +340,13 @@ docsgen-openrpc-storage: docsgen-openrpc-bin
./docgen-openrpc "api/api_storage.go" "StorageMiner" "api" "./api" -gzip > build/openrpc/miner.json.gz
docsgen-openrpc-worker: docsgen-openrpc-bin
./docgen-openrpc "api/api_worker.go" "Worker" "api" "./api" -gzip > build/openrpc/worker.json.gz
docsgen-openrpc-gateway: docsgen-openrpc-bin
./docgen-openrpc "api/api_gateway.go" "Gateway" "api" "./api" -gzip > build/openrpc/gateway.json.gz
.PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin
fiximports:
./scripts/fiximports
gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci bundle-gen fiximports
@echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO MAKE docsgen-cli"
gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci
@echo ">>> IF YOU'VE MODIFIED THE CLI, REMEMBER TO ALSO MAKE docsgen-cli"
.PHONY: gen
jen: gen
snap: lotus lotus-miner lotus-worker
snapcraft
# snapcraft upload ./lotus_*.snap
@@ -377,12 +354,10 @@ snap: lotus lotus-miner lotus-worker
# separate from gen because it needs binaries
docsgen-cli: lotus lotus-miner lotus-worker
python ./scripts/generate-lotus-cli.py
./lotus config default > documentation/en/default-lotus-config.toml
./lotus-miner config default > documentation/en/default-lotus-miner-config.toml
.PHONY: docsgen-cli
print-%:
@echo $*=$($*)
circleci:
go generate -x ./.circleci
go generate -x ./.circleci
+6 -5
View File
@@ -10,7 +10,7 @@
<a href="https://circleci.com/gh/filecoin-project/lotus"><img src="https://circleci.com/gh/filecoin-project/lotus.svg?style=svg"></a>
<a href="https://codecov.io/gh/filecoin-project/lotus"><img src="https://codecov.io/gh/filecoin-project/lotus/branch/master/graph/badge.svg"></a>
<a href="https://goreportcard.com/report/github.com/filecoin-project/lotus"><img src="https://goreportcard.com/badge/github.com/filecoin-project/lotus" /></a>
<a href=""><img src="https://img.shields.io/badge/golang-%3E%3D1.17-blue.svg" /></a>
<a href=""><img src="https://img.shields.io/badge/golang-%3E%3D1.16-blue.svg" /></a>
<br>
</p>
@@ -20,7 +20,7 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more
> Note: The default `master` branch is the dev branch, please use with caution. For the latest stable version, checkout the most recent [`Latest release`](https://github.com/filecoin-project/lotus/releases).
For complete instructions on how to build, install and setup lotus, please visit [https://lotus.filecoin.io](https://lotus.filecoin.io/lotus/install/prerequisites/#supported-platforms). Basic build instructions can be found further down in this readme.
For complete instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/). Basic build instructions can be found further down in this readme.
## Reporting a Vulnerability
@@ -64,17 +64,17 @@ sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config cu
Fedora:
```
sudo dnf -y install gcc make git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm wget hwloc hwloc-devel
sudo dnf -y install gcc make git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm wget hwloc libhwloc-dev
```
For other distributions you can find the required dependencies [here.](https://docs.filecoin.io/get-started/lotus/installation/#system-specific) For instructions specific to macOS, you can find them [here.](https://docs.filecoin.io/get-started/lotus/installation/#macos)
#### Go
To build Lotus, you need a working installation of [Go 1.17.9 or higher](https://golang.org/dl/):
To build Lotus, you need a working installation of [Go 1.16.4 or higher](https://golang.org/dl/):
```bash
wget -c https://golang.org/dl/go1.17.9.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
wget -c https://golang.org/dl/go1.16.4.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
```
**TIP:**
@@ -123,6 +123,7 @@ Note: The default branch `master` is the dev branch where the latest new feature
# Or to join a testnet or devnet:
make clean calibnet # Calibration with min 32GiB sectors
make clean nerpanet # Nerpa with min 512MiB sectors
sudo make install
```
+2 -7
View File
@@ -4,12 +4,11 @@ import (
"context"
"fmt"
apitypes "github.com/filecoin-project/lotus/api/types"
"github.com/google/uuid"
"github.com/filecoin-project/go-jsonrpc/auth"
apitypes "github.com/filecoin-project/lotus/api/types"
"github.com/filecoin-project/lotus/journal/alerting"
)
// MODIFYING THE API INTERFACE
@@ -34,10 +33,6 @@ type Common interface {
LogList(context.Context) ([]string, error) //perm:write
LogSetLevel(context.Context, string, string) error //perm:write
// LogAlerts returns list of all, active and inactive alerts tracked by the
// node
LogAlerts(ctx context.Context) ([]alerting.Alert, error) //perm:admin
// MethodGroup: Common
// Version provides information about API provider
+46 -94
View File
@@ -6,29 +6,28 @@ import (
"fmt"
"time"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/builtin/v8/market"
"github.com/filecoin-project/go-state-types/builtin/v8/miner"
"github.com/filecoin-project/go-state-types/builtin/v8/paych"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/go-state-types/dline"
abinetwork "github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/storagemarket"
apitypes "github.com/filecoin-project/lotus/api/types"
"github.com/filecoin-project/lotus/chain/actors/builtin"
lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
"github.com/filecoin-project/lotus/chain/actors/builtin/power"
"github.com/filecoin-project/lotus/chain/types"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo/imports"
)
@@ -39,7 +38,6 @@ import (
type ChainIO interface {
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
ChainHasObj(context.Context, cid.Cid) (bool, error)
ChainPutObj(context.Context, blocks.Block) error
}
const LookbackNoLimit = abi.ChainEpoch(-1)
@@ -74,6 +72,12 @@ type FullNode interface {
// ChainHead returns the current head of the chain.
ChainHead(context.Context) (*types.TipSet, error) //perm:read
// ChainGetRandomnessFromTickets is used to sample the chain for randomness.
ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) //perm:read
// ChainGetRandomnessFromBeacon is used to sample the beacon for randomness.
ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) //perm:read
// ChainGetBlock returns the block specified by the given CID.
ChainGetBlock(context.Context, cid.Cid) (*types.BlockHeader, error) //perm:read
// ChainGetTipSet returns the tipset specified by the given TipSetKey.
@@ -125,9 +129,6 @@ type FullNode interface {
// ChainHasObj checks if a given CID exists in the chain blockstore.
ChainHasObj(context.Context, cid.Cid) (bool, error) //perm:read
// ChainPutObj puts a given object into the block store
ChainPutObj(context.Context, blocks.Block) error //perm:admin
// ChainStatObj returns statistics about the graph referenced by 'obj'.
// If 'base' is also specified, then the returned stat will be a diff
// between the two objects.
@@ -176,6 +177,14 @@ type FullNode interface {
// ChainBlockstoreInfo returns some basic information about the blockstore
ChainBlockstoreInfo(context.Context) (map[string]interface{}, error) //perm:read
// MethodGroup: Beacon
// The Beacon method group contains methods for interacting with the random beacon (DRAND)
// BeaconGetEntry returns the beacon entry for the given filecoin epoch. If
// the entry has not yet been produced, the call will block until the entry
// becomes available
BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) //perm:read
// GasEstimateFeeCap estimates gas fee cap
GasEstimateFeeCap(context.Context, *types.Message, int64, types.TipSetKey) (types.BigInt, error) //perm:read
@@ -287,7 +296,7 @@ type FullNode interface {
// // UX ?
// MethodGroup: WalletF
// MethodGroup: Wallet
// WalletNew creates a new address in the wallet with the given sigType.
// Available key types: bls, secp256k1, secp256k1-ledger
@@ -348,17 +357,16 @@ type FullNode interface {
// ClientMinerQueryOffer returns a QueryOffer for the specific miner and file.
ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (QueryOffer, error) //perm:read
// ClientRetrieve initiates the retrieval of a file, as specified in the order.
ClientRetrieve(ctx context.Context, params RetrievalOrder) (*RestrievalRes, error) //perm:admin
// ClientRetrieveWait waits for retrieval to be complete
ClientRetrieveWait(ctx context.Context, deal retrievalmarket.DealID) error //perm:admin
// ClientExport exports a file stored in the local filestore to a system file
ClientExport(ctx context.Context, exportRef ExportRef, fileRef FileRef) error //perm:admin
ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *FileRef) error //perm:admin
// ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel
// of status updates.
ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin
// ClientListRetrievals returns information about retrievals made by the local client
ClientListRetrievals(ctx context.Context) ([]RetrievalInfo, error) //perm:write
// ClientGetRetrievalUpdates returns status of updated retrieval deals
ClientGetRetrievalUpdates(ctx context.Context) (<-chan RetrievalInfo, error) //perm:write
// ClientQueryAsk returns a signed StorageAsk from the specified miner.
ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*StorageAsk, error) //perm:read
ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error) //perm:read
// ClientCalcCommP calculates the CommP and data size of the specified CID
ClientDealPieceCID(ctx context.Context, root cid.Cid) (DataCIDSize, error) //perm:read
// ClientCalcCommP calculates the CommP for a specified file
@@ -441,7 +449,7 @@ type FullNode interface {
// StateMinerPower returns the power of the indicated miner
StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error) //perm:read
// StateMinerInfo returns info about the indicated miner
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (MinerInfo, error) //perm:read
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) //perm:read
// StateMinerDeadlines returns all the proving deadlines for the given miner
StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]Deadline, error) //perm:read
// StateMinerPartitions returns all partitions in the specified deadline
@@ -458,23 +466,18 @@ type FullNode interface {
StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) //perm:read
// StateMinerAvailableBalance returns the portion of a miner's balance that can be withdrawn or spent
StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) //perm:read
// StateMinerSectorAllocated checks if a sector number is marked as allocated.
// StateMinerSectorAllocated checks if a sector is allocated
StateMinerSectorAllocated(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (bool, error) //perm:read
// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector.
// Returns nil and no error if the sector isn't precommitted.
//
// Note that the sector number may be allocated while PreCommitInfo is nil. This means that either allocated sector
// numbers were compacted, and the sector number was marked as allocated in order to reduce size of the allocated
// sectors bitfield, or that the sector was precommitted, but the precommit has expired.
StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorPreCommitOnChainInfo, error) //perm:read
// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector
StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) //perm:read
// StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found
// NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate
// expiration epoch
StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) //perm:read
// StateSectorExpiration returns epoch at which given sector will expire
StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*lminer.SectorExpiration, error) //perm:read
StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error) //perm:read
// StateSectorPartition finds deadline/partition with the specified sector
StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*lminer.SectorLocation, error) //perm:read
StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) //perm:read
// StateSearchMsg looks back up to limit epochs in the chain for a message, and returns its receipt and the tipset where it was executed
//
// NOTE: If a replacing message is found on chain, this method will return
@@ -520,17 +523,13 @@ type FullNode interface {
// StateMarketParticipants returns the Escrow and Locked balances of every participant in the Storage Market
StateMarketParticipants(context.Context, types.TipSetKey) (map[string]MarketBalance, error) //perm:read
// StateMarketDeals returns information about every deal in the Storage Market
StateMarketDeals(context.Context, types.TipSetKey) (map[string]*MarketDeal, error) //perm:read
StateMarketDeals(context.Context, types.TipSetKey) (map[string]MarketDeal, error) //perm:read
// StateMarketStorageDeal returns information about the indicated deal
StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*MarketDeal, error) //perm:read
// StateComputeDataCID computes DataCID from a set of on-chain deals
StateComputeDataCID(ctx context.Context, maddr address.Address, sectorType abi.RegisteredSealProof, deals []abi.DealID, tsk types.TipSetKey) (cid.Cid, error) //perm:read
// StateLookupID retrieves the ID address of the given address
StateLookupID(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
// StateAccountKey returns the public key address of the given ID address for secp and bls accounts
// StateAccountKey returns the public key address of the given ID address
StateAccountKey(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
// StateLookupRobustAddress returns the public key address of the given ID address for non-account addresses (multisig, miners etc)
StateLookupRobustAddress(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
// StateChangedActors returns all the actors whose states change between the two given state CIDs
// TODO: Should this take tipset keys instead?
StateChangedActors(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) //perm:read
@@ -591,21 +590,6 @@ type FullNode interface {
StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (CirculatingSupply, error) //perm:read
// StateNetworkVersion returns the network version at the given tipset
StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) //perm:read
// StateActorCodeCIDs returns the CIDs of all the builtin actors for the given network version
StateActorCodeCIDs(context.Context, abinetwork.Version) (map[string]cid.Cid, error) //perm:read
// StateGetRandomnessFromTickets is used to sample the chain for randomness.
StateGetRandomnessFromTickets(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
// StateGetRandomnessFromBeacon is used to sample the beacon for randomness.
StateGetRandomnessFromBeacon(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
// StateGetBeaconEntry returns the beacon entry for the given filecoin epoch. If
// the entry has not yet been produced, the call will block until the entry
// becomes available
StateGetBeaconEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) //perm:read
// StateGetNetworkParams return current network params
StateGetNetworkParams(ctx context.Context) (*NetworkParams, error) //perm:read
// MethodGroup: Msig
// The Msig methods are used to interact with multisig wallets on the
@@ -646,14 +630,10 @@ type FullNode interface {
// <sender address of the approve msg>, <method to call in the proposed message>, <params to include in the proposed message>
MsigApproveTxnHash(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (*MessagePrototype, error) //perm:sign
// MsigCancel cancels a previously-proposed multisig message
// It takes the following params: <multisig address>, <proposed transaction ID> <signer address>
MsigCancel(context.Context, address.Address, uint64, address.Address) (*MessagePrototype, error) //perm:sign
// MsigCancel cancels a previously-proposed multisig message
// It takes the following params: <multisig address>, <proposed transaction ID>, <recipient address>, <value to transfer>,
// <sender address of the cancel msg>, <method to call in the proposed message>, <params to include in the proposed message>
MsigCancelTxnHash(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (*MessagePrototype, error) //perm:sign
MsigCancel(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (*MessagePrototype, error) //perm:sign
// MsigAddPropose proposes adding a signer in the multisig
// It takes the following params: <multisig address>, <sender address of the propose msg>,
@@ -706,17 +686,7 @@ type FullNode interface {
// MethodGroup: Paych
// The Paych methods are for interacting with and managing payment channels
// PaychGet gets or creates a payment channel between address pair
// The specified amount will be reserved for use. If there aren't enough non-reserved funds
// available, funds will be added through an on-chain message.
// - When opts.OffChain is true, this call will not cause any messages to be sent to the chain (no automatic
// channel creation/funds adding). If the operation can't be performed without sending a message an error will be
// returned. Note that even when this option is specified, this call can be blocked by previous operations on the
// channel waiting for on-chain operations.
PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt, opts PaychGetOpts) (*ChannelInfo, error) //perm:sign
// PaychFund gets or creates a payment channel between address pair.
// The specified amount will be added to the channel through on-chain send for future use
PaychFund(ctx context.Context, from, to address.Address, amt types.BigInt) (*ChannelInfo, error) //perm:sign
PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*ChannelInfo, error) //perm:sign
PaychGetWaitReady(context.Context, cid.Cid) (address.Address, error) //perm:sign
PaychAvailableFunds(ctx context.Context, ch address.Address) (*ChannelAvailableFunds, error) //perm:sign
PaychAvailableFundsByFromTo(ctx context.Context, from, to address.Address) (*ChannelAvailableFunds, error) //perm:sign
@@ -745,12 +715,6 @@ type FullNode interface {
CreateBackup(ctx context.Context, fpath string) error //perm:admin
}
type StorageAsk struct {
Response *storagemarket.StorageAsk
DealProtocols []string
}
type FileRef struct {
Path string
IsCAR bool
@@ -861,10 +825,6 @@ const (
PCHOutbound
)
type PaychGetOpts struct {
OffChain bool
}
type PaychStatus struct {
ControlAddr address.Address
Direction PCHDir
@@ -882,23 +842,16 @@ type ChannelAvailableFunds struct {
From address.Address
// To is the to address of the channel
To address.Address
// ConfirmedAmt is the total amount of funds that have been confirmed on-chain for the channel
// ConfirmedAmt is the amount of funds that have been confirmed on-chain
// for the channel
ConfirmedAmt types.BigInt
// PendingAmt is the amount of funds that are pending confirmation on-chain
PendingAmt types.BigInt
// NonReservedAmt is part of ConfirmedAmt that is available for use (e.g. when the payment channel was pre-funded)
NonReservedAmt types.BigInt
// PendingAvailableAmt is the amount of funds that are pending confirmation on-chain that will become available once confirmed
PendingAvailableAmt types.BigInt
// PendingWaitSentinel can be used with PaychGetWaitReady to wait for
// confirmation of pending funds
PendingWaitSentinel *cid.Cid
// QueuedAmt is the amount that is queued up behind a pending request
QueuedAmt types.BigInt
// VoucherRedeemedAmt is the amount that is redeemed by vouchers on-chain
// and in the local datastore
VoucherReedeemedAmt types.BigInt
@@ -944,7 +897,6 @@ type QueryOffer struct {
Size uint64
MinPrice types.BigInt
UnsealPrice types.BigInt
PricePerByte abi.TokenAmount
PaymentInterval uint64
PaymentIntervalIncrease uint64
Miner address.Address
@@ -978,14 +930,14 @@ type MarketDeal struct {
}
type RetrievalOrder struct {
Root cid.Cid
Piece *cid.Cid
DataSelector *Selector
// todo: Size/Total are only used for calculating price per byte; we should let users just pass that
// TODO: make this less unixfs specific
Root cid.Cid
Piece *cid.Cid
Size uint64
Total types.BigInt
FromLocalCAR string // if specified, get data from a local CARv2 file.
// TODO: support offset
Total types.BigInt
UnsealPrice types.BigInt
PaymentInterval uint64
PaymentIntervalIncrease uint64
@@ -1128,7 +1080,7 @@ type CirculatingSupply struct {
type MiningBaseInfo struct {
MinerPower types.BigInt
NetworkPower types.BigInt
Sectors []builtin.ExtendedSectorInfo
Sectors []builtin.SectorInfo
WorkerKey address.Address
SectorSize abi.SectorSize
PrevBeaconEntry types.BeaconEntry
+3 -11
View File
@@ -3,15 +3,14 @@ package api
import (
"context"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/builtin/v8/miner"
"github.com/filecoin-project/go-state-types/dline"
apitypes "github.com/filecoin-project/lotus/api/types"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/types"
)
@@ -31,25 +30,19 @@ import (
type Gateway interface {
ChainHasObj(context.Context, cid.Cid) (bool, error)
ChainPutObj(context.Context, blocks.Block) error
ChainHead(ctx context.Context) (*types.TipSet, error)
ChainGetParentMessages(context.Context, cid.Cid) ([]Message, error)
ChainGetParentReceipts(context.Context, cid.Cid) ([]*types.MessageReceipt, error)
ChainGetBlockMessages(context.Context, cid.Cid) (*BlockMessages, error)
ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*HeadChange, error)
ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error)
ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error)
ChainGetTipSetAfterHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error)
ChainNotify(context.Context) (<-chan []*HeadChange, error)
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
ChainGetGenesis(context.Context) (*types.TipSet, error)
GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MsigVesting, error)
MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
@@ -58,7 +51,7 @@ type Gateway interface {
StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MarketBalance, error)
StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*MarketDeal, error)
StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (MinerInfo, error)
StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error)
StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error)
StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error)
@@ -68,5 +61,4 @@ type Gateway interface {
StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
WalletBalance(context.Context, address.Address) (types.BigInt, error)
Version(context.Context) (APIVersion, error)
Discover(context.Context) (apitypes.OpenRPCDocument, error)
}
+1 -12
View File
@@ -2,9 +2,8 @@ package api
import (
"context"
"time"
"github.com/libp2p/go-libp2p-core/metrics"
metrics "github.com/libp2p/go-libp2p-core/metrics"
"github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/libp2p/go-libp2p-core/protocol"
@@ -26,7 +25,6 @@ type Net interface {
NetConnectedness(context.Context, peer.ID) (network.Connectedness, error) //perm:read
NetPeers(context.Context) ([]peer.AddrInfo, error) //perm:read
NetPing(context.Context, peer.ID) (time.Duration, error) //perm:read
NetConnect(context.Context, peer.AddrInfo) error //perm:write
NetAddrsListen(context.Context) (peer.AddrInfo, error) //perm:read
NetDisconnect(context.Context, peer.ID) error //perm:write
@@ -53,15 +51,6 @@ type Net interface {
NetBlockRemove(ctx context.Context, acl NetBlockList) error //perm:admin
NetBlockList(ctx context.Context) (NetBlockList, error) //perm:read
NetProtectAdd(ctx context.Context, acl []peer.ID) error //perm:admin
NetProtectRemove(ctx context.Context, acl []peer.ID) error //perm:admin
NetProtectList(ctx context.Context) ([]peer.ID, error) //perm:read
// ResourceManager API
NetStat(ctx context.Context, scope string) (NetStat, error) //perm:read
NetLimit(ctx context.Context, scope string) (NetLimit, error) //perm:read
NetSetLimit(ctx context.Context, scope string, limit NetLimit) error //perm:admin
// ID returns peerID of libp2p node backing this API
ID(context.Context) (peer.ID, error) //perm:read
}
+56 -93
View File
@@ -5,26 +5,27 @@ import (
"context"
"time"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/google/uuid"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/go-address"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
"github.com/filecoin-project/specs-storage/storage"
"github.com/filecoin-project/go-fil-markets/piecestore"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/builtin/v8/market"
"github.com/filecoin-project/go-state-types/builtin/v8/miner"
abinetwork "github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/storage/pipeline/sealiface"
"github.com/filecoin-project/lotus/storage/sealer/fsutil"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
)
// MODIFYING THE API INTERFACE
@@ -50,10 +51,6 @@ type StorageMiner interface {
MiningBase(context.Context) (*types.TipSet, error) //perm:read
ComputeWindowPoSt(ctx context.Context, dlIdx uint64, tsk types.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) //perm:admin
ComputeDataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (abi.PieceInfo, error) //perm:admin
// Temp api for testing
PledgeSector(context.Context) (abi.SectorID, error) //perm:write
@@ -63,9 +60,9 @@ type StorageMiner interface {
// Add piece to an open sector. If no sectors with enough space are open,
// either a new sector will be created, or this call will block until more
// sectors can be created.
SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storiface.Data, d PieceDealInfo) (SectorOffset, error) //perm:admin
SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storage.Data, d PieceDealInfo) (SectorOffset, error) //perm:admin
SectorsUnsealPiece(ctx context.Context, sector storiface.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error //perm:admin
SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error //perm:admin
// List all staged sectors
SectorsList(context.Context) ([]abi.SectorNumber, error) //perm:read
@@ -94,8 +91,7 @@ type StorageMiner interface {
SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error //perm:admin
// SectorRemove removes the sector from storage. It doesn't terminate it on-chain, which can
// be done with SectorTerminate. Removing and not terminating live sectors will cause additional penalties.
SectorRemove(context.Context, abi.SectorNumber) error //perm:admin
SectorMarkForUpgrade(ctx context.Context, id abi.SectorNumber, snap bool) error //perm:admin
SectorRemove(context.Context, abi.SectorNumber) error //perm:admin
// SectorTerminate terminates the sector on-chain (adding it to a termination batch first), then
// automatically removes it from storage
SectorTerminate(context.Context, abi.SectorNumber) error //perm:admin
@@ -103,7 +99,8 @@ type StorageMiner interface {
// Returns null if message wasn't sent
SectorTerminateFlush(ctx context.Context) (*cid.Cid, error) //perm:admin
// SectorTerminatePending returns a list of pending sector terminations to be sent in the next batch message
SectorTerminatePending(ctx context.Context) ([]abi.SectorID, error) //perm:admin
SectorTerminatePending(ctx context.Context) ([]abi.SectorID, error) //perm:admin
SectorMarkForUpgrade(ctx context.Context, id abi.SectorNumber) error //perm:admin
// SectorPreCommitFlush immediately sends a PreCommit message with sectors batched for PreCommit.
// Returns null if message wasn't sent
SectorPreCommitFlush(ctx context.Context) ([]sealiface.PreCommitBatchRes, error) //perm:admin
@@ -114,9 +111,6 @@ type StorageMiner interface {
SectorCommitFlush(ctx context.Context) ([]sealiface.CommitBatchRes, error) //perm:admin
// SectorCommitPending returns a list of pending Commit sectors to be sent in the next aggregate message
SectorCommitPending(ctx context.Context) ([]abi.SectorID, error) //perm:admin
SectorMatchPendingPiecesToOpenSectors(ctx context.Context) error //perm:admin
// SectorAbortUpgrade can be called on sectors that are in the process of being upgraded to abort it
SectorAbortUpgrade(context.Context, abi.SectorNumber) error //perm:admin
// WorkerConnect tells the node to connect to workers RPC
WorkerConnect(context.Context, string) error //perm:admin retry:true
@@ -124,48 +118,39 @@ type StorageMiner interface {
WorkerJobs(context.Context) (map[uuid.UUID][]storiface.WorkerJob, error) //perm:admin
//storiface.WorkerReturn
ReturnDataCid(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error //perm:admin retry:true
ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error //perm:admin retry:true
ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storiface.PreCommit1Out, err *storiface.CallError) error //perm:admin retry:true
ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storiface.SectorCids, err *storiface.CallError) error //perm:admin retry:true
ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storiface.Commit1Out, err *storiface.CallError) error //perm:admin retry:true
ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storiface.Proof, err *storiface.CallError) error //perm:admin retry:true
ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnReplicaUpdate(ctx context.Context, callID storiface.CallID, out storiface.ReplicaUpdateOut, err *storiface.CallError) error //perm:admin retry:true
ReturnProveReplicaUpdate1(ctx context.Context, callID storiface.CallID, vanillaProofs storiface.ReplicaVanillaProofs, err *storiface.CallError) error //perm:admin retry:true
ReturnProveReplicaUpdate2(ctx context.Context, callID storiface.CallID, proof storiface.ReplicaUpdateProof, err *storiface.CallError) error //perm:admin retry:true
ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err *storiface.CallError) error //perm:admin retry:true
ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error //perm:admin retry:true
ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storage.PreCommit1Out, err *storiface.CallError) error //perm:admin retry:true
ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storage.SectorCids, err *storiface.CallError) error //perm:admin retry:true
ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storage.Commit1Out, err *storiface.CallError) error //perm:admin retry:true
ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storage.Proof, err *storiface.CallError) error //perm:admin retry:true
ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err *storiface.CallError) error //perm:admin retry:true
ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
// SealingSchedDiag dumps internal sealing scheduler state
SealingSchedDiag(ctx context.Context, doSched bool) (interface{}, error) //perm:admin
SealingAbort(ctx context.Context, call storiface.CallID) error //perm:admin
// SectorIndex
StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error //perm:admin
StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error) //perm:admin
StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error //perm:admin
StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error //perm:admin
StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error //perm:admin
StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error) //perm:admin
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) //perm:admin
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) //perm:admin
StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error) //perm:admin
//stores.SectorIndex
StorageAttach(context.Context, stores.StorageInfo, fsutil.FsStat) error //perm:admin
StorageInfo(context.Context, stores.ID) (stores.StorageInfo, error) //perm:admin
StorageReportHealth(context.Context, stores.ID, stores.HealthReport) error //perm:admin
StorageDeclareSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error //perm:admin
StorageDropSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType) error //perm:admin
StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]stores.SectorStorageInfo, error) //perm:admin
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]stores.StorageInfo, error) //perm:admin
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) //perm:admin
StorageLocal(ctx context.Context) (map[storiface.ID]string, error) //perm:admin
StorageStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error) //perm:admin
StorageAuthVerify(ctx context.Context, token string) ([]auth.Permission, error) //perm:read
StorageLocal(ctx context.Context) (map[stores.ID]string, error) //perm:admin
StorageStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) //perm:admin
MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error //perm:write
MarketListDeals(ctx context.Context) ([]*MarketDeal, error) //perm:read
MarketListDeals(ctx context.Context) ([]MarketDeal, error) //perm:read
MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error) //perm:read
MarketGetDealUpdates(ctx context.Context) (<-chan storagemarket.MinerDeal, error) //perm:read
MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error) //perm:read
@@ -175,15 +160,12 @@ type StorageMiner interface {
MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error) //perm:read
MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error) //perm:write
MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error) //perm:write
// MarketDataTransferDiagnostics generates debugging information about current data transfers over graphsync
MarketDataTransferDiagnostics(ctx context.Context, p peer.ID) (*TransferDiagnostics, error) //perm:write
// MarketRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer
MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
// MarketCancelDataTransfer cancels a data transfer with the given transfer ID and other peer
MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
MarketPendingDeals(ctx context.Context) (PendingDealInfo, error) //perm:write
MarketPublishPendingDeals(ctx context.Context) error //perm:admin
MarketRetryPublishDeal(ctx context.Context, propcid cid.Cid) error //perm:admin
// DagstoreListShards returns information about all shards known to the
// DAG store. Only available on nodes running the markets subsystem.
@@ -227,25 +209,12 @@ type StorageMiner interface {
// DagstoreGC runs garbage collection on the DAG store.
DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin
// DagstoreRegisterShard registers a shard manually with dagstore with given pieceCID
DagstoreRegisterShard(ctx context.Context, key string) error //perm:admin
// IndexerAnnounceDeal informs indexer nodes that a new deal was received,
// so they can download its index
IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin
// IndexerAnnounceAllDeals informs the indexer nodes aboutall active deals.
IndexerAnnounceAllDeals(ctx context.Context) error //perm:admin
// DagstoreLookupPieces returns information about shards that contain the given CID.
DagstoreLookupPieces(ctx context.Context, cid cid.Cid) ([]DagstoreShardInfo, error) //perm:admin
// RuntimeSubsystems returns the subsystems that are enabled
// in this instance.
RuntimeSubsystems(ctx context.Context) (MinerSubsystems, error) //perm:read
DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error //perm:admin
DealsList(ctx context.Context) ([]*MarketDeal, error) //perm:admin
DealsList(ctx context.Context) ([]MarketDeal, error) //perm:admin
DealsConsiderOnlineStorageDeals(context.Context) (bool, error) //perm:admin
DealsSetConsiderOnlineStorageDeals(context.Context, bool) error //perm:admin
DealsConsiderOnlineRetrievalDeals(context.Context) (bool, error) //perm:admin
@@ -274,12 +243,13 @@ type StorageMiner interface {
// the path specified when calling CreateBackup is within the base path
CreateBackup(ctx context.Context, fpath string) error //perm:admin
CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storiface.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
ComputeProof(ctx context.Context, ssi []builtin.ExtendedSectorInfo, rand abi.PoStRandomness, poStEpoch abi.ChainEpoch, nv abinetwork.Version) ([]builtin.PoStProof, error) //perm:read
ComputeProof(ctx context.Context, ssi []builtin.SectorInfo, rand abi.PoStRandomness) ([]builtin.PoStProof, error) //perm:read
}
var _ storiface.WorkerReturn = *new(StorageMiner)
var _ stores.SectorIndex = *new(StorageMiner)
type SealRes struct {
Err string
@@ -297,26 +267,19 @@ type SectorLog struct {
Message string
}
type SectorPiece struct {
Piece abi.PieceInfo
DealInfo *PieceDealInfo // nil for pieces which do not appear in deals (e.g. filler pieces)
}
type SectorInfo struct {
SectorID abi.SectorNumber
State SectorState
CommD *cid.Cid
CommR *cid.Cid
Proof []byte
Deals []abi.DealID
Pieces []SectorPiece
Ticket SealTicket
Seed SealSeed
PreCommitMsg *cid.Cid
CommitMsg *cid.Cid
Retries uint64
ToUpgrade bool
ReplicaUpdateMessage *cid.Cid
SectorID abi.SectorNumber
State SectorState
CommD *cid.Cid
CommR *cid.Cid
Proof []byte
Deals []abi.DealID
Ticket SealTicket
Seed SealSeed
PreCommitMsg *cid.Cid
CommitMsg *cid.Cid
Retries uint64
ToUpgrade bool
LastErr string
-5
View File
@@ -1,4 +1,3 @@
//stm: #unit
package api
import (
@@ -27,7 +26,6 @@ func goCmd() string {
}
func TestDoesntDependOnFFI(t *testing.T) {
//stm: @OTHER_IMPLEMENTATION_FFI_DEPENDENCE_001
deps, err := exec.Command(goCmd(), "list", "-deps", "github.com/filecoin-project/lotus/api").Output()
if err != nil {
t.Fatal(err)
@@ -40,7 +38,6 @@ func TestDoesntDependOnFFI(t *testing.T) {
}
func TestDoesntDependOnBuild(t *testing.T) {
//stm: @OTHER_IMPLEMENTATION_FFI_DEPENDENCE_002
deps, err := exec.Command(goCmd(), "list", "-deps", "github.com/filecoin-project/lotus/api").Output()
if err != nil {
t.Fatal(err)
@@ -53,7 +50,6 @@ func TestDoesntDependOnBuild(t *testing.T) {
}
func TestReturnTypes(t *testing.T) {
//stm: @OTHER_IMPLEMENTATION_001
errType := reflect.TypeOf(new(error)).Elem()
bareIface := reflect.TypeOf(new(interface{})).Elem()
jmarsh := reflect.TypeOf(new(json.Marshaler)).Elem()
@@ -119,7 +115,6 @@ func TestReturnTypes(t *testing.T) {
}
func TestPermTags(t *testing.T) {
//stm: @OTHER_IMPLEMENTATION_PERM_TAGS_001
_ = PermissionedFullAPI(&FullNodeStruct{})
_ = PermissionedStorMinerAPI(&StorageMinerStruct{})
_ = PermissionedWorkerAPI(&WorkerStruct{})
+15 -24
View File
@@ -7,10 +7,10 @@ import (
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/proof"
"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
"github.com/filecoin-project/specs-storage/storage"
)
// MODIFYING THE API INTERFACE
@@ -29,29 +29,20 @@ type Worker interface {
// TaskType -> Weight
TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) //perm:admin
Paths(context.Context) ([]storiface.StoragePath, error) //perm:admin
Paths(context.Context) ([]stores.StoragePath, error) //perm:admin
Info(context.Context) (storiface.WorkerInfo, error) //perm:admin
// storiface.WorkerCalls
DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (storiface.CallID, error) //perm:admin
AddPiece(ctx context.Context, sector storiface.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (storiface.CallID, error) //perm:admin
SealPreCommit1(ctx context.Context, sector storiface.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error) //perm:admin
SealPreCommit2(ctx context.Context, sector storiface.SectorRef, pc1o storiface.PreCommit1Out) (storiface.CallID, error) //perm:admin
SealCommit1(ctx context.Context, sector storiface.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storiface.SectorCids) (storiface.CallID, error) //perm:admin
SealCommit2(ctx context.Context, sector storiface.SectorRef, c1o storiface.Commit1Out) (storiface.CallID, error) //perm:admin
FinalizeSector(ctx context.Context, sector storiface.SectorRef, keepUnsealed []storiface.Range) (storiface.CallID, error) //perm:admin
FinalizeReplicaUpdate(ctx context.Context, sector storiface.SectorRef, keepUnsealed []storiface.Range) (storiface.CallID, error) //perm:admin
ReplicaUpdate(ctx context.Context, sector storiface.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error) //perm:admin
ProveReplicaUpdate1(ctx context.Context, sector storiface.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error) //perm:admin
ProveReplicaUpdate2(ctx context.Context, sector storiface.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storiface.ReplicaVanillaProofs) (storiface.CallID, error) //perm:admin
GenerateSectorKeyFromData(ctx context.Context, sector storiface.SectorRef, commD cid.Cid) (storiface.CallID, error) //perm:admin
ReleaseUnsealed(ctx context.Context, sector storiface.SectorRef, safeToFree []storiface.Range) (storiface.CallID, error) //perm:admin
MoveStorage(ctx context.Context, sector storiface.SectorRef, types storiface.SectorFileType) (storiface.CallID, error) //perm:admin
UnsealPiece(context.Context, storiface.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error) //perm:admin
Fetch(context.Context, storiface.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error) //perm:admin
GenerateWinningPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, randomness abi.PoStRandomness) ([]proof.PoStProof, error) //perm:admin
GenerateWindowPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, partitionIdx int, randomness abi.PoStRandomness) (storiface.WindowPoStResult, error) //perm:admin
AddPiece(ctx context.Context, sector storage.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (storiface.CallID, error) //perm:admin
SealPreCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error) //perm:admin
SealPreCommit2(ctx context.Context, sector storage.SectorRef, pc1o storage.PreCommit1Out) (storiface.CallID, error) //perm:admin
SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error) //perm:admin
SealCommit2(ctx context.Context, sector storage.SectorRef, c1o storage.Commit1Out) (storiface.CallID, error) //perm:admin
FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error) //perm:admin
ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) (storiface.CallID, error) //perm:admin
MoveStorage(ctx context.Context, sector storage.SectorRef, types storiface.SectorFileType) (storiface.CallID, error) //perm:admin
UnsealPiece(context.Context, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error) //perm:admin
Fetch(context.Context, storage.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error) //perm:admin
TaskDisable(ctx context.Context, tt sealtasks.TaskType) error //perm:admin
TaskEnable(ctx context.Context, tt sealtasks.TaskType) error //perm:admin
+129 -172
View File
@@ -8,13 +8,12 @@ import (
"math"
"sort"
abi "github.com/filecoin-project/go-state-types/abi"
market "github.com/filecoin-project/specs-actors/actors/builtin/market"
paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
cid "github.com/ipfs/go-cid"
cbg "github.com/whyrusleeping/cbor-gen"
xerrors "golang.org/x/xerrors"
abi "github.com/filecoin-project/go-state-types/abi"
market "github.com/filecoin-project/go-state-types/builtin/v8/market"
paych "github.com/filecoin-project/go-state-types/builtin/v8/paych"
)
var _ = xerrors.Errorf
@@ -27,26 +26,25 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
cw := cbg.NewCborWriter(w)
if _, err := cw.Write([]byte{163}); err != nil {
if _, err := w.Write([]byte{163}); err != nil {
return err
}
scratch := make([]byte, 9)
// t.Channel (address.Address) (struct)
if len("Channel") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"Channel\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Channel"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Channel"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("Channel")); err != nil {
return err
}
if err := t.Channel.MarshalCBOR(cw); err != nil {
if err := t.Channel.MarshalCBOR(w); err != nil {
return err
}
@@ -55,14 +53,14 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("Value in field \"WaitSentinel\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("WaitSentinel"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("WaitSentinel"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("WaitSentinel")); err != nil {
return err
}
if err := cbg.WriteCid(cw, t.WaitSentinel); err != nil {
if err := cbg.WriteCidBuf(scratch, w, t.WaitSentinel); err != nil {
return xerrors.Errorf("failed to write cid field t.WaitSentinel: %w", err)
}
@@ -71,7 +69,7 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("Value in field \"Vouchers\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Vouchers"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Vouchers"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("Vouchers")); err != nil {
@@ -82,32 +80,27 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("Slice value in field t.Vouchers was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Vouchers))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Vouchers))); err != nil {
return err
}
for _, v := range t.Vouchers {
if err := v.MarshalCBOR(cw); err != nil {
if err := v.MarshalCBOR(w); err != nil {
return err
}
}
return nil
}
func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) (err error) {
func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
*t = PaymentInfo{}
cr := cbg.NewCborReader(r)
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
maj, extra, err := cr.ReadHeader()
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
defer func() {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
}()
if maj != cbg.MajMap {
return fmt.Errorf("cbor input should be of type map")
}
@@ -122,7 +115,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) (err error) {
for i := uint64(0); i < n; i++ {
{
sval, err := cbg.ReadString(cr)
sval, err := cbg.ReadStringBuf(br, scratch)
if err != nil {
return err
}
@@ -136,7 +129,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) (err error) {
{
if err := t.Channel.UnmarshalCBOR(cr); err != nil {
if err := t.Channel.UnmarshalCBOR(br); err != nil {
return xerrors.Errorf("unmarshaling t.Channel: %w", err)
}
@@ -146,7 +139,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) (err error) {
{
c, err := cbg.ReadCid(cr)
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.WaitSentinel: %w", err)
}
@@ -157,7 +150,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) (err error) {
// t.Vouchers ([]*paych.SignedVoucher) (slice)
case "Vouchers":
maj, extra, err = cr.ReadHeader()
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
@@ -177,7 +170,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) (err error) {
for i := 0; i < int(extra); i++ {
var v paych.SignedVoucher
if err := v.UnmarshalCBOR(cr); err != nil {
if err := v.UnmarshalCBOR(br); err != nil {
return err
}
@@ -197,26 +190,25 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
cw := cbg.NewCborWriter(w)
if _, err := cw.Write([]byte{163}); err != nil {
if _, err := w.Write([]byte{163}); err != nil {
return err
}
scratch := make([]byte, 9)
// t.SectorID (abi.SectorNumber) (uint64)
if len("SectorID") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"SectorID\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("SectorID"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SectorID"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("SectorID")); err != nil {
return err
}
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.SectorID)); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.SectorID)); err != nil {
return err
}
@@ -225,14 +217,14 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("Value in field \"Offset\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Offset"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Offset"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("Offset")); err != nil {
return err
}
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Offset)); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Offset)); err != nil {
return err
}
@@ -241,35 +233,30 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("Value in field \"Size\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Size"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Size"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("Size")); err != nil {
return err
}
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Size)); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Size)); err != nil {
return err
}
return nil
}
func (t *SealedRef) UnmarshalCBOR(r io.Reader) (err error) {
func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
*t = SealedRef{}
cr := cbg.NewCborReader(r)
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
maj, extra, err := cr.ReadHeader()
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
defer func() {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
}()
if maj != cbg.MajMap {
return fmt.Errorf("cbor input should be of type map")
}
@@ -284,7 +271,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) (err error) {
for i := uint64(0); i < n; i++ {
{
sval, err := cbg.ReadString(cr)
sval, err := cbg.ReadStringBuf(br, scratch)
if err != nil {
return err
}
@@ -298,7 +285,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) (err error) {
{
maj, extra, err = cr.ReadHeader()
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
@@ -313,7 +300,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) (err error) {
{
maj, extra, err = cr.ReadHeader()
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
@@ -328,7 +315,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) (err error) {
{
maj, extra, err = cr.ReadHeader()
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
@@ -352,19 +339,18 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
cw := cbg.NewCborWriter(w)
if _, err := cw.Write([]byte{161}); err != nil {
if _, err := w.Write([]byte{161}); err != nil {
return err
}
scratch := make([]byte, 9)
// t.Refs ([]api.SealedRef) (slice)
if len("Refs") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"Refs\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Refs"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Refs"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("Refs")); err != nil {
@@ -375,32 +361,27 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("Slice value in field t.Refs was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Refs))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Refs))); err != nil {
return err
}
for _, v := range t.Refs {
if err := v.MarshalCBOR(cw); err != nil {
if err := v.MarshalCBOR(w); err != nil {
return err
}
}
return nil
}
func (t *SealedRefs) UnmarshalCBOR(r io.Reader) (err error) {
func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
*t = SealedRefs{}
cr := cbg.NewCborReader(r)
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
maj, extra, err := cr.ReadHeader()
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
defer func() {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
}()
if maj != cbg.MajMap {
return fmt.Errorf("cbor input should be of type map")
}
@@ -415,7 +396,7 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) (err error) {
for i := uint64(0); i < n; i++ {
{
sval, err := cbg.ReadString(cr)
sval, err := cbg.ReadStringBuf(br, scratch)
if err != nil {
return err
}
@@ -427,7 +408,7 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) (err error) {
// t.Refs ([]api.SealedRef) (slice)
case "Refs":
maj, extra, err = cr.ReadHeader()
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
@@ -447,7 +428,7 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) (err error) {
for i := 0; i < int(extra); i++ {
var v SealedRef
if err := v.UnmarshalCBOR(cr); err != nil {
if err := v.UnmarshalCBOR(br); err != nil {
return err
}
@@ -467,19 +448,18 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
cw := cbg.NewCborWriter(w)
if _, err := cw.Write([]byte{162}); err != nil {
if _, err := w.Write([]byte{162}); err != nil {
return err
}
scratch := make([]byte, 9)
// t.Value (abi.SealRandomness) (slice)
if len("Value") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"Value\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Value"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("Value")); err != nil {
@@ -490,11 +470,11 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("Byte array in field t.Value was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Value))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Value))); err != nil {
return err
}
if _, err := cw.Write(t.Value[:]); err != nil {
if _, err := w.Write(t.Value[:]); err != nil {
return err
}
@@ -503,7 +483,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("Value in field \"Epoch\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Epoch"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("Epoch")); err != nil {
@@ -511,32 +491,27 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
}
if t.Epoch >= 0 {
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Epoch)); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Epoch)); err != nil {
return err
}
} else {
if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Epoch-1)); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Epoch-1)); err != nil {
return err
}
}
return nil
}
func (t *SealTicket) UnmarshalCBOR(r io.Reader) (err error) {
func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
*t = SealTicket{}
cr := cbg.NewCborReader(r)
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
maj, extra, err := cr.ReadHeader()
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
defer func() {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
}()
if maj != cbg.MajMap {
return fmt.Errorf("cbor input should be of type map")
}
@@ -551,7 +526,7 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) (err error) {
for i := uint64(0); i < n; i++ {
{
sval, err := cbg.ReadString(cr)
sval, err := cbg.ReadStringBuf(br, scratch)
if err != nil {
return err
}
@@ -563,7 +538,7 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) (err error) {
// t.Value (abi.SealRandomness) (slice)
case "Value":
maj, extra, err = cr.ReadHeader()
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
@@ -579,13 +554,13 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) (err error) {
t.Value = make([]uint8, extra)
}
if _, err := io.ReadFull(cr, t.Value[:]); err != nil {
if _, err := io.ReadFull(br, t.Value[:]); err != nil {
return err
}
// t.Epoch (abi.ChainEpoch) (int64)
case "Epoch":
{
maj, extra, err := cr.ReadHeader()
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
var extraI int64
if err != nil {
return err
@@ -622,19 +597,18 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
cw := cbg.NewCborWriter(w)
if _, err := cw.Write([]byte{162}); err != nil {
if _, err := w.Write([]byte{162}); err != nil {
return err
}
scratch := make([]byte, 9)
// t.Value (abi.InteractiveSealRandomness) (slice)
if len("Value") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"Value\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Value"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("Value")); err != nil {
@@ -645,11 +619,11 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("Byte array in field t.Value was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Value))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Value))); err != nil {
return err
}
if _, err := cw.Write(t.Value[:]); err != nil {
if _, err := w.Write(t.Value[:]); err != nil {
return err
}
@@ -658,7 +632,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("Value in field \"Epoch\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Epoch"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("Epoch")); err != nil {
@@ -666,32 +640,27 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
}
if t.Epoch >= 0 {
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Epoch)); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Epoch)); err != nil {
return err
}
} else {
if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Epoch-1)); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Epoch-1)); err != nil {
return err
}
}
return nil
}
func (t *SealSeed) UnmarshalCBOR(r io.Reader) (err error) {
func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
*t = SealSeed{}
cr := cbg.NewCborReader(r)
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
maj, extra, err := cr.ReadHeader()
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
defer func() {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
}()
if maj != cbg.MajMap {
return fmt.Errorf("cbor input should be of type map")
}
@@ -706,7 +675,7 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) (err error) {
for i := uint64(0); i < n; i++ {
{
sval, err := cbg.ReadString(cr)
sval, err := cbg.ReadStringBuf(br, scratch)
if err != nil {
return err
}
@@ -718,7 +687,7 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) (err error) {
// t.Value (abi.InteractiveSealRandomness) (slice)
case "Value":
maj, extra, err = cr.ReadHeader()
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
@@ -734,13 +703,13 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) (err error) {
t.Value = make([]uint8, extra)
}
if _, err := io.ReadFull(cr, t.Value[:]); err != nil {
if _, err := io.ReadFull(br, t.Value[:]); err != nil {
return err
}
// t.Epoch (abi.ChainEpoch) (int64)
case "Epoch":
{
maj, extra, err := cr.ReadHeader()
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
var extraI int64
if err != nil {
return err
@@ -777,19 +746,18 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
cw := cbg.NewCborWriter(w)
if _, err := cw.Write([]byte{165}); err != nil {
if _, err := w.Write([]byte{165}); err != nil {
return err
}
scratch := make([]byte, 9)
// t.PublishCid (cid.Cid) (struct)
if len("PublishCid") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"PublishCid\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PublishCid"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PublishCid"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("PublishCid")); err != nil {
@@ -797,11 +765,11 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
}
if t.PublishCid == nil {
if _, err := cw.Write(cbg.CborNull); err != nil {
if _, err := w.Write(cbg.CborNull); err != nil {
return err
}
} else {
if err := cbg.WriteCid(cw, *t.PublishCid); err != nil {
if err := cbg.WriteCidBuf(scratch, w, *t.PublishCid); err != nil {
return xerrors.Errorf("failed to write cid field t.PublishCid: %w", err)
}
}
@@ -811,14 +779,14 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("Value in field \"DealID\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("DealID"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealID"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("DealID")); err != nil {
return err
}
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.DealID)); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.DealID)); err != nil {
return err
}
@@ -827,14 +795,14 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("Value in field \"DealProposal\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("DealProposal"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealProposal"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("DealProposal")); err != nil {
return err
}
if err := t.DealProposal.MarshalCBOR(cw); err != nil {
if err := t.DealProposal.MarshalCBOR(w); err != nil {
return err
}
@@ -843,14 +811,14 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("Value in field \"DealSchedule\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("DealSchedule"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealSchedule"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("DealSchedule")); err != nil {
return err
}
if err := t.DealSchedule.MarshalCBOR(cw); err != nil {
if err := t.DealSchedule.MarshalCBOR(w); err != nil {
return err
}
@@ -859,7 +827,7 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("Value in field \"KeepUnsealed\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("KeepUnsealed"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("KeepUnsealed"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("KeepUnsealed")); err != nil {
@@ -872,21 +840,16 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
return nil
}
func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) (err error) {
func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
*t = PieceDealInfo{}
cr := cbg.NewCborReader(r)
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
maj, extra, err := cr.ReadHeader()
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
defer func() {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
}()
if maj != cbg.MajMap {
return fmt.Errorf("cbor input should be of type map")
}
@@ -901,7 +864,7 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) (err error) {
for i := uint64(0); i < n; i++ {
{
sval, err := cbg.ReadString(cr)
sval, err := cbg.ReadStringBuf(br, scratch)
if err != nil {
return err
}
@@ -915,16 +878,16 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) (err error) {
{
b, err := cr.ReadByte()
b, err := br.ReadByte()
if err != nil {
return err
}
if b != cbg.CborNull[0] {
if err := cr.UnreadByte(); err != nil {
if err := br.UnreadByte(); err != nil {
return err
}
c, err := cbg.ReadCid(cr)
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.PublishCid: %w", err)
}
@@ -938,7 +901,7 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) (err error) {
{
maj, extra, err = cr.ReadHeader()
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
@@ -953,16 +916,16 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) (err error) {
{
b, err := cr.ReadByte()
b, err := br.ReadByte()
if err != nil {
return err
}
if b != cbg.CborNull[0] {
if err := cr.UnreadByte(); err != nil {
if err := br.UnreadByte(); err != nil {
return err
}
t.DealProposal = new(market.DealProposal)
if err := t.DealProposal.UnmarshalCBOR(cr); err != nil {
if err := t.DealProposal.UnmarshalCBOR(br); err != nil {
return xerrors.Errorf("unmarshaling t.DealProposal pointer: %w", err)
}
}
@@ -973,7 +936,7 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) (err error) {
{
if err := t.DealSchedule.UnmarshalCBOR(cr); err != nil {
if err := t.DealSchedule.UnmarshalCBOR(br); err != nil {
return xerrors.Errorf("unmarshaling t.DealSchedule: %w", err)
}
@@ -981,7 +944,7 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) (err error) {
// t.KeepUnsealed (bool) (bool)
case "KeepUnsealed":
maj, extra, err = cr.ReadHeader()
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
@@ -1010,19 +973,18 @@ func (t *DealSchedule) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
cw := cbg.NewCborWriter(w)
if _, err := cw.Write([]byte{162}); err != nil {
if _, err := w.Write([]byte{162}); err != nil {
return err
}
scratch := make([]byte, 9)
// t.StartEpoch (abi.ChainEpoch) (int64)
if len("StartEpoch") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"StartEpoch\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("StartEpoch"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("StartEpoch"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("StartEpoch")); err != nil {
@@ -1030,11 +992,11 @@ func (t *DealSchedule) MarshalCBOR(w io.Writer) error {
}
if t.StartEpoch >= 0 {
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.StartEpoch)); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.StartEpoch)); err != nil {
return err
}
} else {
if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.StartEpoch-1)); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.StartEpoch-1)); err != nil {
return err
}
}
@@ -1044,7 +1006,7 @@ func (t *DealSchedule) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("Value in field \"EndEpoch\" was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("EndEpoch"))); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("EndEpoch"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("EndEpoch")); err != nil {
@@ -1052,32 +1014,27 @@ func (t *DealSchedule) MarshalCBOR(w io.Writer) error {
}
if t.EndEpoch >= 0 {
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.EndEpoch)); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.EndEpoch)); err != nil {
return err
}
} else {
if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.EndEpoch-1)); err != nil {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.EndEpoch-1)); err != nil {
return err
}
}
return nil
}
func (t *DealSchedule) UnmarshalCBOR(r io.Reader) (err error) {
func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error {
*t = DealSchedule{}
cr := cbg.NewCborReader(r)
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
maj, extra, err := cr.ReadHeader()
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
defer func() {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
}()
if maj != cbg.MajMap {
return fmt.Errorf("cbor input should be of type map")
}
@@ -1092,7 +1049,7 @@ func (t *DealSchedule) UnmarshalCBOR(r io.Reader) (err error) {
for i := uint64(0); i < n; i++ {
{
sval, err := cbg.ReadString(cr)
sval, err := cbg.ReadStringBuf(br, scratch)
if err != nil {
return err
}
@@ -1104,7 +1061,7 @@ func (t *DealSchedule) UnmarshalCBOR(r io.Reader) (err error) {
// t.StartEpoch (abi.ChainEpoch) (int64)
case "StartEpoch":
{
maj, extra, err := cr.ReadHeader()
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
var extraI int64
if err != nil {
return err
@@ -1130,7 +1087,7 @@ func (t *DealSchedule) UnmarshalCBOR(r io.Reader) (err error) {
// t.EndEpoch (abi.ChainEpoch) (int64)
case "EndEpoch":
{
maj, extra, err := cr.ReadHeader()
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
var extraI int64
if err != nil {
return err
+1
View File
@@ -8,6 +8,7 @@ import (
"os"
"github.com/filecoin-project/lotus/api/docgen"
docgen_openrpc "github.com/filecoin-project/lotus/api/docgen-openrpc"
)
+2 -3
View File
@@ -8,11 +8,10 @@ import (
"github.com/alecthomas/jsonschema"
go_openrpc_reflect "github.com/etclabscore/go-openrpc-reflect"
"github.com/ipfs/go-cid"
meta_schema "github.com/open-rpc/meta-schema"
"github.com/filecoin-project/lotus/api/docgen"
"github.com/filecoin-project/lotus/build"
"github.com/ipfs/go-cid"
meta_schema "github.com/open-rpc/meta-schema"
)
// schemaDictEntry represents a type association passed to the jsonschema reflector.
+21 -76
View File
@@ -1,7 +1,6 @@
package docgen
import (
"encoding/json"
"fmt"
"go/ast"
"go/parser"
@@ -12,11 +11,12 @@ import (
"time"
"unicode"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
"github.com/filecoin-project/go-multistore"
"github.com/google/uuid"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-graphsync"
textselector "github.com/ipld/go-ipld-selector-text-lite"
"github.com/ipfs/go-filestore"
"github.com/libp2p/go-libp2p-core/metrics"
"github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
@@ -24,12 +24,11 @@ import (
pubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/multiformats/go-multiaddr"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-fil-markets/filestore"
filestore2 "github.com/filecoin-project/go-fil-markets/filestore"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/go-state-types/exitcode"
@@ -39,11 +38,12 @@ import (
"github.com/filecoin-project/lotus/api/v0api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo/imports"
sealing "github.com/filecoin-project/lotus/storage/pipeline"
"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
var ExampleValues = map[reflect.Type]interface{}{
@@ -90,13 +90,8 @@ func init() {
addExample(pid)
addExample(&pid)
multistoreIDExample := multistore.StoreID(50)
storeIDExample := imports.ID(50)
textSelExample := textselector.Expression("Links/21/Hash/Links/42/Hash")
apiSelExample := api.Selector("Links/21/Hash/Links/42/Hash")
clientEvent := retrievalmarket.ClientEventDealAccepted
block := blocks.Block(&blocks.BasicBlock{})
ExampleValues[reflect.TypeOf(&block).Elem()] = block
addExample(bitfield.NewFromSet([]uint64{5}))
addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1)
@@ -115,6 +110,7 @@ func init() {
addExample(abi.UnpaddedPieceSize(1024))
addExample(abi.UnpaddedPieceSize(1024).Padded())
addExample(abi.DealID(5432))
addExample(filestore.StatusFileChanged)
addExample(abi.SectorNumber(9))
addExample(abi.SectorSize(32 * 1024 * 1024 * 1024))
addExample(api.MpoolChange(0))
@@ -124,17 +120,14 @@ func init() {
addExample(api.FullAPIVersion1)
addExample(api.PCHInbound)
addExample(time.Minute)
addExample(graphsync.NewRequestID())
addExample(datatransfer.TransferID(3))
addExample(datatransfer.Ongoing)
addExample(storeIDExample)
addExample(&storeIDExample)
addExample(clientEvent)
addExample(&clientEvent)
addExample(multistoreIDExample)
addExample(&multistoreIDExample)
addExample(retrievalmarket.ClientEventDealAccepted)
addExample(retrievalmarket.DealStatusNew)
addExample(&textSelExample)
addExample(&apiSelExample)
addExample(network.ReachabilityPublic)
addExample(build.NewestNetworkVersion)
addExample(map[string]int{"name": 42})
@@ -149,10 +142,6 @@ func init() {
addExample(map[string]api.MarketDeal{
"t026363": ExampleValue("init", reflect.TypeOf(api.MarketDeal{}), nil).(api.MarketDeal),
})
addExample(map[string]*api.MarketDeal{
"t026363": ExampleValue("init", reflect.TypeOf(&api.MarketDeal{}), nil).(*api.MarketDeal),
})
addExample(map[string]api.MarketBalance{
"t026363": ExampleValue("init", reflect.TypeOf(api.MarketBalance{}), nil).(api.MarketBalance),
})
@@ -190,12 +179,11 @@ func init() {
ExampleValues[reflect.TypeOf(struct{ A multiaddr.Multiaddr }{}).Field(0).Type] = maddr
// miner specific
addExample(filestore.Path(".lotusminer/fstmp123"))
addExample(filestore2.Path(".lotusminer/fstmp123"))
si := uint64(12)
addExample(&si)
addExample(retrievalmarket.DealID(5))
addExample(abi.ActorID(1000))
addExample(map[string]cid.Cid{})
addExample(map[string][]api.SealedRef{
"98000": {
api.SealedRef{
@@ -206,10 +194,10 @@ func init() {
},
})
addExample(api.SectorState(sealing.Proving))
addExample(storiface.ID("76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8"))
addExample(stores.ID("76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8"))
addExample(storiface.FTUnsealed)
addExample(storiface.PathSealing)
addExample(map[storiface.ID][]storiface.Decl{
addExample(map[stores.ID][]stores.Decl{
"76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8": {
{
SectorID: abi.SectorID{Miner: 1000, Number: 100},
@@ -217,7 +205,7 @@ func init() {
},
},
})
addExample(map[storiface.ID]string{
addExample(map[stores.ID]string{
"76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8": "/data/path",
})
addExample(map[uuid.UUID][]storiface.WorkerJob{
@@ -241,18 +229,16 @@ func init() {
Hostname: "host",
Resources: storiface.WorkerResources{
MemPhysical: 256 << 30,
MemUsed: 2 << 30,
MemSwap: 120 << 30,
MemSwapUsed: 2 << 30,
MemReserved: 2 << 30,
CPUs: 64,
GPUs: []string{"aGPU 1337"},
Resources: storiface.ResourceTable,
},
},
Enabled: true,
MemUsedMin: 0,
MemUsedMax: 0,
GpuUsed: 0,
GpuUsed: false,
CpuUse: 0,
},
})
@@ -260,18 +246,10 @@ func init() {
addExample(map[abi.SectorNumber]string{
123: "can't acquire read lock",
})
addExample(json.RawMessage(`"json raw message"`))
addExample(map[api.SectorState]int{
api.SectorState(sealing.Proving): 120,
})
addExample([]abi.SectorNumber{123, 124})
addExample([]storiface.SectorLock{
{
Sector: abi.SectorID{Number: 123, Miner: 1000},
Write: [storiface.FileTypes]uint{0, 0, 1},
Read: [storiface.FileTypes]uint{2, 3, 0},
},
})
// worker specific
addExample(storiface.AcquireMove)
@@ -306,35 +284,6 @@ func init() {
State: "ShardStateAvailable",
Error: "<error>",
})
addExample(storiface.ResourceTable)
addExample(network.ScopeStat{
Memory: 123,
NumStreamsInbound: 1,
NumStreamsOutbound: 2,
NumConnsInbound: 3,
NumConnsOutbound: 4,
NumFD: 5,
})
addExample(map[string]network.ScopeStat{
"abc": {
Memory: 123,
NumStreamsInbound: 1,
NumStreamsOutbound: 2,
NumConnsInbound: 3,
NumConnsOutbound: 4,
NumFD: 5,
}})
addExample(api.NetLimit{
Memory: 123,
StreamsInbound: 1,
StreamsOutbound: 2,
Streams: 3,
ConnsInbound: 3,
ConnsOutbound: 4,
Conns: 4,
FD: 5,
})
}
func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) {
@@ -358,10 +307,6 @@ func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []r
i = &api.WorkerStruct{}
t = reflect.TypeOf(new(struct{ api.Worker })).Elem()
permStruct = append(permStruct, reflect.TypeOf(api.WorkerStruct{}.Internal))
case "Gateway":
i = &api.GatewayStruct{}
t = reflect.TypeOf(new(struct{ api.Gateway })).Elem()
permStruct = append(permStruct, reflect.TypeOf(api.GatewayStruct{}.Internal))
default:
panic("unknown type")
}
@@ -389,7 +334,7 @@ func ExampleValue(method string, t, parent reflect.Type) interface{} {
switch t.Kind() {
case reflect.Slice:
out := reflect.New(t).Elem()
out = reflect.Append(out, reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
reflect.Append(out, reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
return out.Interface()
case reflect.Chan:
return ExampleValue(method, t.Elem(), nil)
+94 -332
View File
@@ -8,37 +8,34 @@ import (
context "context"
json "encoding/json"
reflect "reflect"
time "time"
gomock "github.com/golang/mock/gomock"
uuid "github.com/google/uuid"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
metrics "github.com/libp2p/go-libp2p-core/metrics"
network0 "github.com/libp2p/go-libp2p-core/network"
peer "github.com/libp2p/go-libp2p-core/peer"
protocol "github.com/libp2p/go-libp2p-core/protocol"
address "github.com/filecoin-project/go-address"
bitfield "github.com/filecoin-project/go-bitfield"
datatransfer "github.com/filecoin-project/go-data-transfer"
retrievalmarket "github.com/filecoin-project/go-fil-markets/retrievalmarket"
storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket"
auth "github.com/filecoin-project/go-jsonrpc/auth"
abi "github.com/filecoin-project/go-state-types/abi"
big "github.com/filecoin-project/go-state-types/big"
miner "github.com/filecoin-project/go-state-types/builtin/v8/miner"
paych "github.com/filecoin-project/go-state-types/builtin/v8/paych"
crypto "github.com/filecoin-project/go-state-types/crypto"
dline "github.com/filecoin-project/go-state-types/dline"
network "github.com/filecoin-project/go-state-types/network"
api "github.com/filecoin-project/lotus/api"
apitypes "github.com/filecoin-project/lotus/api/types"
miner0 "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
miner "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
types "github.com/filecoin-project/lotus/chain/types"
alerting "github.com/filecoin-project/lotus/journal/alerting"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
imports "github.com/filecoin-project/lotus/node/repo/imports"
miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
gomock "github.com/golang/mock/gomock"
uuid "github.com/google/uuid"
cid "github.com/ipfs/go-cid"
metrics "github.com/libp2p/go-libp2p-core/metrics"
network0 "github.com/libp2p/go-libp2p-core/network"
peer "github.com/libp2p/go-libp2p-core/peer"
protocol "github.com/libp2p/go-libp2p-core/protocol"
)
// MockFullNode is a mock of FullNode interface.
@@ -94,6 +91,21 @@ func (mr *MockFullNodeMockRecorder) AuthVerify(arg0, arg1 interface{}) *gomock.C
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AuthVerify", reflect.TypeOf((*MockFullNode)(nil).AuthVerify), arg0, arg1)
}
// BeaconGetEntry mocks base method.
func (m *MockFullNode) BeaconGetEntry(arg0 context.Context, arg1 abi.ChainEpoch) (*types.BeaconEntry, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "BeaconGetEntry", arg0, arg1)
ret0, _ := ret[0].(*types.BeaconEntry)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// BeaconGetEntry indicates an expected call of BeaconGetEntry.
func (mr *MockFullNodeMockRecorder) BeaconGetEntry(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BeaconGetEntry", reflect.TypeOf((*MockFullNode)(nil).BeaconGetEntry), arg0, arg1)
}
// ChainBlockstoreInfo mocks base method.
func (m *MockFullNode) ChainBlockstoreInfo(arg0 context.Context) (map[string]interface{}, error) {
m.ctrl.T.Helper()
@@ -287,6 +299,36 @@ func (mr *MockFullNodeMockRecorder) ChainGetPath(arg0, arg1, arg2 interface{}) *
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainGetPath", reflect.TypeOf((*MockFullNode)(nil).ChainGetPath), arg0, arg1, arg2)
}
// ChainGetRandomnessFromBeacon mocks base method.
func (m *MockFullNode) ChainGetRandomnessFromBeacon(arg0 context.Context, arg1 types.TipSetKey, arg2 crypto.DomainSeparationTag, arg3 abi.ChainEpoch, arg4 []byte) (abi.Randomness, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ChainGetRandomnessFromBeacon", arg0, arg1, arg2, arg3, arg4)
ret0, _ := ret[0].(abi.Randomness)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ChainGetRandomnessFromBeacon indicates an expected call of ChainGetRandomnessFromBeacon.
func (mr *MockFullNodeMockRecorder) ChainGetRandomnessFromBeacon(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainGetRandomnessFromBeacon", reflect.TypeOf((*MockFullNode)(nil).ChainGetRandomnessFromBeacon), arg0, arg1, arg2, arg3, arg4)
}
// ChainGetRandomnessFromTickets mocks base method.
func (m *MockFullNode) ChainGetRandomnessFromTickets(arg0 context.Context, arg1 types.TipSetKey, arg2 crypto.DomainSeparationTag, arg3 abi.ChainEpoch, arg4 []byte) (abi.Randomness, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ChainGetRandomnessFromTickets", arg0, arg1, arg2, arg3, arg4)
ret0, _ := ret[0].(abi.Randomness)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ChainGetRandomnessFromTickets indicates an expected call of ChainGetRandomnessFromTickets.
func (mr *MockFullNodeMockRecorder) ChainGetRandomnessFromTickets(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainGetRandomnessFromTickets", reflect.TypeOf((*MockFullNode)(nil).ChainGetRandomnessFromTickets), arg0, arg1, arg2, arg3, arg4)
}
// ChainGetTipSet mocks base method.
func (m *MockFullNode) ChainGetTipSet(arg0 context.Context, arg1 types.TipSetKey) (*types.TipSet, error) {
m.ctrl.T.Helper()
@@ -377,20 +419,6 @@ func (mr *MockFullNodeMockRecorder) ChainNotify(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainNotify", reflect.TypeOf((*MockFullNode)(nil).ChainNotify), arg0)
}
// ChainPutObj mocks base method.
func (m *MockFullNode) ChainPutObj(arg0 context.Context, arg1 blocks.Block) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ChainPutObj", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// ChainPutObj indicates an expected call of ChainPutObj.
func (mr *MockFullNodeMockRecorder) ChainPutObj(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainPutObj", reflect.TypeOf((*MockFullNode)(nil).ChainPutObj), arg0, arg1)
}
// ChainReadObj mocks base method.
func (m *MockFullNode) ChainReadObj(arg0 context.Context, arg1 cid.Cid) ([]byte, error) {
m.ctrl.T.Helper()
@@ -538,20 +566,6 @@ func (mr *MockFullNodeMockRecorder) ClientDealSize(arg0, arg1 interface{}) *gomo
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientDealSize", reflect.TypeOf((*MockFullNode)(nil).ClientDealSize), arg0, arg1)
}
// ClientExport mocks base method.
func (m *MockFullNode) ClientExport(arg0 context.Context, arg1 api.ExportRef, arg2 api.FileRef) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ClientExport", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
return ret0
}
// ClientExport indicates an expected call of ClientExport.
func (mr *MockFullNodeMockRecorder) ClientExport(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientExport", reflect.TypeOf((*MockFullNode)(nil).ClientExport), arg0, arg1, arg2)
}
// ClientFindData mocks base method.
func (m *MockFullNode) ClientFindData(arg0 context.Context, arg1 cid.Cid, arg2 *cid.Cid) ([]api.QueryOffer, error) {
m.ctrl.T.Helper()
@@ -747,10 +761,10 @@ func (mr *MockFullNodeMockRecorder) ClientMinerQueryOffer(arg0, arg1, arg2, arg3
}
// ClientQueryAsk mocks base method.
func (m *MockFullNode) ClientQueryAsk(arg0 context.Context, arg1 peer.ID, arg2 address.Address) (*api.StorageAsk, error) {
func (m *MockFullNode) ClientQueryAsk(arg0 context.Context, arg1 peer.ID, arg2 address.Address) (*storagemarket.StorageAsk, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ClientQueryAsk", arg0, arg1, arg2)
ret0, _ := ret[0].(*api.StorageAsk)
ret0, _ := ret[0].(*storagemarket.StorageAsk)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@@ -790,18 +804,17 @@ func (mr *MockFullNodeMockRecorder) ClientRestartDataTransfer(arg0, arg1, arg2,
}
// ClientRetrieve mocks base method.
func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 api.RetrievalOrder) (*api.RestrievalRes, error) {
func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ClientRetrieve", arg0, arg1)
ret0, _ := ret[0].(*api.RestrievalRes)
ret1, _ := ret[1].(error)
return ret0, ret1
ret := m.ctrl.Call(m, "ClientRetrieve", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
return ret0
}
// ClientRetrieve indicates an expected call of ClientRetrieve.
func (mr *MockFullNodeMockRecorder) ClientRetrieve(arg0, arg1 interface{}) *gomock.Call {
func (mr *MockFullNodeMockRecorder) ClientRetrieve(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieve", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieve), arg0, arg1)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieve", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieve), arg0, arg1, arg2)
}
// ClientRetrieveTryRestartInsufficientFunds mocks base method.
@@ -818,18 +831,19 @@ func (mr *MockFullNodeMockRecorder) ClientRetrieveTryRestartInsufficientFunds(ar
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieveTryRestartInsufficientFunds", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieveTryRestartInsufficientFunds), arg0, arg1)
}
// ClientRetrieveWait mocks base method.
func (m *MockFullNode) ClientRetrieveWait(arg0 context.Context, arg1 retrievalmarket.DealID) error {
// ClientRetrieveWithEvents mocks base method.
func (m *MockFullNode) ClientRetrieveWithEvents(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ClientRetrieveWait", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
ret := m.ctrl.Call(m, "ClientRetrieveWithEvents", arg0, arg1, arg2)
ret0, _ := ret[0].(<-chan marketevents.RetrievalEvent)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ClientRetrieveWait indicates an expected call of ClientRetrieveWait.
func (mr *MockFullNodeMockRecorder) ClientRetrieveWait(arg0, arg1 interface{}) *gomock.Call {
// ClientRetrieveWithEvents indicates an expected call of ClientRetrieveWithEvents.
func (mr *MockFullNodeMockRecorder) ClientRetrieveWithEvents(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieveWait", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieveWait), arg0, arg1)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieveWithEvents", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieveWithEvents), arg0, arg1, arg2)
}
// ClientStartDeal mocks base method.
@@ -981,21 +995,6 @@ func (mr *MockFullNodeMockRecorder) ID(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ID", reflect.TypeOf((*MockFullNode)(nil).ID), arg0)
}
// LogAlerts mocks base method.
func (m *MockFullNode) LogAlerts(arg0 context.Context) ([]alerting.Alert, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "LogAlerts", arg0)
ret0, _ := ret[0].([]alerting.Alert)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// LogAlerts indicates an expected call of LogAlerts.
func (mr *MockFullNodeMockRecorder) LogAlerts(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogAlerts", reflect.TypeOf((*MockFullNode)(nil).LogAlerts), arg0)
}
// LogList mocks base method.
func (m *MockFullNode) LogList(arg0 context.Context) ([]string, error) {
m.ctrl.T.Helper()
@@ -1443,33 +1442,18 @@ func (mr *MockFullNodeMockRecorder) MsigApproveTxnHash(arg0, arg1, arg2, arg3, a
}
// MsigCancel mocks base method.
func (m *MockFullNode) MsigCancel(arg0 context.Context, arg1 address.Address, arg2 uint64, arg3 address.Address) (*api.MessagePrototype, error) {
func (m *MockFullNode) MsigCancel(arg0 context.Context, arg1 address.Address, arg2 uint64, arg3 address.Address, arg4 big.Int, arg5 address.Address, arg6 uint64, arg7 []byte) (*api.MessagePrototype, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "MsigCancel", arg0, arg1, arg2, arg3)
ret := m.ctrl.Call(m, "MsigCancel", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7)
ret0, _ := ret[0].(*api.MessagePrototype)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// MsigCancel indicates an expected call of MsigCancel.
func (mr *MockFullNodeMockRecorder) MsigCancel(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
func (mr *MockFullNodeMockRecorder) MsigCancel(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MsigCancel", reflect.TypeOf((*MockFullNode)(nil).MsigCancel), arg0, arg1, arg2, arg3)
}
// MsigCancelTxnHash mocks base method.
func (m *MockFullNode) MsigCancelTxnHash(arg0 context.Context, arg1 address.Address, arg2 uint64, arg3 address.Address, arg4 big.Int, arg5 address.Address, arg6 uint64, arg7 []byte) (*api.MessagePrototype, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "MsigCancelTxnHash", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7)
ret0, _ := ret[0].(*api.MessagePrototype)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// MsigCancelTxnHash indicates an expected call of MsigCancelTxnHash.
func (mr *MockFullNodeMockRecorder) MsigCancelTxnHash(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MsigCancelTxnHash", reflect.TypeOf((*MockFullNode)(nil).MsigCancelTxnHash), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MsigCancel", reflect.TypeOf((*MockFullNode)(nil).MsigCancel), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7)
}
// MsigCreate mocks base method.
@@ -1813,21 +1797,6 @@ func (mr *MockFullNodeMockRecorder) NetFindPeer(arg0, arg1 interface{}) *gomock.
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetFindPeer", reflect.TypeOf((*MockFullNode)(nil).NetFindPeer), arg0, arg1)
}
// NetLimit mocks base method.
func (m *MockFullNode) NetLimit(arg0 context.Context, arg1 string) (api.NetLimit, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetLimit", arg0, arg1)
ret0, _ := ret[0].(api.NetLimit)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// NetLimit indicates an expected call of NetLimit.
func (mr *MockFullNodeMockRecorder) NetLimit(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetLimit", reflect.TypeOf((*MockFullNode)(nil).NetLimit), arg0, arg1)
}
// NetPeerInfo mocks base method.
func (m *MockFullNode) NetPeerInfo(arg0 context.Context, arg1 peer.ID) (*api.ExtendedPeerInfo, error) {
m.ctrl.T.Helper()
@@ -1858,64 +1827,6 @@ func (mr *MockFullNodeMockRecorder) NetPeers(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPeers", reflect.TypeOf((*MockFullNode)(nil).NetPeers), arg0)
}
// NetPing mocks base method.
func (m *MockFullNode) NetPing(arg0 context.Context, arg1 peer.ID) (time.Duration, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetPing", arg0, arg1)
ret0, _ := ret[0].(time.Duration)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// NetPing indicates an expected call of NetPing.
func (mr *MockFullNodeMockRecorder) NetPing(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPing", reflect.TypeOf((*MockFullNode)(nil).NetPing), arg0, arg1)
}
// NetProtectAdd mocks base method.
func (m *MockFullNode) NetProtectAdd(arg0 context.Context, arg1 []peer.ID) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetProtectAdd", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// NetProtectAdd indicates an expected call of NetProtectAdd.
func (mr *MockFullNodeMockRecorder) NetProtectAdd(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectAdd", reflect.TypeOf((*MockFullNode)(nil).NetProtectAdd), arg0, arg1)
}
// NetProtectList mocks base method.
func (m *MockFullNode) NetProtectList(arg0 context.Context) ([]peer.ID, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetProtectList", arg0)
ret0, _ := ret[0].([]peer.ID)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// NetProtectList indicates an expected call of NetProtectList.
func (mr *MockFullNodeMockRecorder) NetProtectList(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectList", reflect.TypeOf((*MockFullNode)(nil).NetProtectList), arg0)
}
// NetProtectRemove mocks base method.
func (m *MockFullNode) NetProtectRemove(arg0 context.Context, arg1 []peer.ID) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetProtectRemove", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// NetProtectRemove indicates an expected call of NetProtectRemove.
func (mr *MockFullNodeMockRecorder) NetProtectRemove(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectRemove", reflect.TypeOf((*MockFullNode)(nil).NetProtectRemove), arg0, arg1)
}
// NetPubsubScores mocks base method.
func (m *MockFullNode) NetPubsubScores(arg0 context.Context) ([]api.PubsubScore, error) {
m.ctrl.T.Helper()
@@ -1931,35 +1842,6 @@ func (mr *MockFullNodeMockRecorder) NetPubsubScores(arg0 interface{}) *gomock.Ca
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPubsubScores", reflect.TypeOf((*MockFullNode)(nil).NetPubsubScores), arg0)
}
// NetSetLimit mocks base method.
func (m *MockFullNode) NetSetLimit(arg0 context.Context, arg1 string, arg2 api.NetLimit) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetSetLimit", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
return ret0
}
// NetSetLimit indicates an expected call of NetSetLimit.
func (mr *MockFullNodeMockRecorder) NetSetLimit(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetSetLimit", reflect.TypeOf((*MockFullNode)(nil).NetSetLimit), arg0, arg1, arg2)
}
// NetStat mocks base method.
func (m *MockFullNode) NetStat(arg0 context.Context, arg1 string) (api.NetStat, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetStat", arg0, arg1)
ret0, _ := ret[0].(api.NetStat)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// NetStat indicates an expected call of NetStat.
func (mr *MockFullNodeMockRecorder) NetStat(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetStat", reflect.TypeOf((*MockFullNode)(nil).NetStat), arg0, arg1)
}
// NodeStatus mocks base method.
func (m *MockFullNode) NodeStatus(arg0 context.Context, arg1 bool) (api.NodeStatus, error) {
m.ctrl.T.Helper()
@@ -2035,34 +1917,19 @@ func (mr *MockFullNodeMockRecorder) PaychCollect(arg0, arg1 interface{}) *gomock
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychCollect", reflect.TypeOf((*MockFullNode)(nil).PaychCollect), arg0, arg1)
}
// PaychFund mocks base method.
func (m *MockFullNode) PaychFund(arg0 context.Context, arg1, arg2 address.Address, arg3 big.Int) (*api.ChannelInfo, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "PaychFund", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(*api.ChannelInfo)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// PaychFund indicates an expected call of PaychFund.
func (mr *MockFullNodeMockRecorder) PaychFund(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychFund", reflect.TypeOf((*MockFullNode)(nil).PaychFund), arg0, arg1, arg2, arg3)
}
// PaychGet mocks base method.
func (m *MockFullNode) PaychGet(arg0 context.Context, arg1, arg2 address.Address, arg3 big.Int, arg4 api.PaychGetOpts) (*api.ChannelInfo, error) {
func (m *MockFullNode) PaychGet(arg0 context.Context, arg1, arg2 address.Address, arg3 big.Int) (*api.ChannelInfo, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "PaychGet", arg0, arg1, arg2, arg3, arg4)
ret := m.ctrl.Call(m, "PaychGet", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(*api.ChannelInfo)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// PaychGet indicates an expected call of PaychGet.
func (mr *MockFullNodeMockRecorder) PaychGet(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
func (mr *MockFullNodeMockRecorder) PaychGet(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychGet", reflect.TypeOf((*MockFullNode)(nil).PaychGet), arg0, arg1, arg2, arg3, arg4)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychGet", reflect.TypeOf((*MockFullNode)(nil).PaychGet), arg0, arg1, arg2, arg3)
}
// PaychGetWaitReady mocks base method.
@@ -2273,21 +2140,6 @@ func (mr *MockFullNodeMockRecorder) StateAccountKey(arg0, arg1, arg2 interface{}
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateAccountKey", reflect.TypeOf((*MockFullNode)(nil).StateAccountKey), arg0, arg1, arg2)
}
// StateActorCodeCIDs mocks base method.
func (m *MockFullNode) StateActorCodeCIDs(arg0 context.Context, arg1 network.Version) (map[string]cid.Cid, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateActorCodeCIDs", arg0, arg1)
ret0, _ := ret[0].(map[string]cid.Cid)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateActorCodeCIDs indicates an expected call of StateActorCodeCIDs.
func (mr *MockFullNodeMockRecorder) StateActorCodeCIDs(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateActorCodeCIDs", reflect.TypeOf((*MockFullNode)(nil).StateActorCodeCIDs), arg0, arg1)
}
// StateAllMinerFaults mocks base method.
func (m *MockFullNode) StateAllMinerFaults(arg0 context.Context, arg1 abi.ChainEpoch, arg2 types.TipSetKey) ([]*api.Fault, error) {
m.ctrl.T.Helper()
@@ -2363,21 +2215,6 @@ func (mr *MockFullNodeMockRecorder) StateCompute(arg0, arg1, arg2, arg3 interfac
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateCompute", reflect.TypeOf((*MockFullNode)(nil).StateCompute), arg0, arg1, arg2, arg3)
}
// StateComputeDataCID mocks base method.
func (m *MockFullNode) StateComputeDataCID(arg0 context.Context, arg1 address.Address, arg2 abi.RegisteredSealProof, arg3 []abi.DealID, arg4 types.TipSetKey) (cid.Cid, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateComputeDataCID", arg0, arg1, arg2, arg3, arg4)
ret0, _ := ret[0].(cid.Cid)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateComputeDataCID indicates an expected call of StateComputeDataCID.
func (mr *MockFullNodeMockRecorder) StateComputeDataCID(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateComputeDataCID", reflect.TypeOf((*MockFullNode)(nil).StateComputeDataCID), arg0, arg1, arg2, arg3, arg4)
}
// StateDealProviderCollateralBounds mocks base method.
func (m *MockFullNode) StateDealProviderCollateralBounds(arg0 context.Context, arg1 abi.PaddedPieceSize, arg2 bool, arg3 types.TipSetKey) (api.DealCollateralBounds, error) {
m.ctrl.T.Helper()
@@ -2438,66 +2275,6 @@ func (mr *MockFullNodeMockRecorder) StateGetActor(arg0, arg1, arg2 interface{})
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetActor", reflect.TypeOf((*MockFullNode)(nil).StateGetActor), arg0, arg1, arg2)
}
// StateGetBeaconEntry mocks base method.
func (m *MockFullNode) StateGetBeaconEntry(arg0 context.Context, arg1 abi.ChainEpoch) (*types.BeaconEntry, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateGetBeaconEntry", arg0, arg1)
ret0, _ := ret[0].(*types.BeaconEntry)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateGetBeaconEntry indicates an expected call of StateGetBeaconEntry.
func (mr *MockFullNodeMockRecorder) StateGetBeaconEntry(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetBeaconEntry", reflect.TypeOf((*MockFullNode)(nil).StateGetBeaconEntry), arg0, arg1)
}
// StateGetNetworkParams mocks base method.
func (m *MockFullNode) StateGetNetworkParams(arg0 context.Context) (*api.NetworkParams, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateGetNetworkParams", arg0)
ret0, _ := ret[0].(*api.NetworkParams)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateGetNetworkParams indicates an expected call of StateGetNetworkParams.
func (mr *MockFullNodeMockRecorder) StateGetNetworkParams(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetNetworkParams", reflect.TypeOf((*MockFullNode)(nil).StateGetNetworkParams), arg0)
}
// StateGetRandomnessFromBeacon mocks base method.
func (m *MockFullNode) StateGetRandomnessFromBeacon(arg0 context.Context, arg1 crypto.DomainSeparationTag, arg2 abi.ChainEpoch, arg3 []byte, arg4 types.TipSetKey) (abi.Randomness, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateGetRandomnessFromBeacon", arg0, arg1, arg2, arg3, arg4)
ret0, _ := ret[0].(abi.Randomness)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateGetRandomnessFromBeacon indicates an expected call of StateGetRandomnessFromBeacon.
func (mr *MockFullNodeMockRecorder) StateGetRandomnessFromBeacon(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetRandomnessFromBeacon", reflect.TypeOf((*MockFullNode)(nil).StateGetRandomnessFromBeacon), arg0, arg1, arg2, arg3, arg4)
}
// StateGetRandomnessFromTickets mocks base method.
func (m *MockFullNode) StateGetRandomnessFromTickets(arg0 context.Context, arg1 crypto.DomainSeparationTag, arg2 abi.ChainEpoch, arg3 []byte, arg4 types.TipSetKey) (abi.Randomness, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateGetRandomnessFromTickets", arg0, arg1, arg2, arg3, arg4)
ret0, _ := ret[0].(abi.Randomness)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateGetRandomnessFromTickets indicates an expected call of StateGetRandomnessFromTickets.
func (mr *MockFullNodeMockRecorder) StateGetRandomnessFromTickets(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetRandomnessFromTickets", reflect.TypeOf((*MockFullNode)(nil).StateGetRandomnessFromTickets), arg0, arg1, arg2, arg3, arg4)
}
// StateListActors mocks base method.
func (m *MockFullNode) StateListActors(arg0 context.Context, arg1 types.TipSetKey) ([]address.Address, error) {
m.ctrl.T.Helper()
@@ -2558,21 +2335,6 @@ func (mr *MockFullNodeMockRecorder) StateLookupID(arg0, arg1, arg2 interface{})
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateLookupID", reflect.TypeOf((*MockFullNode)(nil).StateLookupID), arg0, arg1, arg2)
}
// StateLookupRobustAddress mocks base method.
func (m *MockFullNode) StateLookupRobustAddress(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (address.Address, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateLookupRobustAddress", arg0, arg1, arg2)
ret0, _ := ret[0].(address.Address)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateLookupRobustAddress indicates an expected call of StateLookupRobustAddress.
func (mr *MockFullNodeMockRecorder) StateLookupRobustAddress(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateLookupRobustAddress", reflect.TypeOf((*MockFullNode)(nil).StateLookupRobustAddress), arg0, arg1, arg2)
}
// StateMarketBalance mocks base method.
func (m *MockFullNode) StateMarketBalance(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (api.MarketBalance, error) {
m.ctrl.T.Helper()
@@ -2589,10 +2351,10 @@ func (mr *MockFullNodeMockRecorder) StateMarketBalance(arg0, arg1, arg2 interfac
}
// StateMarketDeals mocks base method.
func (m *MockFullNode) StateMarketDeals(arg0 context.Context, arg1 types.TipSetKey) (map[string]*api.MarketDeal, error) {
func (m *MockFullNode) StateMarketDeals(arg0 context.Context, arg1 types.TipSetKey) (map[string]api.MarketDeal, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateMarketDeals", arg0, arg1)
ret0, _ := ret[0].(map[string]*api.MarketDeal)
ret0, _ := ret[0].(map[string]api.MarketDeal)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@@ -2694,10 +2456,10 @@ func (mr *MockFullNodeMockRecorder) StateMinerFaults(arg0, arg1, arg2 interface{
}
// StateMinerInfo mocks base method.
func (m *MockFullNode) StateMinerInfo(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (api.MinerInfo, error) {
func (m *MockFullNode) StateMinerInfo(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (miner.MinerInfo, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateMinerInfo", arg0, arg1, arg2)
ret0, _ := ret[0].(api.MinerInfo)
ret0, _ := ret[0].(miner.MinerInfo)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@@ -2709,7 +2471,7 @@ func (mr *MockFullNodeMockRecorder) StateMinerInfo(arg0, arg1, arg2 interface{})
}
// StateMinerInitialPledgeCollateral mocks base method.
func (m *MockFullNode) StateMinerInitialPledgeCollateral(arg0 context.Context, arg1 address.Address, arg2 miner.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
func (m *MockFullNode) StateMinerInitialPledgeCollateral(arg0 context.Context, arg1 address.Address, arg2 miner0.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateMinerInitialPledgeCollateral", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(big.Int)
@@ -2754,7 +2516,7 @@ func (mr *MockFullNodeMockRecorder) StateMinerPower(arg0, arg1, arg2 interface{}
}
// StateMinerPreCommitDepositForPower mocks base method.
func (m *MockFullNode) StateMinerPreCommitDepositForPower(arg0 context.Context, arg1 address.Address, arg2 miner.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
func (m *MockFullNode) StateMinerPreCommitDepositForPower(arg0 context.Context, arg1 address.Address, arg2 miner0.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateMinerPreCommitDepositForPower", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(big.Int)
@@ -2919,10 +2681,10 @@ func (mr *MockFullNodeMockRecorder) StateSearchMsg(arg0, arg1, arg2, arg3, arg4
}
// StateSectorExpiration mocks base method.
func (m *MockFullNode) StateSectorExpiration(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner0.SectorExpiration, error) {
func (m *MockFullNode) StateSectorExpiration(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner.SectorExpiration, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateSectorExpiration", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(*miner0.SectorExpiration)
ret0, _ := ret[0].(*miner.SectorExpiration)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@@ -2949,10 +2711,10 @@ func (mr *MockFullNodeMockRecorder) StateSectorGetInfo(arg0, arg1, arg2, arg3 in
}
// StateSectorPartition mocks base method.
func (m *MockFullNode) StateSectorPartition(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner0.SectorLocation, error) {
func (m *MockFullNode) StateSectorPartition(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner.SectorLocation, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateSectorPartition", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(*miner0.SectorLocation)
ret0, _ := ret[0].(*miner.SectorLocation)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@@ -2964,10 +2726,10 @@ func (mr *MockFullNodeMockRecorder) StateSectorPartition(arg0, arg1, arg2, arg3
}
// StateSectorPreCommitInfo mocks base method.
func (m *MockFullNode) StateSectorPreCommitInfo(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner.SectorPreCommitOnChainInfo, error) {
func (m *MockFullNode) StateSectorPreCommitInfo(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateSectorPreCommitInfo", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(*miner.SectorPreCommitOnChainInfo)
ret0, _ := ret[0].(miner.SectorPreCommitOnChainInfo)
ret1, _ := ret[1].(error)
return ret0, ret1
}
+219 -873
View File
File diff suppressed because it is too large. Load diff
-3
View File
@@ -1,4 +1,3 @@
//stm: #unit
package api
import (
@@ -30,7 +29,6 @@ type StrC struct {
}
func TestGetInternalStructs(t *testing.T) {
//stm: @OTHER_IMPLEMENTATION_API_STRUCTS_001
var proxy StrA
sts := GetInternalStructs(&proxy)
@@ -46,7 +44,6 @@ func TestGetInternalStructs(t *testing.T) {
}
func TestNestedInternalStructs(t *testing.T) {
//stm: @OTHER_IMPLEMENTATION_API_STRUCTS_001
var proxy StrC
// check that only the top-level internal struct gets picked up
+8 -145
View File
@@ -5,22 +5,20 @@ import (
"fmt"
"time"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/lotus/chain/types"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-state-types/abi"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-graphsync"
"github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
pubsub "github.com/libp2p/go-libp2p-pubsub"
ma "github.com/multiformats/go-multiaddr"
"github.com/filecoin-project/go-address"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
// TODO: check if this exists anywhere else
type MultiaddrSlice []ma.Multiaddr
func (m *MultiaddrSlice) UnmarshalJSON(raw []byte) (err error) {
@@ -56,30 +54,6 @@ type MessageSendSpec struct {
MaxFee abi.TokenAmount
}
// GraphSyncDataTransfer provides diagnostics on a data transfer happening over graphsync
type GraphSyncDataTransfer struct {
// GraphSync request id for this transfer
RequestID *graphsync.RequestID
// Graphsync state for this transfer
RequestState string
// If a channel ID is present, indicates whether this is the current graphsync request for this channel
// (could have changed in a restart)
IsCurrentChannelRequest bool
// Data transfer channel ID for this transfer
ChannelID *datatransfer.ChannelID
// Data transfer state for this transfer
ChannelState *DataTransferChannel
// Diagnostic information about this request -- and unexpected inconsistencies in
// request state
Diagnostics []string
}
// TransferDiagnostics give current information about transfers going over graphsync that may be helpful for debugging
type TransferDiagnostics struct {
ReceivingTransfers []*GraphSyncDataTransfer
SendingTransfers []*GraphSyncDataTransfer
}
type DataTransferChannel struct {
TransferID datatransfer.TransferID
Status datatransfer.Status
@@ -125,28 +99,6 @@ func NewDataTransferChannel(hostID peer.ID, channelState datatransfer.ChannelSta
return channel
}
type NetStat struct {
System *network.ScopeStat `json:",omitempty"`
Transient *network.ScopeStat `json:",omitempty"`
Services map[string]network.ScopeStat `json:",omitempty"`
Protocols map[string]network.ScopeStat `json:",omitempty"`
Peers map[string]network.ScopeStat `json:",omitempty"`
}
type NetLimit struct {
Dynamic bool `json:",omitempty"`
// set if Dynamic is false
Memory int64 `json:",omitempty"`
// set if Dynamic is true
MemoryFraction float64 `json:",omitempty"`
MinMemory int64 `json:",omitempty"`
MaxMemory int64 `json:",omitempty"`
Streams, StreamsInbound, StreamsOutbound int
Conns, ConnsInbound, ConnsOutbound int
FD int
}
type NetBlockList struct {
Peers []peer.ID
IPAddrs []string
@@ -242,93 +194,4 @@ type RetrievalInfo struct {
TransferChannelID *datatransfer.ChannelID
DataTransfer *DataTransferChannel
// optional event if part of ClientGetRetrievalUpdates
Event *retrievalmarket.ClientEvent
}
type RestrievalRes struct {
DealID retrievalmarket.DealID
}
// Selector specifies ipld selector string
// - if the string starts with '{', it's interpreted as json selector string
// see https://ipld.io/specs/selectors/ and https://ipld.io/specs/selectors/fixtures/selector-fixtures-1/
// - otherwise the string is interpreted as ipld-selector-text-lite (simple ipld path)
// see https://github.com/ipld/go-ipld-selector-text-lite
type Selector string
type DagSpec struct {
// DataSelector matches data to be retrieved
// - when using textselector, the path specifies subtree
// - the matched graph must have a single root
DataSelector *Selector
// ExportMerkleProof is applicable only when exporting to a CAR file via a path textselector
// When true, in addition to the selection target, the resulting CAR will contain every block along the
// path back to, and including the original root
// When false the resulting CAR contains only the blocks of the target subdag
ExportMerkleProof bool
}
type ExportRef struct {
Root cid.Cid
// DAGs array specifies a list of DAGs to export
// - If exporting into unixfs files, only one DAG is supported, DataSelector is only used to find the targeted root node
// - If exporting into a car file
// - When exactly one text-path DataSelector is specified exports the subgraph and its full merkle-path from the original root
// - Otherwise ( multiple paths and/or JSON selector specs) determines each individual subroot and exports the subtrees as a multi-root car
// - When not specified defaults to a single DAG:
// - Data - the entire DAG: `{"R":{"l":{"none":{}},":>":{"a":{">":{"@":{}}}}}}`
DAGs []DagSpec
FromLocalCAR string // if specified, get data from a local CARv2 file.
DealID retrievalmarket.DealID
}
type MinerInfo struct {
Owner address.Address // Must be an ID-address.
Worker address.Address // Must be an ID-address.
NewWorker address.Address // Must be an ID-address.
ControlAddresses []address.Address // Must be an ID-addresses.
WorkerChangeEpoch abi.ChainEpoch
PeerId *peer.ID
Multiaddrs []abi.Multiaddrs
WindowPoStProofType abi.RegisteredPoStProof
SectorSize abi.SectorSize
WindowPoStPartitionSectors uint64
ConsensusFaultElapsed abi.ChainEpoch
}
type NetworkParams struct {
NetworkName dtypes.NetworkName
BlockDelaySecs uint64
ConsensusMinerMinPower abi.StoragePower
SupportedProofTypes []abi.RegisteredSealProof
PreCommitChallengeDelay abi.ChainEpoch
ForkUpgradeParams ForkUpgradeParams
}
type ForkUpgradeParams struct {
UpgradeSmokeHeight abi.ChainEpoch
UpgradeBreezeHeight abi.ChainEpoch
UpgradeIgnitionHeight abi.ChainEpoch
UpgradeLiftoffHeight abi.ChainEpoch
UpgradeAssemblyHeight abi.ChainEpoch
UpgradeRefuelHeight abi.ChainEpoch
UpgradeTapeHeight abi.ChainEpoch
UpgradeKumquatHeight abi.ChainEpoch
UpgradePriceListOopsHeight abi.ChainEpoch
BreezeGasTampingDuration abi.ChainEpoch
UpgradeCalicoHeight abi.ChainEpoch
UpgradePersianHeight abi.ChainEpoch
UpgradeOrangeHeight abi.ChainEpoch
UpgradeClausHeight abi.ChainEpoch
UpgradeTrustHeight abi.ChainEpoch
UpgradeNorwegianHeight abi.ChainEpoch
UpgradeTurboHeight abi.ChainEpoch
UpgradeHyperdriveHeight abi.ChainEpoch
UpgradeChocolateHeight abi.ChainEpoch
UpgradeOhSnapHeight abi.ChainEpoch
}
+10 -62
View File
@@ -3,26 +3,21 @@ package v0api
import (
"context"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
textselector "github.com/ipld/go-ipld-selector-text-lite"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/builtin/v8/miner"
"github.com/filecoin-project/go-state-types/builtin/v8/paych"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/go-state-types/dline"
abinetwork "github.com/filecoin-project/go-state-types/network"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/lotus/api"
apitypes "github.com/filecoin-project/lotus/api/types"
lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
"github.com/filecoin-project/lotus/chain/types"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
"github.com/filecoin-project/lotus/node/modules/dtypes"
@@ -113,9 +108,6 @@ type FullNode interface {
// ChainDeleteObj deletes node referenced by the given CID
ChainDeleteObj(context.Context, cid.Cid) error //perm:admin
// ChainPutObj puts and object into the blockstore
ChainPutObj(context.Context, blocks.Block) error
// ChainHasObj checks if a given CID exists in the chain blockstore.
ChainHasObj(context.Context, cid.Cid) (bool, error) //perm:read
@@ -333,10 +325,10 @@ type FullNode interface {
// ClientMinerQueryOffer returns a QueryOffer for the specific miner and file.
ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error) //perm:read
// ClientRetrieve initiates the retrieval of a file, as specified in the order.
ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *api.FileRef) error //perm:admin
ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error //perm:admin
// ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel
// of status updates.
ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin
ClientRetrieveWithEvents(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin
// ClientQueryAsk returns a signed StorageAsk from the specified miner.
// ClientListRetrievals returns information about retrievals made by the local client
ClientListRetrievals(ctx context.Context) ([]api.RetrievalInfo, error) //perm:write
@@ -423,7 +415,7 @@ type FullNode interface {
// StateMinerPower returns the power of the indicated miner
StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) //perm:read
// StateMinerInfo returns info about the indicated miner
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) //perm:read
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) //perm:read
// StateMinerDeadlines returns all the proving deadlines for the given miner
StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]api.Deadline, error) //perm:read
// StateMinerPartitions returns all partitions in the specified deadline
@@ -449,9 +441,9 @@ type FullNode interface {
// expiration epoch
StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) //perm:read
// StateSectorExpiration returns epoch at which given sector will expire
StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*lminer.SectorExpiration, error) //perm:read
StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error) //perm:read
// StateSectorPartition finds deadline/partition with the specified sector
StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*lminer.SectorLocation, error) //perm:read
StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) //perm:read
// StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed
//
// NOTE: If a replacing message is found on chain, this method will return
@@ -528,7 +520,7 @@ type FullNode interface {
// StateMarketParticipants returns the Escrow and Locked balances of every participant in the Storage Market
StateMarketParticipants(context.Context, types.TipSetKey) (map[string]api.MarketBalance, error) //perm:read
// StateMarketDeals returns information about every deal in the Storage Market
StateMarketDeals(context.Context, types.TipSetKey) (map[string]*api.MarketDeal, error) //perm:read
StateMarketDeals(context.Context, types.TipSetKey) (map[string]api.MarketDeal, error) //perm:read
// StateMarketStorageDeal returns information about the indicated deal
StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error) //perm:read
// StateLookupID retrieves the ID address of the given address
@@ -605,16 +597,6 @@ type FullNode interface {
StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error) //perm:read
// StateNetworkVersion returns the network version at the given tipset
StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) //perm:read
// StateActorCodeCIDs returns the CIDs of all the builtin actors for the given network version
StateActorCodeCIDs(context.Context, abinetwork.Version) (map[string]cid.Cid, error) //perm:read
// StateGetRandomnessFromTickets is used to sample the chain for randomness.
StateGetRandomnessFromTickets(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
// StateGetRandomnessFromBeacon is used to sample the beacon for randomness.
StateGetRandomnessFromBeacon(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
// StateGetNetworkParams return current network params
StateGetNetworkParams(ctx context.Context) (*api.NetworkParams, error) //perm:read
// MethodGroup: Msig
// The Msig methods are used to interact with multisig wallets on the
@@ -727,37 +709,3 @@ type FullNode interface {
// the path specified when calling CreateBackup is within the base path
CreateBackup(ctx context.Context, fpath string) error //perm:admin
}
func OfferOrder(o api.QueryOffer, client address.Address) RetrievalOrder {
return RetrievalOrder{
Root: o.Root,
Piece: o.Piece,
Size: o.Size,
Total: o.MinPrice,
UnsealPrice: o.UnsealPrice,
PaymentInterval: o.PaymentInterval,
PaymentIntervalIncrease: o.PaymentIntervalIncrease,
Client: client,
Miner: o.Miner,
MinerPeer: &o.MinerPeer,
}
}
type RetrievalOrder struct {
// TODO: make this less unixfs specific
Root cid.Cid
Piece *cid.Cid
DatamodelPathSelector *textselector.Expression
Size uint64
FromLocalCAR string // if specified, get data from a local CARv2 file.
// TODO: support offset
Total types.BigInt
UnsealPrice types.BigInt
PaymentInterval uint64
PaymentIntervalIncrease uint64
Client address.Address
Miner address.Address
MinerPeer *retrievalmarket.RetrievalPeer
}
+4 -6
View File
@@ -3,16 +3,15 @@ package v0api
import (
"context"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/builtin/v8/miner"
"github.com/filecoin-project/go-state-types/dline"
abinetwork "github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/types"
)
@@ -35,7 +34,6 @@ import (
type Gateway interface {
ChainHasObj(context.Context, cid.Cid) (bool, error)
ChainPutObj(context.Context, blocks.Block) error
ChainHead(ctx context.Context) (*types.TipSet, error)
ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error)
ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
@@ -56,10 +54,10 @@ type Gateway interface {
StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error)
StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error)
StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (api.MinerInfo, error)
StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error)
StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)
StateNetworkVersion(context.Context, types.TipSetKey) (abinetwork.Version, error)
StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error)
StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error)
StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
-1
View File
@@ -2,7 +2,6 @@ package v0api
import (
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/lotus/api"
)
+38 -120
View File
@@ -5,30 +5,26 @@ package v0api
import (
"context"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/builtin/v8/miner"
"github.com/filecoin-project/go-state-types/builtin/v8/paych"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/go-state-types/dline"
abinetwork "github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/lotus/api"
apitypes "github.com/filecoin-project/lotus/api/types"
lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
"github.com/filecoin-project/lotus/chain/types"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo/imports"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors"
)
var ErrNotSupported = xerrors.New("method not supported")
@@ -77,8 +73,6 @@ type FullNodeStruct struct {
ChainNotify func(p0 context.Context) (<-chan []*api.HeadChange, error) `perm:"read"`
ChainPutObj func(p0 context.Context, p1 blocks.Block) error ``
ChainReadObj func(p0 context.Context, p1 cid.Cid) ([]byte, error) `perm:"read"`
ChainSetHead func(p0 context.Context, p1 types.TipSetKey) error `perm:"admin"`
@@ -131,11 +125,11 @@ type FullNodeStruct struct {
ClientRestartDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"`
ClientRetrieve func(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) error `perm:"admin"`
ClientRetrieve func(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) error `perm:"admin"`
ClientRetrieveTryRestartInsufficientFunds func(p0 context.Context, p1 address.Address) error `perm:"write"`
ClientRetrieveWithEvents func(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) `perm:"admin"`
ClientRetrieveWithEvents func(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) `perm:"admin"`
ClientStartDeal func(p0 context.Context, p1 *api.StartDealParams) (*cid.Cid, error) `perm:"admin"`
@@ -257,8 +251,6 @@ type FullNodeStruct struct {
StateAccountKey func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) `perm:"read"`
StateActorCodeCIDs func(p0 context.Context, p1 abinetwork.Version) (map[string]cid.Cid, error) `perm:"read"`
StateAllMinerFaults func(p0 context.Context, p1 abi.ChainEpoch, p2 types.TipSetKey) ([]*api.Fault, error) `perm:"read"`
StateCall func(p0 context.Context, p1 *types.Message, p2 types.TipSetKey) (*api.InvocResult, error) `perm:"read"`
@@ -275,12 +267,6 @@ type FullNodeStruct struct {
StateGetActor func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*types.Actor, error) `perm:"read"`
StateGetNetworkParams func(p0 context.Context) (*api.NetworkParams, error) `perm:"read"`
StateGetRandomnessFromBeacon func(p0 context.Context, p1 crypto.DomainSeparationTag, p2 abi.ChainEpoch, p3 []byte, p4 types.TipSetKey) (abi.Randomness, error) `perm:"read"`
StateGetRandomnessFromTickets func(p0 context.Context, p1 crypto.DomainSeparationTag, p2 abi.ChainEpoch, p3 []byte, p4 types.TipSetKey) (abi.Randomness, error) `perm:"read"`
StateGetReceipt func(p0 context.Context, p1 cid.Cid, p2 types.TipSetKey) (*types.MessageReceipt, error) `perm:"read"`
StateListActors func(p0 context.Context, p1 types.TipSetKey) ([]address.Address, error) `perm:"read"`
@@ -293,7 +279,7 @@ type FullNodeStruct struct {
StateMarketBalance func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (api.MarketBalance, error) `perm:"read"`
StateMarketDeals func(p0 context.Context, p1 types.TipSetKey) (map[string]*api.MarketDeal, error) `perm:"read"`
StateMarketDeals func(p0 context.Context, p1 types.TipSetKey) (map[string]api.MarketDeal, error) `perm:"read"`
StateMarketParticipants func(p0 context.Context, p1 types.TipSetKey) (map[string]api.MarketBalance, error) `perm:"read"`
@@ -307,7 +293,7 @@ type FullNodeStruct struct {
StateMinerFaults func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (bitfield.BitField, error) `perm:"read"`
StateMinerInfo func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (api.MinerInfo, error) `perm:"read"`
StateMinerInfo func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (miner.MinerInfo, error) `perm:"read"`
StateMinerInitialPledgeCollateral func(p0 context.Context, p1 address.Address, p2 miner.SectorPreCommitInfo, p3 types.TipSetKey) (types.BigInt, error) `perm:"read"`
@@ -339,11 +325,11 @@ type FullNodeStruct struct {
StateSearchMsgLimited func(p0 context.Context, p1 cid.Cid, p2 abi.ChainEpoch) (*api.MsgLookup, error) `perm:"read"`
StateSectorExpiration func(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*lminer.SectorExpiration, error) `perm:"read"`
StateSectorExpiration func(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*miner.SectorExpiration, error) `perm:"read"`
StateSectorGetInfo func(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*miner.SectorOnChainInfo, error) `perm:"read"`
StateSectorPartition func(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*lminer.SectorLocation, error) `perm:"read"`
StateSectorPartition func(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*miner.SectorLocation, error) `perm:"read"`
StateSectorPreCommitInfo func(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) `perm:"read"`
@@ -427,8 +413,6 @@ type GatewayStruct struct {
ChainNotify func(p0 context.Context) (<-chan []*api.HeadChange, error) ``
ChainPutObj func(p0 context.Context, p1 blocks.Block) error ``
ChainReadObj func(p0 context.Context, p1 cid.Cid) ([]byte, error) ``
GasEstimateMessageGas func(p0 context.Context, p1 *types.Message, p2 *api.MessageSendSpec, p3 types.TipSetKey) (*types.Message, error) ``
@@ -457,13 +441,13 @@ type GatewayStruct struct {
StateMarketStorageDeal func(p0 context.Context, p1 abi.DealID, p2 types.TipSetKey) (*api.MarketDeal, error) ``
StateMinerInfo func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (api.MinerInfo, error) ``
StateMinerInfo func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (miner.MinerInfo, error) ``
StateMinerPower func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*api.MinerPower, error) ``
StateMinerProvingDeadline func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*dline.Info, error) ``
StateNetworkVersion func(p0 context.Context, p1 types.TipSetKey) (abinetwork.Version, error) ``
StateNetworkVersion func(p0 context.Context, p1 types.TipSetKey) (network.Version, error) ``
StateSearchMsg func(p0 context.Context, p1 cid.Cid) (*api.MsgLookup, error) ``
@@ -691,17 +675,6 @@ func (s *FullNodeStub) ChainNotify(p0 context.Context) (<-chan []*api.HeadChange
return nil, ErrNotSupported
}
func (s *FullNodeStruct) ChainPutObj(p0 context.Context, p1 blocks.Block) error {
if s.Internal.ChainPutObj == nil {
return ErrNotSupported
}
return s.Internal.ChainPutObj(p0, p1)
}
func (s *FullNodeStub) ChainPutObj(p0 context.Context, p1 blocks.Block) error {
return ErrNotSupported
}
func (s *FullNodeStruct) ChainReadObj(p0 context.Context, p1 cid.Cid) ([]byte, error) {
if s.Internal.ChainReadObj == nil {
return *new([]byte), ErrNotSupported
@@ -988,14 +961,14 @@ func (s *FullNodeStub) ClientRestartDataTransfer(p0 context.Context, p1 datatran
return ErrNotSupported
}
func (s *FullNodeStruct) ClientRetrieve(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) error {
func (s *FullNodeStruct) ClientRetrieve(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) error {
if s.Internal.ClientRetrieve == nil {
return ErrNotSupported
}
return s.Internal.ClientRetrieve(p0, p1, p2)
}
func (s *FullNodeStub) ClientRetrieve(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) error {
func (s *FullNodeStub) ClientRetrieve(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) error {
return ErrNotSupported
}
@@ -1010,14 +983,14 @@ func (s *FullNodeStub) ClientRetrieveTryRestartInsufficientFunds(p0 context.Cont
return ErrNotSupported
}
func (s *FullNodeStruct) ClientRetrieveWithEvents(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
func (s *FullNodeStruct) ClientRetrieveWithEvents(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
if s.Internal.ClientRetrieveWithEvents == nil {
return nil, ErrNotSupported
}
return s.Internal.ClientRetrieveWithEvents(p0, p1, p2)
}
func (s *FullNodeStub) ClientRetrieveWithEvents(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
func (s *FullNodeStub) ClientRetrieveWithEvents(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
return nil, ErrNotSupported
}
@@ -1681,17 +1654,6 @@ func (s *FullNodeStub) StateAccountKey(p0 context.Context, p1 address.Address, p
return *new(address.Address), ErrNotSupported
}
func (s *FullNodeStruct) StateActorCodeCIDs(p0 context.Context, p1 abinetwork.Version) (map[string]cid.Cid, error) {
if s.Internal.StateActorCodeCIDs == nil {
return *new(map[string]cid.Cid), ErrNotSupported
}
return s.Internal.StateActorCodeCIDs(p0, p1)
}
func (s *FullNodeStub) StateActorCodeCIDs(p0 context.Context, p1 abinetwork.Version) (map[string]cid.Cid, error) {
return *new(map[string]cid.Cid), ErrNotSupported
}
func (s *FullNodeStruct) StateAllMinerFaults(p0 context.Context, p1 abi.ChainEpoch, p2 types.TipSetKey) ([]*api.Fault, error) {
if s.Internal.StateAllMinerFaults == nil {
return *new([]*api.Fault), ErrNotSupported
@@ -1780,39 +1742,6 @@ func (s *FullNodeStub) StateGetActor(p0 context.Context, p1 address.Address, p2
return nil, ErrNotSupported
}
func (s *FullNodeStruct) StateGetNetworkParams(p0 context.Context) (*api.NetworkParams, error) {
if s.Internal.StateGetNetworkParams == nil {
return nil, ErrNotSupported
}
return s.Internal.StateGetNetworkParams(p0)
}
func (s *FullNodeStub) StateGetNetworkParams(p0 context.Context) (*api.NetworkParams, error) {
return nil, ErrNotSupported
}
func (s *FullNodeStruct) StateGetRandomnessFromBeacon(p0 context.Context, p1 crypto.DomainSeparationTag, p2 abi.ChainEpoch, p3 []byte, p4 types.TipSetKey) (abi.Randomness, error) {
if s.Internal.StateGetRandomnessFromBeacon == nil {
return *new(abi.Randomness), ErrNotSupported
}
return s.Internal.StateGetRandomnessFromBeacon(p0, p1, p2, p3, p4)
}
func (s *FullNodeStub) StateGetRandomnessFromBeacon(p0 context.Context, p1 crypto.DomainSeparationTag, p2 abi.ChainEpoch, p3 []byte, p4 types.TipSetKey) (abi.Randomness, error) {
return *new(abi.Randomness), ErrNotSupported
}
func (s *FullNodeStruct) StateGetRandomnessFromTickets(p0 context.Context, p1 crypto.DomainSeparationTag, p2 abi.ChainEpoch, p3 []byte, p4 types.TipSetKey) (abi.Randomness, error) {
if s.Internal.StateGetRandomnessFromTickets == nil {
return *new(abi.Randomness), ErrNotSupported
}
return s.Internal.StateGetRandomnessFromTickets(p0, p1, p2, p3, p4)
}
func (s *FullNodeStub) StateGetRandomnessFromTickets(p0 context.Context, p1 crypto.DomainSeparationTag, p2 abi.ChainEpoch, p3 []byte, p4 types.TipSetKey) (abi.Randomness, error) {
return *new(abi.Randomness), ErrNotSupported
}
func (s *FullNodeStruct) StateGetReceipt(p0 context.Context, p1 cid.Cid, p2 types.TipSetKey) (*types.MessageReceipt, error) {
if s.Internal.StateGetReceipt == nil {
return nil, ErrNotSupported
@@ -1879,15 +1808,15 @@ func (s *FullNodeStub) StateMarketBalance(p0 context.Context, p1 address.Address
return *new(api.MarketBalance), ErrNotSupported
}
func (s *FullNodeStruct) StateMarketDeals(p0 context.Context, p1 types.TipSetKey) (map[string]*api.MarketDeal, error) {
func (s *FullNodeStruct) StateMarketDeals(p0 context.Context, p1 types.TipSetKey) (map[string]api.MarketDeal, error) {
if s.Internal.StateMarketDeals == nil {
return *new(map[string]*api.MarketDeal), ErrNotSupported
return *new(map[string]api.MarketDeal), ErrNotSupported
}
return s.Internal.StateMarketDeals(p0, p1)
}
func (s *FullNodeStub) StateMarketDeals(p0 context.Context, p1 types.TipSetKey) (map[string]*api.MarketDeal, error) {
return *new(map[string]*api.MarketDeal), ErrNotSupported
func (s *FullNodeStub) StateMarketDeals(p0 context.Context, p1 types.TipSetKey) (map[string]api.MarketDeal, error) {
return *new(map[string]api.MarketDeal), ErrNotSupported
}
func (s *FullNodeStruct) StateMarketParticipants(p0 context.Context, p1 types.TipSetKey) (map[string]api.MarketBalance, error) {
@@ -1956,15 +1885,15 @@ func (s *FullNodeStub) StateMinerFaults(p0 context.Context, p1 address.Address,
return *new(bitfield.BitField), ErrNotSupported
}
func (s *FullNodeStruct) StateMinerInfo(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (api.MinerInfo, error) {
func (s *FullNodeStruct) StateMinerInfo(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (miner.MinerInfo, error) {
if s.Internal.StateMinerInfo == nil {
return *new(api.MinerInfo), ErrNotSupported
return *new(miner.MinerInfo), ErrNotSupported
}
return s.Internal.StateMinerInfo(p0, p1, p2)
}
func (s *FullNodeStub) StateMinerInfo(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (api.MinerInfo, error) {
return *new(api.MinerInfo), ErrNotSupported
func (s *FullNodeStub) StateMinerInfo(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (miner.MinerInfo, error) {
return *new(miner.MinerInfo), ErrNotSupported
}
func (s *FullNodeStruct) StateMinerInitialPledgeCollateral(p0 context.Context, p1 address.Address, p2 miner.SectorPreCommitInfo, p3 types.TipSetKey) (types.BigInt, error) {
@@ -2132,14 +2061,14 @@ func (s *FullNodeStub) StateSearchMsgLimited(p0 context.Context, p1 cid.Cid, p2
return nil, ErrNotSupported
}
func (s *FullNodeStruct) StateSectorExpiration(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*lminer.SectorExpiration, error) {
func (s *FullNodeStruct) StateSectorExpiration(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*miner.SectorExpiration, error) {
if s.Internal.StateSectorExpiration == nil {
return nil, ErrNotSupported
}
return s.Internal.StateSectorExpiration(p0, p1, p2, p3)
}
func (s *FullNodeStub) StateSectorExpiration(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*lminer.SectorExpiration, error) {
func (s *FullNodeStub) StateSectorExpiration(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*miner.SectorExpiration, error) {
return nil, ErrNotSupported
}
@@ -2154,14 +2083,14 @@ func (s *FullNodeStub) StateSectorGetInfo(p0 context.Context, p1 address.Address
return nil, ErrNotSupported
}
func (s *FullNodeStruct) StateSectorPartition(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*lminer.SectorLocation, error) {
func (s *FullNodeStruct) StateSectorPartition(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*miner.SectorLocation, error) {
if s.Internal.StateSectorPartition == nil {
return nil, ErrNotSupported
}
return s.Internal.StateSectorPartition(p0, p1, p2, p3)
}
func (s *FullNodeStub) StateSectorPartition(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*lminer.SectorLocation, error) {
func (s *FullNodeStub) StateSectorPartition(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*miner.SectorLocation, error) {
return nil, ErrNotSupported
}
@@ -2561,17 +2490,6 @@ func (s *GatewayStub) ChainNotify(p0 context.Context) (<-chan []*api.HeadChange,
return nil, ErrNotSupported
}
func (s *GatewayStruct) ChainPutObj(p0 context.Context, p1 blocks.Block) error {
if s.Internal.ChainPutObj == nil {
return ErrNotSupported
}
return s.Internal.ChainPutObj(p0, p1)
}
func (s *GatewayStub) ChainPutObj(p0 context.Context, p1 blocks.Block) error {
return ErrNotSupported
}
func (s *GatewayStruct) ChainReadObj(p0 context.Context, p1 cid.Cid) ([]byte, error) {
if s.Internal.ChainReadObj == nil {
return *new([]byte), ErrNotSupported
@@ -2726,15 +2644,15 @@ func (s *GatewayStub) StateMarketStorageDeal(p0 context.Context, p1 abi.DealID,
return nil, ErrNotSupported
}
func (s *GatewayStruct) StateMinerInfo(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (api.MinerInfo, error) {
func (s *GatewayStruct) StateMinerInfo(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (miner.MinerInfo, error) {
if s.Internal.StateMinerInfo == nil {
return *new(api.MinerInfo), ErrNotSupported
return *new(miner.MinerInfo), ErrNotSupported
}
return s.Internal.StateMinerInfo(p0, p1, p2)
}
func (s *GatewayStub) StateMinerInfo(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (api.MinerInfo, error) {
return *new(api.MinerInfo), ErrNotSupported
func (s *GatewayStub) StateMinerInfo(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (miner.MinerInfo, error) {
return *new(miner.MinerInfo), ErrNotSupported
}
func (s *GatewayStruct) StateMinerPower(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*api.MinerPower, error) {
@@ -2759,15 +2677,15 @@ func (s *GatewayStub) StateMinerProvingDeadline(p0 context.Context, p1 address.A
return nil, ErrNotSupported
}
func (s *GatewayStruct) StateNetworkVersion(p0 context.Context, p1 types.TipSetKey) (abinetwork.Version, error) {
func (s *GatewayStruct) StateNetworkVersion(p0 context.Context, p1 types.TipSetKey) (network.Version, error) {
if s.Internal.StateNetworkVersion == nil {
return *new(abinetwork.Version), ErrNotSupported
return *new(network.Version), ErrNotSupported
}
return s.Internal.StateNetworkVersion(p0, p1)
}
func (s *GatewayStub) StateNetworkVersion(p0 context.Context, p1 types.TipSetKey) (abinetwork.Version, error) {
return *new(abinetwork.Version), ErrNotSupported
func (s *GatewayStub) StateNetworkVersion(p0 context.Context, p1 types.TipSetKey) (network.Version, error) {
return *new(network.Version), ErrNotSupported
}
func (s *GatewayStruct) StateSearchMsg(p0 context.Context, p1 cid.Cid) (*api.MsgLookup, error) {
+22 -219
View File
@@ -7,16 +7,6 @@ package v0mocks
import (
context "context"
reflect "reflect"
time "time"
gomock "github.com/golang/mock/gomock"
uuid "github.com/google/uuid"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
metrics "github.com/libp2p/go-libp2p-core/metrics"
network0 "github.com/libp2p/go-libp2p-core/network"
peer "github.com/libp2p/go-libp2p-core/peer"
protocol "github.com/libp2p/go-libp2p-core/protocol"
address "github.com/filecoin-project/go-address"
bitfield "github.com/filecoin-project/go-bitfield"
@@ -26,21 +16,25 @@ import (
auth "github.com/filecoin-project/go-jsonrpc/auth"
abi "github.com/filecoin-project/go-state-types/abi"
big "github.com/filecoin-project/go-state-types/big"
miner "github.com/filecoin-project/go-state-types/builtin/v8/miner"
paych "github.com/filecoin-project/go-state-types/builtin/v8/paych"
crypto "github.com/filecoin-project/go-state-types/crypto"
dline "github.com/filecoin-project/go-state-types/dline"
network "github.com/filecoin-project/go-state-types/network"
api "github.com/filecoin-project/lotus/api"
apitypes "github.com/filecoin-project/lotus/api/types"
v0api "github.com/filecoin-project/lotus/api/v0api"
miner0 "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
miner "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
types "github.com/filecoin-project/lotus/chain/types"
alerting "github.com/filecoin-project/lotus/journal/alerting"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
imports "github.com/filecoin-project/lotus/node/repo/imports"
miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
gomock "github.com/golang/mock/gomock"
uuid "github.com/google/uuid"
cid "github.com/ipfs/go-cid"
metrics "github.com/libp2p/go-libp2p-core/metrics"
network0 "github.com/libp2p/go-libp2p-core/network"
peer "github.com/libp2p/go-libp2p-core/peer"
protocol "github.com/libp2p/go-libp2p-core/protocol"
)
// MockFullNode is a mock of FullNode interface.
@@ -380,20 +374,6 @@ func (mr *MockFullNodeMockRecorder) ChainNotify(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainNotify", reflect.TypeOf((*MockFullNode)(nil).ChainNotify), arg0)
}
// ChainPutObj mocks base method.
func (m *MockFullNode) ChainPutObj(arg0 context.Context, arg1 blocks.Block) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ChainPutObj", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// ChainPutObj indicates an expected call of ChainPutObj.
func (mr *MockFullNodeMockRecorder) ChainPutObj(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainPutObj", reflect.TypeOf((*MockFullNode)(nil).ChainPutObj), arg0, arg1)
}
// ChainReadObj mocks base method.
func (m *MockFullNode) ChainReadObj(arg0 context.Context, arg1 cid.Cid) ([]byte, error) {
m.ctrl.T.Helper()
@@ -779,7 +759,7 @@ func (mr *MockFullNodeMockRecorder) ClientRestartDataTransfer(arg0, arg1, arg2,
}
// ClientRetrieve mocks base method.
func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 v0api.RetrievalOrder, arg2 *api.FileRef) error {
func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ClientRetrieve", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
@@ -807,7 +787,7 @@ func (mr *MockFullNodeMockRecorder) ClientRetrieveTryRestartInsufficientFunds(ar
}
// ClientRetrieveWithEvents mocks base method.
func (m *MockFullNode) ClientRetrieveWithEvents(arg0 context.Context, arg1 v0api.RetrievalOrder, arg2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
func (m *MockFullNode) ClientRetrieveWithEvents(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ClientRetrieveWithEvents", arg0, arg1, arg2)
ret0, _ := ret[0].(<-chan marketevents.RetrievalEvent)
@@ -970,21 +950,6 @@ func (mr *MockFullNodeMockRecorder) ID(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ID", reflect.TypeOf((*MockFullNode)(nil).ID), arg0)
}
// LogAlerts mocks base method.
func (m *MockFullNode) LogAlerts(arg0 context.Context) ([]alerting.Alert, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "LogAlerts", arg0)
ret0, _ := ret[0].([]alerting.Alert)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// LogAlerts indicates an expected call of LogAlerts.
func (mr *MockFullNodeMockRecorder) LogAlerts(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogAlerts", reflect.TypeOf((*MockFullNode)(nil).LogAlerts), arg0)
}
// LogList mocks base method.
func (m *MockFullNode) LogList(arg0 context.Context) ([]string, error) {
m.ctrl.T.Helper()
@@ -1742,21 +1707,6 @@ func (mr *MockFullNodeMockRecorder) NetFindPeer(arg0, arg1 interface{}) *gomock.
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetFindPeer", reflect.TypeOf((*MockFullNode)(nil).NetFindPeer), arg0, arg1)
}
// NetLimit mocks base method.
func (m *MockFullNode) NetLimit(arg0 context.Context, arg1 string) (api.NetLimit, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetLimit", arg0, arg1)
ret0, _ := ret[0].(api.NetLimit)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// NetLimit indicates an expected call of NetLimit.
func (mr *MockFullNodeMockRecorder) NetLimit(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetLimit", reflect.TypeOf((*MockFullNode)(nil).NetLimit), arg0, arg1)
}
// NetPeerInfo mocks base method.
func (m *MockFullNode) NetPeerInfo(arg0 context.Context, arg1 peer.ID) (*api.ExtendedPeerInfo, error) {
m.ctrl.T.Helper()
@@ -1787,64 +1737,6 @@ func (mr *MockFullNodeMockRecorder) NetPeers(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPeers", reflect.TypeOf((*MockFullNode)(nil).NetPeers), arg0)
}
// NetPing mocks base method.
func (m *MockFullNode) NetPing(arg0 context.Context, arg1 peer.ID) (time.Duration, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetPing", arg0, arg1)
ret0, _ := ret[0].(time.Duration)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// NetPing indicates an expected call of NetPing.
func (mr *MockFullNodeMockRecorder) NetPing(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPing", reflect.TypeOf((*MockFullNode)(nil).NetPing), arg0, arg1)
}
// NetProtectAdd mocks base method.
func (m *MockFullNode) NetProtectAdd(arg0 context.Context, arg1 []peer.ID) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetProtectAdd", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// NetProtectAdd indicates an expected call of NetProtectAdd.
func (mr *MockFullNodeMockRecorder) NetProtectAdd(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectAdd", reflect.TypeOf((*MockFullNode)(nil).NetProtectAdd), arg0, arg1)
}
// NetProtectList mocks base method.
func (m *MockFullNode) NetProtectList(arg0 context.Context) ([]peer.ID, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetProtectList", arg0)
ret0, _ := ret[0].([]peer.ID)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// NetProtectList indicates an expected call of NetProtectList.
func (mr *MockFullNodeMockRecorder) NetProtectList(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectList", reflect.TypeOf((*MockFullNode)(nil).NetProtectList), arg0)
}
// NetProtectRemove mocks base method.
func (m *MockFullNode) NetProtectRemove(arg0 context.Context, arg1 []peer.ID) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetProtectRemove", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// NetProtectRemove indicates an expected call of NetProtectRemove.
func (mr *MockFullNodeMockRecorder) NetProtectRemove(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectRemove", reflect.TypeOf((*MockFullNode)(nil).NetProtectRemove), arg0, arg1)
}
// NetPubsubScores mocks base method.
func (m *MockFullNode) NetPubsubScores(arg0 context.Context) ([]api.PubsubScore, error) {
m.ctrl.T.Helper()
@@ -1860,35 +1752,6 @@ func (mr *MockFullNodeMockRecorder) NetPubsubScores(arg0 interface{}) *gomock.Ca
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPubsubScores", reflect.TypeOf((*MockFullNode)(nil).NetPubsubScores), arg0)
}
// NetSetLimit mocks base method.
func (m *MockFullNode) NetSetLimit(arg0 context.Context, arg1 string, arg2 api.NetLimit) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetSetLimit", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
return ret0
}
// NetSetLimit indicates an expected call of NetSetLimit.
func (mr *MockFullNodeMockRecorder) NetSetLimit(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetSetLimit", reflect.TypeOf((*MockFullNode)(nil).NetSetLimit), arg0, arg1, arg2)
}
// NetStat mocks base method.
func (m *MockFullNode) NetStat(arg0 context.Context, arg1 string) (api.NetStat, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetStat", arg0, arg1)
ret0, _ := ret[0].(api.NetStat)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// NetStat indicates an expected call of NetStat.
func (mr *MockFullNodeMockRecorder) NetStat(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetStat", reflect.TypeOf((*MockFullNode)(nil).NetStat), arg0, arg1)
}
// PaychAllocateLane mocks base method.
func (m *MockFullNode) PaychAllocateLane(arg0 context.Context, arg1 address.Address) (uint64, error) {
m.ctrl.T.Helper()
@@ -2172,21 +2035,6 @@ func (mr *MockFullNodeMockRecorder) StateAccountKey(arg0, arg1, arg2 interface{}
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateAccountKey", reflect.TypeOf((*MockFullNode)(nil).StateAccountKey), arg0, arg1, arg2)
}
// StateActorCodeCIDs mocks base method.
func (m *MockFullNode) StateActorCodeCIDs(arg0 context.Context, arg1 network.Version) (map[string]cid.Cid, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateActorCodeCIDs", arg0, arg1)
ret0, _ := ret[0].(map[string]cid.Cid)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateActorCodeCIDs indicates an expected call of StateActorCodeCIDs.
func (mr *MockFullNodeMockRecorder) StateActorCodeCIDs(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateActorCodeCIDs", reflect.TypeOf((*MockFullNode)(nil).StateActorCodeCIDs), arg0, arg1)
}
// StateAllMinerFaults mocks base method.
func (m *MockFullNode) StateAllMinerFaults(arg0 context.Context, arg1 abi.ChainEpoch, arg2 types.TipSetKey) ([]*api.Fault, error) {
m.ctrl.T.Helper()
@@ -2307,51 +2155,6 @@ func (mr *MockFullNodeMockRecorder) StateGetActor(arg0, arg1, arg2 interface{})
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetActor", reflect.TypeOf((*MockFullNode)(nil).StateGetActor), arg0, arg1, arg2)
}
// StateGetNetworkParams mocks base method.
func (m *MockFullNode) StateGetNetworkParams(arg0 context.Context) (*api.NetworkParams, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateGetNetworkParams", arg0)
ret0, _ := ret[0].(*api.NetworkParams)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateGetNetworkParams indicates an expected call of StateGetNetworkParams.
func (mr *MockFullNodeMockRecorder) StateGetNetworkParams(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetNetworkParams", reflect.TypeOf((*MockFullNode)(nil).StateGetNetworkParams), arg0)
}
// StateGetRandomnessFromBeacon mocks base method.
func (m *MockFullNode) StateGetRandomnessFromBeacon(arg0 context.Context, arg1 crypto.DomainSeparationTag, arg2 abi.ChainEpoch, arg3 []byte, arg4 types.TipSetKey) (abi.Randomness, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateGetRandomnessFromBeacon", arg0, arg1, arg2, arg3, arg4)
ret0, _ := ret[0].(abi.Randomness)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateGetRandomnessFromBeacon indicates an expected call of StateGetRandomnessFromBeacon.
func (mr *MockFullNodeMockRecorder) StateGetRandomnessFromBeacon(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetRandomnessFromBeacon", reflect.TypeOf((*MockFullNode)(nil).StateGetRandomnessFromBeacon), arg0, arg1, arg2, arg3, arg4)
}
// StateGetRandomnessFromTickets mocks base method.
func (m *MockFullNode) StateGetRandomnessFromTickets(arg0 context.Context, arg1 crypto.DomainSeparationTag, arg2 abi.ChainEpoch, arg3 []byte, arg4 types.TipSetKey) (abi.Randomness, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateGetRandomnessFromTickets", arg0, arg1, arg2, arg3, arg4)
ret0, _ := ret[0].(abi.Randomness)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateGetRandomnessFromTickets indicates an expected call of StateGetRandomnessFromTickets.
func (mr *MockFullNodeMockRecorder) StateGetRandomnessFromTickets(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetRandomnessFromTickets", reflect.TypeOf((*MockFullNode)(nil).StateGetRandomnessFromTickets), arg0, arg1, arg2, arg3, arg4)
}
// StateGetReceipt mocks base method.
func (m *MockFullNode) StateGetReceipt(arg0 context.Context, arg1 cid.Cid, arg2 types.TipSetKey) (*types.MessageReceipt, error) {
m.ctrl.T.Helper()
@@ -2443,10 +2246,10 @@ func (mr *MockFullNodeMockRecorder) StateMarketBalance(arg0, arg1, arg2 interfac
}
// StateMarketDeals mocks base method.
func (m *MockFullNode) StateMarketDeals(arg0 context.Context, arg1 types.TipSetKey) (map[string]*api.MarketDeal, error) {
func (m *MockFullNode) StateMarketDeals(arg0 context.Context, arg1 types.TipSetKey) (map[string]api.MarketDeal, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateMarketDeals", arg0, arg1)
ret0, _ := ret[0].(map[string]*api.MarketDeal)
ret0, _ := ret[0].(map[string]api.MarketDeal)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@@ -2548,10 +2351,10 @@ func (mr *MockFullNodeMockRecorder) StateMinerFaults(arg0, arg1, arg2 interface{
}
// StateMinerInfo mocks base method.
func (m *MockFullNode) StateMinerInfo(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (api.MinerInfo, error) {
func (m *MockFullNode) StateMinerInfo(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (miner.MinerInfo, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateMinerInfo", arg0, arg1, arg2)
ret0, _ := ret[0].(api.MinerInfo)
ret0, _ := ret[0].(miner.MinerInfo)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@@ -2563,7 +2366,7 @@ func (mr *MockFullNodeMockRecorder) StateMinerInfo(arg0, arg1, arg2 interface{})
}
// StateMinerInitialPledgeCollateral mocks base method.
func (m *MockFullNode) StateMinerInitialPledgeCollateral(arg0 context.Context, arg1 address.Address, arg2 miner.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
func (m *MockFullNode) StateMinerInitialPledgeCollateral(arg0 context.Context, arg1 address.Address, arg2 miner0.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateMinerInitialPledgeCollateral", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(big.Int)
@@ -2608,7 +2411,7 @@ func (mr *MockFullNodeMockRecorder) StateMinerPower(arg0, arg1, arg2 interface{}
}
// StateMinerPreCommitDepositForPower mocks base method.
func (m *MockFullNode) StateMinerPreCommitDepositForPower(arg0 context.Context, arg1 address.Address, arg2 miner.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
func (m *MockFullNode) StateMinerPreCommitDepositForPower(arg0 context.Context, arg1 address.Address, arg2 miner0.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateMinerPreCommitDepositForPower", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(big.Int)
@@ -2788,10 +2591,10 @@ func (mr *MockFullNodeMockRecorder) StateSearchMsgLimited(arg0, arg1, arg2 inter
}
// StateSectorExpiration mocks base method.
func (m *MockFullNode) StateSectorExpiration(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner0.SectorExpiration, error) {
func (m *MockFullNode) StateSectorExpiration(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner.SectorExpiration, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateSectorExpiration", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(*miner0.SectorExpiration)
ret0, _ := ret[0].(*miner.SectorExpiration)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@@ -2818,10 +2621,10 @@ func (mr *MockFullNodeMockRecorder) StateSectorGetInfo(arg0, arg1, arg2, arg3 in
}
// StateSectorPartition mocks base method.
func (m *MockFullNode) StateSectorPartition(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner0.SectorLocation, error) {
func (m *MockFullNode) StateSectorPartition(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner.SectorLocation, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateSectorPartition", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(*miner0.SectorLocation)
ret0, _ := ret[0].(*miner.SectorLocation)
ret1, _ := ret[1].(error)
return ret0, ret1
}
+5 -187
View File
@@ -3,40 +3,22 @@ package v0api
import (
"context"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/chain/types"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/builtin/v8/miner"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/v1api"
"github.com/filecoin-project/lotus/chain/types"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
)
type WrapperV1Full struct {
v1api.FullNode
}
func (w *WrapperV1Full) StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, s abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) {
pi, err := w.FullNode.StateSectorPreCommitInfo(ctx, maddr, s, tsk)
if err != nil {
return miner.SectorPreCommitOnChainInfo{}, err
}
if pi == nil {
return miner.SectorPreCommitOnChainInfo{}, xerrors.Errorf("precommit info does not exist")
}
return *pi, nil
}
func (w *WrapperV1Full) StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error) {
return w.FullNode.StateSearchMsg(ctx, types.EmptyTSK, msg, api.LookbackNoLimit, true)
}
@@ -124,7 +106,7 @@ func (w *WrapperV1Full) MsigApproveTxnHash(ctx context.Context, msig address.Add
}
func (w *WrapperV1Full) MsigCancel(ctx context.Context, msig address.Address, txID uint64, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) {
p, err := w.FullNode.MsigCancelTxnHash(ctx, msig, txID, to, amt, src, method, params)
p, err := w.FullNode.MsigCancel(ctx, msig, txID, to, amt, src, method, params)
if err != nil {
return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
}
@@ -202,168 +184,4 @@ func (w *WrapperV1Full) MsigRemoveSigner(ctx context.Context, msig address.Addre
return w.executePrototype(ctx, p)
}
func (w *WrapperV1Full) ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
return w.StateGetRandomnessFromTickets(ctx, personalization, randEpoch, entropy, tsk)
}
func (w *WrapperV1Full) ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
return w.StateGetRandomnessFromBeacon(ctx, personalization, randEpoch, entropy, tsk)
}
func (w *WrapperV1Full) ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *api.FileRef) error {
events := make(chan marketevents.RetrievalEvent)
go w.clientRetrieve(ctx, order, ref, events)
for {
select {
case evt, ok := <-events:
if !ok { // done successfully
return nil
}
if evt.Err != "" {
return xerrors.Errorf("retrieval failed: %s", evt.Err)
}
case <-ctx.Done():
return xerrors.Errorf("retrieval timed out")
}
}
}
func (w *WrapperV1Full) ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
events := make(chan marketevents.RetrievalEvent)
go w.clientRetrieve(ctx, order, ref, events)
return events, nil
}
func readSubscribeEvents(ctx context.Context, dealID retrievalmarket.DealID, subscribeEvents <-chan api.RetrievalInfo, events chan marketevents.RetrievalEvent) error {
for {
var subscribeEvent api.RetrievalInfo
var evt retrievalmarket.ClientEvent
select {
case <-ctx.Done():
return xerrors.New("Retrieval Timed Out")
case subscribeEvent = <-subscribeEvents:
if subscribeEvent.ID != dealID {
// we can't check the deal ID ahead of time because:
// 1. We need to subscribe before retrieving.
// 2. We won't know the deal ID until after retrieving.
continue
}
if subscribeEvent.Event != nil {
evt = *subscribeEvent.Event
}
}
select {
case <-ctx.Done():
return xerrors.New("Retrieval Timed Out")
case events <- marketevents.RetrievalEvent{
Event: evt,
Status: subscribeEvent.Status,
BytesReceived: subscribeEvent.BytesReceived,
FundsSpent: subscribeEvent.TotalPaid,
}:
}
switch subscribeEvent.Status {
case retrievalmarket.DealStatusCompleted:
return nil
case retrievalmarket.DealStatusRejected:
return xerrors.Errorf("Retrieval Proposal Rejected: %s", subscribeEvent.Message)
case
retrievalmarket.DealStatusDealNotFound,
retrievalmarket.DealStatusErrored:
return xerrors.Errorf("Retrieval Error: %s", subscribeEvent.Message)
}
}
}
func (w *WrapperV1Full) clientRetrieve(ctx context.Context, order RetrievalOrder, ref *api.FileRef, events chan marketevents.RetrievalEvent) {
defer close(events)
finish := func(e error) {
if e != nil {
events <- marketevents.RetrievalEvent{Err: e.Error(), FundsSpent: big.Zero()}
}
}
var dealID retrievalmarket.DealID
if order.FromLocalCAR == "" {
// Subscribe to events before retrieving to avoid losing events.
subscribeCtx, cancel := context.WithCancel(ctx)
defer cancel()
retrievalEvents, err := w.ClientGetRetrievalUpdates(subscribeCtx)
if err != nil {
finish(xerrors.Errorf("GetRetrievalUpdates failed: %w", err))
return
}
retrievalRes, err := w.FullNode.ClientRetrieve(ctx, api.RetrievalOrder{
Root: order.Root,
Piece: order.Piece,
Size: order.Size,
Total: order.Total,
UnsealPrice: order.UnsealPrice,
PaymentInterval: order.PaymentInterval,
PaymentIntervalIncrease: order.PaymentIntervalIncrease,
Client: order.Client,
Miner: order.Miner,
MinerPeer: order.MinerPeer,
})
if err != nil {
finish(xerrors.Errorf("Retrieve failed: %w", err))
return
}
dealID = retrievalRes.DealID
err = readSubscribeEvents(ctx, retrievalRes.DealID, retrievalEvents, events)
if err != nil {
finish(xerrors.Errorf("Retrieve: %w", err))
return
}
}
// If ref is nil, it only fetches the data into the configured blockstore.
if ref == nil {
finish(nil)
return
}
eref := api.ExportRef{
Root: order.Root,
FromLocalCAR: order.FromLocalCAR,
DealID: dealID,
}
if order.DatamodelPathSelector != nil {
s := api.Selector(*order.DatamodelPathSelector)
eref.DAGs = append(eref.DAGs, api.DagSpec{
DataSelector: &s,
ExportMerkleProof: true,
})
}
finish(w.ClientExport(ctx, eref, *ref))
}
func (w *WrapperV1Full) PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*api.ChannelInfo, error) {
return w.FullNode.PaychFund(ctx, from, to, amt)
}
func (w *WrapperV1Full) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error) {
a, err := w.FullNode.ClientQueryAsk(ctx, p, miner)
if err != nil {
return nil, err
}
return a.Response, nil
}
func (w *WrapperV1Full) BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) {
return w.StateGetBeaconEntry(ctx, epoch)
}
var _ FullNode = &WrapperV1Full{}
+5 -5
View File
@@ -3,7 +3,7 @@ package api
import (
"fmt"
"golang.org/x/xerrors"
xerrors "golang.org/x/xerrors"
)
type Version uint32
@@ -54,11 +54,11 @@ func VersionForType(nodeType NodeType) (Version, error) {
// semver versions of the rpc api exposed
var (
FullAPIVersion0 = newVer(1, 5, 0)
FullAPIVersion1 = newVer(2, 3, 0)
FullAPIVersion0 = newVer(1, 3, 0)
FullAPIVersion1 = newVer(2, 1, 0)
MinerAPIVersion0 = newVer(1, 5, 0)
WorkerAPIVersion0 = newVer(1, 6, 0)
MinerAPIVersion0 = newVer(1, 2, 0)
WorkerAPIVersion0 = newVer(1, 1, 0)
)
//nolint:varcheck,deadcode
+11 -18
View File
@@ -11,7 +11,6 @@ import (
type ChainIO interface {
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
ChainHasObj(context.Context, cid.Cid) (bool, error)
ChainPutObj(context.Context, blocks.Block) error
}
type apiBlockstore struct {
@@ -26,42 +25,36 @@ func NewAPIBlockstore(cio ChainIO) Blockstore {
return Adapt(bs) // return an adapted blockstore.
}
func (a *apiBlockstore) DeleteBlock(context.Context, cid.Cid) error {
func (a *apiBlockstore) DeleteBlock(cid.Cid) error {
return xerrors.New("not supported")
}
func (a *apiBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
return a.api.ChainHasObj(ctx, c)
func (a *apiBlockstore) Has(c cid.Cid) (bool, error) {
return a.api.ChainHasObj(context.TODO(), c)
}
func (a *apiBlockstore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
bb, err := a.api.ChainReadObj(ctx, c)
func (a *apiBlockstore) Get(c cid.Cid) (blocks.Block, error) {
bb, err := a.api.ChainReadObj(context.TODO(), c)
if err != nil {
return nil, err
}
return blocks.NewBlockWithCid(bb, c)
}
func (a *apiBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
bb, err := a.api.ChainReadObj(ctx, c)
func (a *apiBlockstore) GetSize(c cid.Cid) (int, error) {
bb, err := a.api.ChainReadObj(context.TODO(), c)
if err != nil {
return 0, err
}
return len(bb), nil
}
func (a *apiBlockstore) Put(ctx context.Context, block blocks.Block) error {
return a.api.ChainPutObj(ctx, block)
func (a *apiBlockstore) Put(blocks.Block) error {
return xerrors.New("not supported")
}
func (a *apiBlockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
for _, block := range blocks {
err := a.api.ChainPutObj(ctx, block)
if err != nil {
return err
}
}
return nil
func (a *apiBlockstore) PutMany([]blocks.Block) error {
return xerrors.New("not supported")
}
func (a *apiBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-262
View File
@@ -1,262 +0,0 @@
package blockstore
import (
"context"
"sync"
"time"
block "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
"golang.org/x/xerrors"
)
// autolog is a logger for the autobatching blockstore. It is subscoped from the
// blockstore logger.
var autolog = log.Named("auto")
// contains the same set of blocks twice, once as an ordered list for flushing, and as a map for fast access
type blockBatch struct {
blockList []block.Block
blockMap map[cid.Cid]block.Block
}
type AutobatchBlockstore struct {
// TODO: drop if memory consumption is too high
addedCids map[cid.Cid]struct{}
stateLock sync.Mutex
bufferedBatch blockBatch
flushingBatch blockBatch
flushErr error
flushCh chan struct{}
doFlushLock sync.Mutex
flushRetryDelay time.Duration
doneCh chan struct{}
shutdown context.CancelFunc
backingBs Blockstore
bufferCapacity int
bufferSize int
}
func NewAutobatch(ctx context.Context, backingBs Blockstore, bufferCapacity int) *AutobatchBlockstore {
ctx, cancel := context.WithCancel(ctx)
bs := &AutobatchBlockstore{
addedCids: make(map[cid.Cid]struct{}),
backingBs: backingBs,
bufferCapacity: bufferCapacity,
flushCh: make(chan struct{}, 1),
doneCh: make(chan struct{}),
// could be made configable
flushRetryDelay: time.Millisecond * 100,
shutdown: cancel,
}
bs.bufferedBatch.blockMap = make(map[cid.Cid]block.Block)
go bs.flushWorker(ctx)
return bs
}
func (bs *AutobatchBlockstore) Put(ctx context.Context, blk block.Block) error {
bs.stateLock.Lock()
defer bs.stateLock.Unlock()
_, ok := bs.addedCids[blk.Cid()]
if !ok {
bs.addedCids[blk.Cid()] = struct{}{}
bs.bufferedBatch.blockList = append(bs.bufferedBatch.blockList, blk)
bs.bufferedBatch.blockMap[blk.Cid()] = blk
bs.bufferSize += len(blk.RawData())
if bs.bufferSize >= bs.bufferCapacity {
// signal that a flush is appropriate, may be ignored
select {
case bs.flushCh <- struct{}{}:
default:
// do nothing
}
}
}
return nil
}
func (bs *AutobatchBlockstore) flushWorker(ctx context.Context) {
defer close(bs.doneCh)
for {
select {
case <-bs.flushCh:
// TODO: check if we _should_ actually flush. We could get a spurious wakeup
// here.
putErr := bs.doFlush(ctx, false)
for putErr != nil {
select {
case <-ctx.Done():
return
case <-time.After(bs.flushRetryDelay):
autolog.Errorf("FLUSH ERRORED: %w, retrying after %v", putErr, bs.flushRetryDelay)
putErr = bs.doFlush(ctx, true)
}
}
case <-ctx.Done():
// Do one last flush.
_ = bs.doFlush(ctx, false)
return
}
}
}
// caller must NOT hold stateLock
// set retryOnly to true to only retry a failed flush and not flush anything new.
func (bs *AutobatchBlockstore) doFlush(ctx context.Context, retryOnly bool) error {
bs.doFlushLock.Lock()
defer bs.doFlushLock.Unlock()
// If we failed to flush last time, try flushing again.
if bs.flushErr != nil {
bs.flushErr = bs.backingBs.PutMany(ctx, bs.flushingBatch.blockList)
}
// If we failed, or we're _only_ retrying, bail.
if retryOnly || bs.flushErr != nil {
return bs.flushErr
}
// Then take the current batch...
bs.stateLock.Lock()
// We do NOT clear addedCids here, because its purpose is to expedite Puts
bs.flushingBatch = bs.bufferedBatch
bs.bufferedBatch.blockList = make([]block.Block, 0, len(bs.flushingBatch.blockList))
bs.bufferedBatch.blockMap = make(map[cid.Cid]block.Block, len(bs.flushingBatch.blockMap))
bs.stateLock.Unlock()
// And try to flush it.
bs.flushErr = bs.backingBs.PutMany(ctx, bs.flushingBatch.blockList)
// If we succeeded, reset the batch. Otherwise, we'll try again next time.
if bs.flushErr == nil {
bs.stateLock.Lock()
bs.flushingBatch = blockBatch{}
bs.stateLock.Unlock()
}
return bs.flushErr
}
// caller must NOT hold stateLock
func (bs *AutobatchBlockstore) Flush(ctx context.Context) error {
return bs.doFlush(ctx, false)
}
func (bs *AutobatchBlockstore) Shutdown(ctx context.Context) error {
// TODO: Prevent puts after we call this to avoid losing data.
bs.shutdown()
select {
case <-bs.doneCh:
case <-ctx.Done():
return ctx.Err()
}
bs.doFlushLock.Lock()
defer bs.doFlushLock.Unlock()
return bs.flushErr
}
func (bs *AutobatchBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block, error) {
// may seem backward to check the backingBs first, but that is the likeliest case
blk, err := bs.backingBs.Get(ctx, c)
if err == nil {
return blk, nil
}
if !ipld.IsNotFound(err) {
return blk, err
}
bs.stateLock.Lock()
defer bs.stateLock.Unlock()
v, ok := bs.flushingBatch.blockMap[c]
if ok {
return v, nil
}
v, ok = bs.bufferedBatch.blockMap[c]
if ok {
return v, nil
}
return bs.Get(ctx, c)
}
func (bs *AutobatchBlockstore) DeleteBlock(context.Context, cid.Cid) error {
// if we wanted to support this, we would have to:
// - flush
// - delete from the backingBs (if present)
// - remove from addedCids (if present)
// - if present in addedCids, also walk the ordered lists and remove if present
return xerrors.New("deletion is unsupported")
}
func (bs *AutobatchBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
// see note in DeleteBlock()
return xerrors.New("deletion is unsupported")
}
func (bs *AutobatchBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
_, err := bs.Get(ctx, c)
if err == nil {
return true, nil
}
if ipld.IsNotFound(err) {
return false, nil
}
return false, err
}
func (bs *AutobatchBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
blk, err := bs.Get(ctx, c)
if err != nil {
return 0, err
}
return len(blk.RawData()), nil
}
func (bs *AutobatchBlockstore) PutMany(ctx context.Context, blks []block.Block) error {
for _, blk := range blks {
if err := bs.Put(ctx, blk); err != nil {
return err
}
}
return nil
}
func (bs *AutobatchBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
if err := bs.Flush(ctx); err != nil {
return nil, err
}
return bs.backingBs.AllKeysChan(ctx)
}
func (bs *AutobatchBlockstore) HashOnRead(enabled bool) {
bs.backingBs.HashOnRead(enabled)
}
func (bs *AutobatchBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error {
blk, err := bs.Get(ctx, cid)
if err != nil {
return err
}
return callback(blk.RawData())
}
-34
View File
@@ -1,34 +0,0 @@
package blockstore
import (
"context"
"testing"
"github.com/stretchr/testify/require"
)
func TestAutobatchBlockstore(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ab := NewAutobatch(ctx, NewMemory(), len(b0.RawData())+len(b1.RawData())-1)
require.NoError(t, ab.Put(ctx, b0))
require.NoError(t, ab.Put(ctx, b1))
require.NoError(t, ab.Put(ctx, b2))
v0, err := ab.Get(ctx, b0.Cid())
require.NoError(t, err)
require.Equal(t, b0.RawData(), v0.RawData())
v1, err := ab.Get(ctx, b1.Cid())
require.NoError(t, err)
require.Equal(t, b1.RawData(), v1.RawData())
v2, err := ab.Get(ctx, b2.Cid())
require.NoError(t, err)
require.Equal(t, b2.RawData(), v2.RawData())
require.NoError(t, ab.Flush(ctx))
require.NoError(t, ab.Shutdown(ctx))
}
+17 -17
View File
@@ -13,14 +13,14 @@ import (
"github.com/dgraph-io/badger/v2"
"github.com/dgraph-io/badger/v2/options"
"github.com/dgraph-io/badger/v2/pb"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
logger "github.com/ipfs/go-log/v2"
pool "github.com/libp2p/go-buffer-pool"
"github.com/multiformats/go-base32"
"go.uber.org/zap"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
logger "github.com/ipfs/go-log/v2"
pool "github.com/libp2p/go-buffer-pool"
"github.com/filecoin-project/lotus/blockstore"
)
@@ -525,7 +525,7 @@ func (b *Blockstore) Size() (int64, error) {
// View implements blockstore.Viewer, which leverages zero-copy read-only
// access to values.
func (b *Blockstore) View(ctx context.Context, cid cid.Cid, fn func([]byte) error) error {
func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
if err := b.access(); err != nil {
return err
}
@@ -544,7 +544,7 @@ func (b *Blockstore) View(ctx context.Context, cid cid.Cid, fn func([]byte) erro
case nil:
return item.Value(fn)
case badger.ErrKeyNotFound:
return ipld.ErrNotFound{Cid: cid}
return blockstore.ErrNotFound
default:
return fmt.Errorf("failed to view block from badger blockstore: %w", err)
}
@@ -552,7 +552,7 @@ func (b *Blockstore) View(ctx context.Context, cid cid.Cid, fn func([]byte) erro
}
// Has implements Blockstore.Has.
func (b *Blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
func (b *Blockstore) Has(cid cid.Cid) (bool, error) {
if err := b.access(); err != nil {
return false, err
}
@@ -582,9 +582,9 @@ func (b *Blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
}
// Get implements Blockstore.Get.
func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
func (b *Blockstore) Get(cid cid.Cid) (blocks.Block, error) {
if !cid.Defined() {
return nil, ipld.ErrNotFound{Cid: cid}
return nil, blockstore.ErrNotFound
}
if err := b.access(); err != nil {
@@ -607,7 +607,7 @@ func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
val, err = item.ValueCopy(nil)
return err
case badger.ErrKeyNotFound:
return ipld.ErrNotFound{Cid: cid}
return blockstore.ErrNotFound
default:
return fmt.Errorf("failed to get block from badger blockstore: %w", err)
}
@@ -619,7 +619,7 @@ func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
}
// GetSize implements Blockstore.GetSize.
func (b *Blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
func (b *Blockstore) GetSize(cid cid.Cid) (int, error) {
if err := b.access(); err != nil {
return 0, err
}
@@ -639,7 +639,7 @@ func (b *Blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
case nil:
size = int(item.ValueSize())
case badger.ErrKeyNotFound:
return ipld.ErrNotFound{Cid: cid}
return blockstore.ErrNotFound
default:
return fmt.Errorf("failed to get block size from badger blockstore: %w", err)
}
@@ -652,7 +652,7 @@ func (b *Blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
}
// Put implements Blockstore.Put.
func (b *Blockstore) Put(ctx context.Context, block blocks.Block) error {
func (b *Blockstore) Put(block blocks.Block) error {
if err := b.access(); err != nil {
return err
}
@@ -691,7 +691,7 @@ func (b *Blockstore) Put(ctx context.Context, block blocks.Block) error {
}
// PutMany implements Blockstore.PutMany.
func (b *Blockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
func (b *Blockstore) PutMany(blocks []blocks.Block) error {
if err := b.access(); err != nil {
return err
}
@@ -755,7 +755,7 @@ func (b *Blockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
}
// DeleteBlock implements Blockstore.DeleteBlock.
func (b *Blockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
func (b *Blockstore) DeleteBlock(cid cid.Cid) error {
if err := b.access(); err != nil {
return err
}
@@ -774,7 +774,7 @@ func (b *Blockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
})
}
func (b *Blockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
if err := b.access(); err != nil {
return err
}
+26 -23
View File
@@ -1,26 +1,24 @@
//stm: #unit
package badgerbs
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
"github.com/filecoin-project/lotus/blockstore"
)
func TestBadgerBlockstore(t *testing.T) {
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
(&Suite{
NewBlockstore: newBlockstore(DefaultOptions),
OpenBlockstore: openBlockstore(DefaultOptions),
@@ -39,8 +37,6 @@ func TestBadgerBlockstore(t *testing.T) {
}
func TestStorageKey(t *testing.T) {
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
//stm: @SPLITSTORE_BADGER_STORAGE_KEY_001
bs, _ := newBlockstore(DefaultOptions)(t)
bbs := bs.(*Blockstore)
defer bbs.Close() //nolint:errcheck
@@ -76,13 +72,20 @@ func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (
return func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
tb.Helper()
path = tb.TempDir()
path, err := ioutil.TempDir("", "")
if err != nil {
tb.Fatal(err)
}
db, err := Open(optsSupplier(path))
if err != nil {
tb.Fatal(err)
}
tb.Cleanup(func() {
_ = os.RemoveAll(path)
})
return db, path
}
}
@@ -95,11 +98,17 @@ func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB,
}
func testMove(t *testing.T, optsF func(string) Options) {
ctx := context.Background()
basePath := t.TempDir()
basePath, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
dbPath := filepath.Join(basePath, "db")
t.Cleanup(func() {
_ = os.RemoveAll(basePath)
})
db, err := Open(optsF(dbPath))
if err != nil {
t.Fatal(err)
@@ -113,7 +122,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
// add some blocks
for i := 0; i < 10; i++ {
blk := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
err := db.Put(ctx, blk)
err := db.Put(blk)
if err != nil {
t.Fatal(err)
}
@@ -123,7 +132,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
// delete some of them
for i := 5; i < 10; i++ {
c := have[i].Cid()
err := db.DeleteBlock(ctx, c)
err := db.DeleteBlock(c)
if err != nil {
t.Fatal(err)
}
@@ -136,7 +145,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
g.Go(func() error {
for i := 10; i < 1000; i++ {
blk := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
err := db.Put(ctx, blk)
err := db.Put(blk)
if err != nil {
return err
}
@@ -156,7 +165,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
// now check that we have all the blocks in have and none in the deleted lists
checkBlocks := func() {
for _, blk := range have {
has, err := db.Has(ctx, blk.Cid())
has, err := db.Has(blk.Cid())
if err != nil {
t.Fatal(err)
}
@@ -165,7 +174,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
t.Fatal("missing block")
}
blk2, err := db.Get(ctx, blk.Cid())
blk2, err := db.Get(blk.Cid())
if err != nil {
t.Fatal(err)
}
@@ -176,7 +185,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
}
for _, c := range deleted {
has, err := db.Has(ctx, c)
has, err := db.Has(c)
if err != nil {
t.Fatal(err)
}
@@ -254,16 +263,10 @@ func testMove(t *testing.T, optsF func(string) Options) {
}
func TestMoveNoPrefix(t *testing.T) {
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
//stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_COLLECT_GARBAGE_001
testMove(t, DefaultOptions)
}
func TestMoveWithPrefix(t *testing.T) {
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
//stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_COLLECT_GARBAGE_001
testMove(t, func(path string) Options {
opts := DefaultOptions(path)
opts.Prefix = "/prefixed/"
+29 -74
View File
@@ -1,4 +1,3 @@
//stm: #unit
package badgerbs
import (
@@ -12,10 +11,10 @@ import (
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
u "github.com/ipfs/go-ipfs-util"
ipld "github.com/ipfs/go-ipld-format"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/lotus/blockstore"
"github.com/stretchr/testify/require"
)
// TODO: move this to go-ipfs-blockstore.
@@ -45,38 +44,28 @@ func (s *Suite) RunTests(t *testing.T, prefix string) {
}
func (s *Suite) TestGetWhenKeyNotPresent(t *testing.T) {
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
//stm: @SPLITSTORE_BADGER_GET_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
ctx := context.Background()
bs, _ := s.NewBlockstore(t)
if c, ok := bs.(io.Closer); ok {
defer func() { require.NoError(t, c.Close()) }()
}
c := cid.NewCidV0(u.Hash([]byte("stuff")))
bl, err := bs.Get(ctx, c)
bl, err := bs.Get(c)
require.Nil(t, bl)
require.Equal(t, ipld.ErrNotFound{Cid: c}, err)
require.Equal(t, blockstore.ErrNotFound, err)
}
func (s *Suite) TestGetWhenKeyIsNil(t *testing.T) {
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
//stm: @SPLITSTORE_BADGER_GET_001
ctx := context.Background()
bs, _ := s.NewBlockstore(t)
if c, ok := bs.(io.Closer); ok {
defer func() { require.NoError(t, c.Close()) }()
}
_, err := bs.Get(ctx, cid.Undef)
require.Equal(t, ipld.ErrNotFound{Cid: cid.Undef}, err)
_, err := bs.Get(cid.Undef)
require.Equal(t, blockstore.ErrNotFound, err)
}
func (s *Suite) TestPutThenGetBlock(t *testing.T) {
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
//stm: @SPLITSTORE_BADGER_GET_001
ctx := context.Background()
bs, _ := s.NewBlockstore(t)
if c, ok := bs.(io.Closer); ok {
defer func() { require.NoError(t, c.Close()) }()
@@ -84,18 +73,15 @@ func (s *Suite) TestPutThenGetBlock(t *testing.T) {
orig := blocks.NewBlock([]byte("some data"))
err := bs.Put(ctx, orig)
err := bs.Put(orig)
require.NoError(t, err)
fetched, err := bs.Get(ctx, orig.Cid())
fetched, err := bs.Get(orig.Cid())
require.NoError(t, err)
require.Equal(t, orig.RawData(), fetched.RawData())
}
func (s *Suite) TestHas(t *testing.T) {
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
//stm: @SPLITSTORE_BADGER_HAS_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
ctx := context.Background()
bs, _ := s.NewBlockstore(t)
if c, ok := bs.(io.Closer); ok {
defer func() { require.NoError(t, c.Close()) }()
@@ -103,23 +89,19 @@ func (s *Suite) TestHas(t *testing.T) {
orig := blocks.NewBlock([]byte("some data"))
err := bs.Put(ctx, orig)
err := bs.Put(orig)
require.NoError(t, err)
ok, err := bs.Has(ctx, orig.Cid())
ok, err := bs.Has(orig.Cid())
require.NoError(t, err)
require.True(t, ok)
ok, err = bs.Has(ctx, blocks.NewBlock([]byte("another thing")).Cid())
ok, err = bs.Has(blocks.NewBlock([]byte("another thing")).Cid())
require.NoError(t, err)
require.False(t, ok)
}
func (s *Suite) TestCidv0v1(t *testing.T) {
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
//stm: @SPLITSTORE_BADGER_GET_001
ctx := context.Background()
bs, _ := s.NewBlockstore(t)
if c, ok := bs.(io.Closer); ok {
defer func() { require.NoError(t, c.Close()) }()
@@ -127,20 +109,15 @@ func (s *Suite) TestCidv0v1(t *testing.T) {
orig := blocks.NewBlock([]byte("some data"))
err := bs.Put(ctx, orig)
err := bs.Put(orig)
require.NoError(t, err)
fetched, err := bs.Get(ctx, cid.NewCidV1(cid.DagProtobuf, orig.Cid().Hash()))
fetched, err := bs.Get(cid.NewCidV1(cid.DagProtobuf, orig.Cid().Hash()))
require.NoError(t, err)
require.Equal(t, orig.RawData(), fetched.RawData())
}
func (s *Suite) TestPutThenGetSizeBlock(t *testing.T) {
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
//stm: @SPLITSTORE_BADGER_GET_SIZE_001
ctx := context.Background()
bs, _ := s.NewBlockstore(t)
if c, ok := bs.(io.Closer); ok {
defer func() { require.NoError(t, c.Close()) }()
@@ -150,29 +127,26 @@ func (s *Suite) TestPutThenGetSizeBlock(t *testing.T) {
missingBlock := blocks.NewBlock([]byte("missingBlock"))
emptyBlock := blocks.NewBlock([]byte{})
err := bs.Put(ctx, block)
err := bs.Put(block)
require.NoError(t, err)
blockSize, err := bs.GetSize(ctx, block.Cid())
blockSize, err := bs.GetSize(block.Cid())
require.NoError(t, err)
require.Len(t, block.RawData(), blockSize)
err = bs.Put(ctx, emptyBlock)
err = bs.Put(emptyBlock)
require.NoError(t, err)
emptySize, err := bs.GetSize(ctx, emptyBlock.Cid())
emptySize, err := bs.GetSize(emptyBlock.Cid())
require.NoError(t, err)
require.Zero(t, emptySize)
missingCid := missingBlock.Cid()
missingSize, err := bs.GetSize(ctx, missingCid)
require.Equal(t, ipld.ErrNotFound{Cid: missingCid}, err)
missingSize, err := bs.GetSize(missingBlock.Cid())
require.Equal(t, blockstore.ErrNotFound, err)
require.Equal(t, -1, missingSize)
}
func (s *Suite) TestAllKeysSimple(t *testing.T) {
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
bs, _ := s.NewBlockstore(t)
if c, ok := bs.(io.Closer); ok {
defer func() { require.NoError(t, c.Close()) }()
@@ -189,9 +163,6 @@ func (s *Suite) TestAllKeysSimple(t *testing.T) {
}
func (s *Suite) TestAllKeysRespectsContext(t *testing.T) {
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
//stm: @SPLITSTORE_BADGER_ALL_KEYS_CHAN_001
bs, _ := s.NewBlockstore(t)
if c, ok := bs.(io.Closer); ok {
defer func() { require.NoError(t, c.Close()) }()
@@ -222,7 +193,6 @@ func (s *Suite) TestAllKeysRespectsContext(t *testing.T) {
}
func (s *Suite) TestDoubleClose(t *testing.T) {
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
bs, _ := s.NewBlockstore(t)
c, ok := bs.(io.Closer)
if !ok {
@@ -233,10 +203,6 @@ func (s *Suite) TestDoubleClose(t *testing.T) {
}
func (s *Suite) TestReopenPutGet(t *testing.T) {
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
//stm: @SPLITSTORE_BADGER_GET_001
ctx := context.Background()
bs, path := s.NewBlockstore(t)
c, ok := bs.(io.Closer)
if !ok {
@@ -244,7 +210,7 @@ func (s *Suite) TestReopenPutGet(t *testing.T) {
}
orig := blocks.NewBlock([]byte("some data"))
err := bs.Put(ctx, orig)
err := bs.Put(orig)
require.NoError(t, err)
err = c.Close()
@@ -253,7 +219,7 @@ func (s *Suite) TestReopenPutGet(t *testing.T) {
bs, err = s.OpenBlockstore(t, path)
require.NoError(t, err)
fetched, err := bs.Get(ctx, orig.Cid())
fetched, err := bs.Get(orig.Cid())
require.NoError(t, err)
require.Equal(t, orig.RawData(), fetched.RawData())
@@ -262,11 +228,6 @@ func (s *Suite) TestReopenPutGet(t *testing.T) {
}
func (s *Suite) TestPutMany(t *testing.T) {
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
//stm: @SPLITSTORE_BADGER_HAS_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
//stm: @SPLITSTORE_BADGER_GET_001, @SPLITSTORE_BADGER_PUT_MANY_001
//stm: @SPLITSTORE_BADGER_ALL_KEYS_CHAN_001
ctx := context.Background()
bs, _ := s.NewBlockstore(t)
if c, ok := bs.(io.Closer); ok {
defer func() { require.NoError(t, c.Close()) }()
@@ -277,15 +238,15 @@ func (s *Suite) TestPutMany(t *testing.T) {
blocks.NewBlock([]byte("foo2")),
blocks.NewBlock([]byte("foo3")),
}
err := bs.PutMany(ctx, blks)
err := bs.PutMany(blks)
require.NoError(t, err)
for _, blk := range blks {
fetched, err := bs.Get(ctx, blk.Cid())
fetched, err := bs.Get(blk.Cid())
require.NoError(t, err)
require.Equal(t, blk.RawData(), fetched.RawData())
ok, err := bs.Has(ctx, blk.Cid())
ok, err := bs.Has(blk.Cid())
require.NoError(t, err)
require.True(t, ok)
}
@@ -298,12 +259,6 @@ func (s *Suite) TestPutMany(t *testing.T) {
}
func (s *Suite) TestDelete(t *testing.T) {
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
//stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_POOLED_STORAGE_HAS_001
//stm: @SPLITSTORE_BADGER_ALL_KEYS_CHAN_001, @SPLITSTORE_BADGER_HAS_001
//stm: @SPLITSTORE_BADGER_PUT_MANY_001
ctx := context.Background()
bs, _ := s.NewBlockstore(t)
if c, ok := bs.(io.Closer); ok {
defer func() { require.NoError(t, c.Close()) }()
@@ -314,10 +269,10 @@ func (s *Suite) TestDelete(t *testing.T) {
blocks.NewBlock([]byte("foo2")),
blocks.NewBlock([]byte("foo3")),
}
err := bs.PutMany(ctx, blks)
err := bs.PutMany(blks)
require.NoError(t, err)
err = bs.DeleteBlock(ctx, blks[1].Cid())
err = bs.DeleteBlock(blks[1].Cid())
require.NoError(t, err)
ch, err := bs.AllKeysChan(context.Background())
@@ -330,17 +285,17 @@ func (s *Suite) TestDelete(t *testing.T) {
cid.NewCidV1(cid.Raw, blks[2].Cid().Hash()),
})
has, err := bs.Has(ctx, blks[1].Cid())
has, err := bs.Has(blks[1].Cid())
require.NoError(t, err)
require.False(t, has)
}
func insertBlocks(t *testing.T, bs blockstore.BasicBlockstore, count int) []cid.Cid {
ctx := context.Background()
keys := make([]cid.Cid, count)
for i := 0; i < count; i++ {
block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
err := bs.Put(ctx, block)
err := bs.Put(block)
require.NoError(t, err)
// NewBlock assigns a CIDv0; we convert it to CIDv1 because that's what
// the store returns.
+10 -9
View File
@@ -1,16 +1,17 @@
package blockstore
import (
"context"
"github.com/ipfs/go-cid"
cid "github.com/ipfs/go-cid"
ds "github.com/ipfs/go-datastore"
blockstore "github.com/ipfs/go-ipfs-blockstore"
logging "github.com/ipfs/go-log/v2"
blockstore "github.com/ipfs/go-ipfs-blockstore"
)
var log = logging.Logger("blockstore")
var ErrNotFound = blockstore.ErrNotFound
// Blockstore is the blockstore interface used by Lotus. It is the union
// of the basic go-ipfs blockstore, with other capabilities required by Lotus,
// e.g. View or Sync.
@@ -26,7 +27,7 @@ type BasicBlockstore = blockstore.Blockstore
type Viewer = blockstore.Viewer
type BatchDeleter interface {
DeleteMany(ctx context.Context, cids []cid.Cid) error
DeleteMany(cids []cid.Cid) error
}
// BlockstoreIterator is a trait for efficient iteration
@@ -92,17 +93,17 @@ type adaptedBlockstore struct {
var _ Blockstore = (*adaptedBlockstore)(nil)
func (a *adaptedBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error {
blk, err := a.Get(ctx, cid)
func (a *adaptedBlockstore) View(cid cid.Cid, callback func([]byte) error) error {
blk, err := a.Get(cid)
if err != nil {
return err
}
return callback(blk.RawData())
}
func (a *adaptedBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
func (a *adaptedBlockstore) DeleteMany(cids []cid.Cid) error {
for _, cid := range cids {
err := a.DeleteBlock(ctx, cid)
err := a.DeleteBlock(cid)
if err != nil {
return err
}
+25 -26
View File
@@ -6,7 +6,6 @@ import (
block "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
)
// buflog is a logger for the buffered blockstore. It is subscoped from the
@@ -89,55 +88,55 @@ func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid,
return out, nil
}
func (bs *BufferedBlockstore) DeleteBlock(ctx context.Context, c cid.Cid) error {
if err := bs.read.DeleteBlock(ctx, c); err != nil {
func (bs *BufferedBlockstore) DeleteBlock(c cid.Cid) error {
if err := bs.read.DeleteBlock(c); err != nil {
return err
}
return bs.write.DeleteBlock(ctx, c)
return bs.write.DeleteBlock(c)
}
func (bs *BufferedBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
if err := bs.read.DeleteMany(ctx, cids); err != nil {
func (bs *BufferedBlockstore) DeleteMany(cids []cid.Cid) error {
if err := bs.read.DeleteMany(cids); err != nil {
return err
}
return bs.write.DeleteMany(ctx, cids)
return bs.write.DeleteMany(cids)
}
func (bs *BufferedBlockstore) View(ctx context.Context, c cid.Cid, callback func([]byte) error) error {
func (bs *BufferedBlockstore) View(c cid.Cid, callback func([]byte) error) error {
// both stores are viewable.
if err := bs.write.View(ctx, c, callback); ipld.IsNotFound(err) {
if err := bs.write.View(c, callback); err == ErrNotFound {
// not found in write blockstore; fall through.
} else {
return err // propagate errors, or nil, i.e. found.
}
return bs.read.View(ctx, c, callback)
return bs.read.View(c, callback)
}
func (bs *BufferedBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block, error) {
if out, err := bs.write.Get(ctx, c); err != nil {
if !ipld.IsNotFound(err) {
func (bs *BufferedBlockstore) Get(c cid.Cid) (block.Block, error) {
if out, err := bs.write.Get(c); err != nil {
if err != ErrNotFound {
return nil, err
}
} else {
return out, nil
}
return bs.read.Get(ctx, c)
return bs.read.Get(c)
}
func (bs *BufferedBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
s, err := bs.read.GetSize(ctx, c)
if ipld.IsNotFound(err) || s == 0 {
return bs.write.GetSize(ctx, c)
func (bs *BufferedBlockstore) GetSize(c cid.Cid) (int, error) {
s, err := bs.read.GetSize(c)
if err == ErrNotFound || s == 0 {
return bs.write.GetSize(c)
}
return s, err
}
func (bs *BufferedBlockstore) Put(ctx context.Context, blk block.Block) error {
has, err := bs.read.Has(ctx, blk.Cid()) // TODO: consider dropping this check
func (bs *BufferedBlockstore) Put(blk block.Block) error {
has, err := bs.read.Has(blk.Cid()) // TODO: consider dropping this check
if err != nil {
return err
}
@@ -146,11 +145,11 @@ func (bs *BufferedBlockstore) Put(ctx context.Context, blk block.Block) error {
return nil
}
return bs.write.Put(ctx, blk)
return bs.write.Put(blk)
}
func (bs *BufferedBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
has, err := bs.write.Has(ctx, c)
func (bs *BufferedBlockstore) Has(c cid.Cid) (bool, error) {
has, err := bs.write.Has(c)
if err != nil {
return false, err
}
@@ -158,7 +157,7 @@ func (bs *BufferedBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error)
return true, nil
}
return bs.read.Has(ctx, c)
return bs.read.Has(c)
}
func (bs *BufferedBlockstore) HashOnRead(hor bool) {
@@ -166,8 +165,8 @@ func (bs *BufferedBlockstore) HashOnRead(hor bool) {
bs.write.HashOnRead(hor)
}
func (bs *BufferedBlockstore) PutMany(ctx context.Context, blks []block.Block) error {
return bs.write.PutMany(ctx, blks)
func (bs *BufferedBlockstore) PutMany(blks []block.Block) error {
return bs.write.PutMany(blks)
}
func (bs *BufferedBlockstore) Read() Blockstore {
-21
View File
@@ -1,21 +0,0 @@
package blockstore
import (
"context"
)
type hotViewKey struct{}
var hotView = hotViewKey{}
// WithHotView constructs a new context with an option that provides a hint to the blockstore
// (e.g. the splitstore) that the object (and its ipld references) should be kept hot.
func WithHotView(ctx context.Context) context.Context {
return context.WithValue(ctx, hotView, struct{}{})
}
// IsHotView returns true if the hot view option is set in the context
func IsHotView(ctx context.Context) bool {
v := ctx.Value(hotView)
return v != nil
}
+13 -13
View File
@@ -5,7 +5,7 @@ import (
"io"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
cid "github.com/ipfs/go-cid"
)
var _ Blockstore = (*discardstore)(nil)
@@ -18,39 +18,39 @@ func NewDiscardStore(bs Blockstore) Blockstore {
return &discardstore{bs: bs}
}
func (b *discardstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
return b.bs.Has(ctx, cid)
func (b *discardstore) Has(cid cid.Cid) (bool, error) {
return b.bs.Has(cid)
}
func (b *discardstore) HashOnRead(hor bool) {
b.bs.HashOnRead(hor)
}
func (b *discardstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
return b.bs.Get(ctx, cid)
func (b *discardstore) Get(cid cid.Cid) (blocks.Block, error) {
return b.bs.Get(cid)
}
func (b *discardstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
return b.bs.GetSize(ctx, cid)
func (b *discardstore) GetSize(cid cid.Cid) (int, error) {
return b.bs.GetSize(cid)
}
func (b *discardstore) View(ctx context.Context, cid cid.Cid, f func([]byte) error) error {
return b.bs.View(ctx, cid, f)
func (b *discardstore) View(cid cid.Cid, f func([]byte) error) error {
return b.bs.View(cid, f)
}
func (b *discardstore) Put(ctx context.Context, blk blocks.Block) error {
func (b *discardstore) Put(blk blocks.Block) error {
return nil
}
func (b *discardstore) PutMany(ctx context.Context, blks []blocks.Block) error {
func (b *discardstore) PutMany(blks []blocks.Block) error {
return nil
}
func (b *discardstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
func (b *discardstore) DeleteBlock(cid cid.Cid) error {
return nil
}
func (b *discardstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
func (b *discardstore) DeleteMany(cids []cid.Cid) error {
return nil
}
+14 -14
View File
@@ -5,10 +5,10 @@ import (
"sync"
"time"
"golang.org/x/xerrors"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
"golang.org/x/xerrors"
)
// UnwrapFallbackStore takes a blockstore, and returns the underlying blockstore
@@ -56,7 +56,7 @@ func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
if fbs.missFn == nil {
log.Errorw("fallbackstore: missFn not configured yet")
return nil, ipld.ErrNotFound{Cid: c}
return nil, ErrNotFound
}
}
@@ -71,30 +71,30 @@ func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
// chain bitswap puts blocks in temp blockstore which is cleaned up
// every few min (to drop any messages we fetched but don't want)
// in this case we want to keep this block around
if err := fbs.Put(ctx, b); err != nil {
if err := fbs.Put(b); err != nil {
return nil, xerrors.Errorf("persisting fallback-fetched block: %w", err)
}
return b, nil
}
func (fbs *FallbackStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
b, err := fbs.Blockstore.Get(ctx, c)
switch {
case err == nil:
func (fbs *FallbackStore) Get(c cid.Cid) (blocks.Block, error) {
b, err := fbs.Blockstore.Get(c)
switch err {
case nil:
return b, nil
case ipld.IsNotFound(err):
case ErrNotFound:
return fbs.getFallback(c)
default:
return b, err
}
}
func (fbs *FallbackStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
sz, err := fbs.Blockstore.GetSize(ctx, c)
switch {
case err == nil:
func (fbs *FallbackStore) GetSize(c cid.Cid) (int, error) {
sz, err := fbs.Blockstore.GetSize(c)
switch err {
case nil:
return sz, nil
case ipld.IsNotFound(err):
case ErrNotFound:
b, err := fbs.getFallback(c)
if err != nil {
return 0, err
+20 -19
View File
@@ -4,10 +4,11 @@ import (
"context"
"io"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
"golang.org/x/xerrors"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
)
var _ Blockstore = (*idstore)(nil)
@@ -37,7 +38,7 @@ func decodeCid(cid cid.Cid) (inline bool, data []byte, err error) {
return false, nil, err
}
func (b *idstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
func (b *idstore) Has(cid cid.Cid) (bool, error) {
inline, _, err := decodeCid(cid)
if err != nil {
return false, xerrors.Errorf("error decoding Cid: %w", err)
@@ -47,10 +48,10 @@ func (b *idstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
return true, nil
}
return b.bs.Has(ctx, cid)
return b.bs.Has(cid)
}
func (b *idstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
func (b *idstore) Get(cid cid.Cid) (blocks.Block, error) {
inline, data, err := decodeCid(cid)
if err != nil {
return nil, xerrors.Errorf("error decoding Cid: %w", err)
@@ -60,10 +61,10 @@ func (b *idstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
return blocks.NewBlockWithCid(data, cid)
}
return b.bs.Get(ctx, cid)
return b.bs.Get(cid)
}
func (b *idstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
func (b *idstore) GetSize(cid cid.Cid) (int, error) {
inline, data, err := decodeCid(cid)
if err != nil {
return 0, xerrors.Errorf("error decoding Cid: %w", err)
@@ -73,10 +74,10 @@ func (b *idstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
return len(data), err
}
return b.bs.GetSize(ctx, cid)
return b.bs.GetSize(cid)
}
func (b *idstore) View(ctx context.Context, cid cid.Cid, cb func([]byte) error) error {
func (b *idstore) View(cid cid.Cid, cb func([]byte) error) error {
inline, data, err := decodeCid(cid)
if err != nil {
return xerrors.Errorf("error decoding Cid: %w", err)
@@ -86,10 +87,10 @@ func (b *idstore) View(ctx context.Context, cid cid.Cid, cb func([]byte) error)
return cb(data)
}
return b.bs.View(ctx, cid, cb)
return b.bs.View(cid, cb)
}
func (b *idstore) Put(ctx context.Context, blk blocks.Block) error {
func (b *idstore) Put(blk blocks.Block) error {
inline, _, err := decodeCid(blk.Cid())
if err != nil {
return xerrors.Errorf("error decoding Cid: %w", err)
@@ -99,10 +100,10 @@ func (b *idstore) Put(ctx context.Context, blk blocks.Block) error {
return nil
}
return b.bs.Put(ctx, blk)
return b.bs.Put(blk)
}
func (b *idstore) PutMany(ctx context.Context, blks []blocks.Block) error {
func (b *idstore) PutMany(blks []blocks.Block) error {
toPut := make([]blocks.Block, 0, len(blks))
for _, blk := range blks {
inline, _, err := decodeCid(blk.Cid())
@@ -117,13 +118,13 @@ func (b *idstore) PutMany(ctx context.Context, blks []blocks.Block) error {
}
if len(toPut) > 0 {
return b.bs.PutMany(ctx, toPut)
return b.bs.PutMany(toPut)
}
return nil
}
func (b *idstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
func (b *idstore) DeleteBlock(cid cid.Cid) error {
inline, _, err := decodeCid(cid)
if err != nil {
return xerrors.Errorf("error decoding Cid: %w", err)
@@ -133,10 +134,10 @@ func (b *idstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
return nil
}
return b.bs.DeleteBlock(ctx, cid)
return b.bs.DeleteBlock(cid)
}
func (b *idstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
func (b *idstore) DeleteMany(cids []cid.Cid) error {
toDelete := make([]cid.Cid, 0, len(cids))
for _, cid := range cids {
inline, _, err := decodeCid(cid)
@@ -151,7 +152,7 @@ func (b *idstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
}
if len(toDelete) > 0 {
return b.bs.DeleteMany(ctx, toDelete)
return b.bs.DeleteMany(toDelete)
}
return nil
+17 -15
View File
@@ -5,15 +5,17 @@ import (
"context"
"io/ioutil"
"golang.org/x/xerrors"
"github.com/multiformats/go-multiaddr"
"github.com/multiformats/go-multihash"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
httpapi "github.com/ipfs/go-ipfs-http-client"
iface "github.com/ipfs/interface-go-ipfs-core"
"github.com/ipfs/interface-go-ipfs-core/options"
"github.com/ipfs/interface-go-ipfs-core/path"
"github.com/multiformats/go-multiaddr"
"github.com/multiformats/go-multihash"
"golang.org/x/xerrors"
)
type IPFSBlockstore struct {
@@ -77,12 +79,12 @@ func NewRemoteIPFSBlockstore(ctx context.Context, maddr multiaddr.Multiaddr, onl
return Adapt(bs), nil
}
func (i *IPFSBlockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
func (i *IPFSBlockstore) DeleteBlock(cid cid.Cid) error {
return xerrors.Errorf("not supported")
}
func (i *IPFSBlockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
_, err := i.offlineAPI.Block().Stat(ctx, path.IpldPath(cid))
func (i *IPFSBlockstore) Has(cid cid.Cid) (bool, error) {
_, err := i.offlineAPI.Block().Stat(i.ctx, path.IpldPath(cid))
if err != nil {
// The underlying client is running in Offline mode.
// Stat() will fail with an err if the block isn't in the
@@ -97,8 +99,8 @@ func (i *IPFSBlockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
return true, nil
}
func (i *IPFSBlockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
rd, err := i.api.Block().Get(ctx, path.IpldPath(cid))
func (i *IPFSBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
rd, err := i.api.Block().Get(i.ctx, path.IpldPath(cid))
if err != nil {
return nil, xerrors.Errorf("getting ipfs block: %w", err)
}
@@ -111,8 +113,8 @@ func (i *IPFSBlockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, er
return blocks.NewBlockWithCid(data, cid)
}
func (i *IPFSBlockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
st, err := i.api.Block().Stat(ctx, path.IpldPath(cid))
func (i *IPFSBlockstore) GetSize(cid cid.Cid) (int, error) {
st, err := i.api.Block().Stat(i.ctx, path.IpldPath(cid))
if err != nil {
return 0, xerrors.Errorf("getting ipfs block: %w", err)
}
@@ -120,23 +122,23 @@ func (i *IPFSBlockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error)
return st.Size(), nil
}
func (i *IPFSBlockstore) Put(ctx context.Context, block blocks.Block) error {
func (i *IPFSBlockstore) Put(block blocks.Block) error {
mhd, err := multihash.Decode(block.Cid().Hash())
if err != nil {
return err
}
_, err = i.api.Block().Put(ctx, bytes.NewReader(block.RawData()),
_, err = i.api.Block().Put(i.ctx, bytes.NewReader(block.RawData()),
options.Block.Hash(mhd.Code, mhd.Length),
options.Block.Format(multihash.Codes[block.Cid().Type()]))
options.Block.Format(cid.CodecToStr[block.Cid().Type()]))
return err
}
func (i *IPFSBlockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
func (i *IPFSBlockstore) PutMany(blocks []blocks.Block) error {
// TODO: could be done in parallel
for _, block := range blocks {
if err := i.Put(ctx, block); err != nil {
if err := i.Put(block); err != nil {
return err
}
}
+12 -13
View File
@@ -5,7 +5,6 @@ import (
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
)
// NewMemory returns a temporary memory-backed blockstore.
@@ -16,50 +15,50 @@ func NewMemory() MemBlockstore {
// MemBlockstore is a terminal blockstore that keeps blocks in memory.
type MemBlockstore map[cid.Cid]blocks.Block
func (m MemBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
func (m MemBlockstore) DeleteBlock(k cid.Cid) error {
delete(m, k)
return nil
}
func (m MemBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error {
func (m MemBlockstore) DeleteMany(ks []cid.Cid) error {
for _, k := range ks {
delete(m, k)
}
return nil
}
func (m MemBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error) {
func (m MemBlockstore) Has(k cid.Cid) (bool, error) {
_, ok := m[k]
return ok, nil
}
func (m MemBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error {
func (m MemBlockstore) View(k cid.Cid, callback func([]byte) error) error {
b, ok := m[k]
if !ok {
return ipld.ErrNotFound{Cid: k}
return ErrNotFound
}
return callback(b.RawData())
}
func (m MemBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) {
func (m MemBlockstore) Get(k cid.Cid) (blocks.Block, error) {
b, ok := m[k]
if !ok {
return nil, ipld.ErrNotFound{Cid: k}
return nil, ErrNotFound
}
return b, nil
}
// GetSize returns the CIDs mapped BlockSize
func (m MemBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
func (m MemBlockstore) GetSize(k cid.Cid) (int, error) {
b, ok := m[k]
if !ok {
return 0, ipld.ErrNotFound{Cid: k}
return 0, ErrNotFound
}
return len(b.RawData()), nil
}
// Put puts a given block to the underlying datastore
func (m MemBlockstore) Put(ctx context.Context, b blocks.Block) error {
func (m MemBlockstore) Put(b blocks.Block) error {
// Convert to a basic block for safety, but try to reuse the existing
// block if it's already a basic block.
k := b.Cid()
@@ -77,9 +76,9 @@ func (m MemBlockstore) Put(ctx context.Context, b blocks.Block) error {
// PutMany puts a slice of blocks at the same time using batching
// capabilities of the underlying datastore whenever possible.
func (m MemBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
func (m MemBlockstore) PutMany(bs []blocks.Block) error {
for _, b := range bs {
_ = m.Put(ctx, b) // can't fail
_ = m.Put(b) // can't fail
}
return nil
}
+4 -11
View File
@@ -49,11 +49,10 @@ These are options in the `[Chainstore.Splitstore]` section of the configuration:
blockstore and discards writes; this is necessary to support syncing from a snapshot.
- `MarkSetType` -- specifies the type of markset to use during compaction.
The markset is the data structure used by compaction/gc to track live objects.
The default value is "badger", which will use a disk backed markset using badger.
If you have a lot of memory (48G or more) you can also use "map", which will use
an in memory markset, speeding up compaction at the cost of higher memory usage.
Note: If you are using a VPS with a network volume, you need to provision at least
3000 IOPs with the badger markset.
The default value is `"map"`, which will use an in-memory map; if you are limited
in memory (or indeed see compaction run out of memory), you can also specify
`"badger"` which will use an disk backed markset, using badger. This will use
much less memory, but will also make compaction slower.
- `HotStoreMessageRetention` -- specifies how many finalities, beyond the 4
finalities maintained by default, to maintain messages and message receipts in the
hotstore. This is useful for assistive nodes that want to support syncing for other
@@ -106,12 +105,6 @@ Compaction works transactionally with the following algorithm:
- We delete in small batches taking a lock; each batch is checked again for marks, from the concurrent transactional mark, so as to never delete anything live
- We then end the transaction and compact/gc the hotstore.
As of [#8008](https://github.com/filecoin-project/lotus/pull/8008) the compaction algorithm has been
modified to eliminate sorting and maintain the cold object set on disk. This drastically reduces
memory usage; in fact, when using badger as the markset compaction uses very little memory, and
it should be now possible to run splitstore with 32GB of RAM or less without danger of running out of
memory during compaction.
## Garbage Collection
TBD -- see [#6577](https://github.com/filecoin-project/lotus/issues/6577)
-117
View File
@@ -1,117 +0,0 @@
package splitstore
import (
"bufio"
"io"
"os"
"github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
"golang.org/x/xerrors"
)
type Checkpoint struct {
file *os.File
buf *bufio.Writer
}
func NewCheckpoint(path string) (*Checkpoint, error) {
file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_SYNC, 0644)
if err != nil {
return nil, xerrors.Errorf("error creating checkpoint: %w", err)
}
buf := bufio.NewWriter(file)
return &Checkpoint{
file: file,
buf: buf,
}, nil
}
func OpenCheckpoint(path string) (*Checkpoint, cid.Cid, error) {
filein, err := os.Open(path)
if err != nil {
return nil, cid.Undef, xerrors.Errorf("error opening checkpoint for reading: %w", err)
}
defer filein.Close() //nolint:errcheck
bufin := bufio.NewReader(filein)
start, err := readRawCid(bufin, nil)
if err != nil && err != io.EOF {
return nil, cid.Undef, xerrors.Errorf("error reading cid from checkpoint: %w", err)
}
fileout, err := os.OpenFile(path, os.O_WRONLY|os.O_SYNC, 0644)
if err != nil {
return nil, cid.Undef, xerrors.Errorf("error opening checkpoint for writing: %w", err)
}
bufout := bufio.NewWriter(fileout)
return &Checkpoint{
file: fileout,
buf: bufout,
}, start, nil
}
func (cp *Checkpoint) Set(c cid.Cid) error {
if _, err := cp.file.Seek(0, io.SeekStart); err != nil {
return xerrors.Errorf("error seeking beginning of checkpoint: %w", err)
}
if err := writeRawCid(cp.buf, c, true); err != nil {
return xerrors.Errorf("error writing cid to checkpoint: %w", err)
}
return nil
}
func (cp *Checkpoint) Close() error {
if cp.file == nil {
return nil
}
err := cp.file.Close()
cp.file = nil
cp.buf = nil
return err
}
func readRawCid(buf *bufio.Reader, hbuf []byte) (cid.Cid, error) {
sz, err := buf.ReadByte()
if err != nil {
return cid.Undef, err // don't wrap EOF as it is not an error here
}
if hbuf == nil {
hbuf = make([]byte, int(sz))
} else {
hbuf = hbuf[:int(sz)]
}
if _, err := io.ReadFull(buf, hbuf); err != nil {
return cid.Undef, xerrors.Errorf("error reading hash: %w", err) // wrap EOF, it's corrupt
}
hash, err := mh.Cast(hbuf)
if err != nil {
return cid.Undef, xerrors.Errorf("error casting multihash: %w", err)
}
return cid.NewCidV1(cid.Raw, hash), nil
}
func writeRawCid(buf *bufio.Writer, c cid.Cid, flush bool) error {
hash := c.Hash()
if err := buf.WriteByte(byte(len(hash))); err != nil {
return err
}
if _, err := buf.Write(hash); err != nil {
return err
}
if flush {
return buf.Flush()
}
return nil
}
-138
View File
@@ -1,138 +0,0 @@
package splitstore
import (
"path/filepath"
"testing"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
)
func TestCheckpoint(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "checkpoint")
makeCid := func(key string) cid.Cid {
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
if err != nil {
t.Fatal(err)
}
return cid.NewCidV1(cid.Raw, h)
}
k1 := makeCid("a")
k2 := makeCid("b")
k3 := makeCid("c")
k4 := makeCid("d")
cp, err := NewCheckpoint(path)
if err != nil {
t.Fatal(err)
}
if err := cp.Set(k1); err != nil {
t.Fatal(err)
}
if err := cp.Set(k2); err != nil {
t.Fatal(err)
}
if err := cp.Close(); err != nil {
t.Fatal(err)
}
cp, start, err := OpenCheckpoint(path)
if err != nil {
t.Fatal(err)
}
if !start.Equals(k2) {
t.Fatalf("expected start to be %s; got %s", k2, start)
}
if err := cp.Set(k3); err != nil {
t.Fatal(err)
}
if err := cp.Set(k4); err != nil {
t.Fatal(err)
}
if err := cp.Close(); err != nil {
t.Fatal(err)
}
cp, start, err = OpenCheckpoint(path)
if err != nil {
t.Fatal(err)
}
if !start.Equals(k4) {
t.Fatalf("expected start to be %s; got %s", k4, start)
}
if err := cp.Close(); err != nil {
t.Fatal(err)
}
// also test correct operation with an empty checkpoint
cp, err = NewCheckpoint(path)
if err != nil {
t.Fatal(err)
}
if err := cp.Close(); err != nil {
t.Fatal(err)
}
cp, start, err = OpenCheckpoint(path)
if err != nil {
t.Fatal(err)
}
if start.Defined() {
t.Fatal("expected start to be undefined")
}
if err := cp.Set(k1); err != nil {
t.Fatal(err)
}
if err := cp.Set(k2); err != nil {
t.Fatal(err)
}
if err := cp.Close(); err != nil {
t.Fatal(err)
}
cp, start, err = OpenCheckpoint(path)
if err != nil {
t.Fatal(err)
}
if !start.Equals(k2) {
t.Fatalf("expected start to be %s; got %s", k2, start)
}
if err := cp.Set(k3); err != nil {
t.Fatal(err)
}
if err := cp.Set(k4); err != nil {
t.Fatal(err)
}
if err := cp.Close(); err != nil {
t.Fatal(err)
}
cp, start, err = OpenCheckpoint(path)
if err != nil {
t.Fatal(err)
}
if !start.Equals(k4) {
t.Fatalf("expected start to be %s; got %s", k4, start)
}
if err := cp.Close(); err != nil {
t.Fatal(err)
}
}
-101
View File
@@ -1,101 +0,0 @@
package splitstore
import (
"bufio"
"io"
"os"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
)
type ColdSetWriter struct {
file *os.File
buf *bufio.Writer
}
type ColdSetReader struct {
file *os.File
buf *bufio.Reader
}
func NewColdSetWriter(path string) (*ColdSetWriter, error) {
file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
if err != nil {
return nil, xerrors.Errorf("error creating coldset: %w", err)
}
buf := bufio.NewWriter(file)
return &ColdSetWriter{
file: file,
buf: buf,
}, nil
}
func NewColdSetReader(path string) (*ColdSetReader, error) {
file, err := os.Open(path)
if err != nil {
return nil, xerrors.Errorf("error opening coldset: %w", err)
}
buf := bufio.NewReader(file)
return &ColdSetReader{
file: file,
buf: buf,
}, nil
}
func (s *ColdSetWriter) Write(c cid.Cid) error {
return writeRawCid(s.buf, c, false)
}
func (s *ColdSetWriter) Close() error {
if s.file == nil {
return nil
}
err1 := s.buf.Flush()
err2 := s.file.Close()
s.buf = nil
s.file = nil
if err1 != nil {
return err1
}
return err2
}
func (s *ColdSetReader) ForEach(f func(cid.Cid) error) error {
hbuf := make([]byte, 256)
for {
next, err := readRawCid(s.buf, hbuf)
if err != nil {
if err == io.EOF {
return nil
}
return xerrors.Errorf("error reading coldset: %w", err)
}
if err := f(next); err != nil {
return err
}
}
}
func (s *ColdSetReader) Reset() error {
_, err := s.file.Seek(0, io.SeekStart)
return err
}
func (s *ColdSetReader) Close() error {
if s.file == nil {
return nil
}
err := s.file.Close()
s.file = nil
s.buf = nil
return err
}
-90
View File
@@ -1,90 +0,0 @@
package splitstore
import (
"fmt"
"path/filepath"
"testing"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
)
func TestColdSet(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "coldset")
makeCid := func(i int) cid.Cid {
h, err := multihash.Sum([]byte(fmt.Sprintf("cid.%d", i)), multihash.SHA2_256, -1)
if err != nil {
t.Fatal(err)
}
return cid.NewCidV1(cid.Raw, h)
}
const count = 1000
cids := make([]cid.Cid, 0, count)
for i := 0; i < count; i++ {
cids = append(cids, makeCid(i))
}
cw, err := NewColdSetWriter(path)
if err != nil {
t.Fatal(err)
}
for _, c := range cids {
if err := cw.Write(c); err != nil {
t.Fatal(err)
}
}
if err := cw.Close(); err != nil {
t.Fatal(err)
}
cr, err := NewColdSetReader(path)
if err != nil {
t.Fatal(err)
}
index := 0
err = cr.ForEach(func(c cid.Cid) error {
if index >= count {
t.Fatal("too many cids")
}
if !c.Equals(cids[index]) {
t.Fatalf("wrong cid %d; expected %s but got %s", index, cids[index], c)
}
index++
return nil
})
if err != nil {
t.Fatal(err)
}
if err := cr.Reset(); err != nil {
t.Fatal(err)
}
index = 0
err = cr.ForEach(func(c cid.Cid) error {
if index >= count {
t.Fatal("too many cids")
}
if !c.Equals(cids[index]) {
t.Fatalf("wrong cid; expected %s but got %s", cids[index], c)
}
index++
return nil
})
if err != nil {
t.Fatal(err)
}
}
+3 -2
View File
@@ -12,10 +12,11 @@ import (
"sync"
"time"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"go.uber.org/multierr"
"golang.org/x/xerrors"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
)
type debugLog struct {
+20 -16
View File
@@ -3,42 +3,46 @@ package splitstore
import (
"errors"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
)
var errMarkSetClosed = errors.New("markset closed")
// MarkSet is an interface for tracking CIDs during chain and object walks
// MarkSet is a utility to keep track of seen CID, and later query for them.
//
// * If the expected dataset is large, it can be backed by a datastore (e.g. bbolt).
// * If a probabilistic result is acceptable, it can be backed by a bloom filter
type MarkSet interface {
ObjectVisitor
Mark(cid.Cid) error
MarkMany([]cid.Cid) error
Has(cid.Cid) (bool, error)
Close() error
SetConcurrent()
}
// BeginCriticalSection ensures that the markset is persisted to disk for recovery in case
// of abnormal termination during the critical section span.
BeginCriticalSection() error
// EndCriticalSection ends the critical section span.
EndCriticalSection()
type MarkSetVisitor interface {
MarkSet
ObjectVisitor
}
type MarkSetEnv interface {
// New creates a new markset within the environment.
// name is a unique name for this markset, mapped to the filesystem for on-disk persistence.
// Create creates a new markset within the environment.
// name is a unique name for this markset, mapped to the filesystem in disk-backed environments
// sizeHint is a hint about the expected size of the markset
New(name string, sizeHint int64) (MarkSet, error)
// Recover recovers an existing markset persisted on-disk.
Recover(name string) (MarkSet, error)
// Close closes the markset
Create(name string, sizeHint int64) (MarkSet, error)
// CreateVisitor is like Create, but returns a wider interface that supports atomic visits.
// It may not be supported by some markset types (e.g. bloom).
CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error)
// SupportsVisitor returns true if the marksets created by this environment support the visitor interface.
SupportsVisitor() bool
Close() error
}
func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) {
switch mtype {
case "map":
return NewMapMarkSetEnv(path)
return NewMapMarkSetEnv()
case "badger":
return NewBadgerMarkSetEnv(path)
default:
+34 -145
View File
@@ -3,14 +3,15 @@ package splitstore
import (
"os"
"path/filepath"
"runtime"
"sync"
"golang.org/x/xerrors"
"github.com/dgraph-io/badger/v2"
"github.com/dgraph-io/badger/v2/options"
"github.com/ipfs/go-cid"
"go.uber.org/zap"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
)
type BadgerMarkSetEnv struct {
@@ -27,13 +28,13 @@ type BadgerMarkSet struct {
writers int
seqno int
version int
persist bool
db *badger.DB
path string
}
var _ MarkSet = (*BadgerMarkSet)(nil)
var _ MarkSetVisitor = (*BadgerMarkSet)(nil)
var badgerMarkSetBatchSize = 16384
@@ -47,10 +48,11 @@ func NewBadgerMarkSetEnv(path string) (MarkSetEnv, error) {
return &BadgerMarkSetEnv{path: msPath}, nil
}
func (e *BadgerMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
func (e *BadgerMarkSetEnv) create(name string, sizeHint int64) (*BadgerMarkSet, error) {
name += ".tmp"
path := filepath.Join(e.path, name)
db, err := openBadgerDB(path, false)
db, err := openTransientBadgerDB(path)
if err != nil {
return nil, xerrors.Errorf("error creating badger db: %w", err)
}
@@ -66,72 +68,18 @@ func (e *BadgerMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
return ms, nil
}
func (e *BadgerMarkSetEnv) Recover(name string) (MarkSet, error) {
path := filepath.Join(e.path, name)
if _, err := os.Stat(path); err != nil {
return nil, xerrors.Errorf("error stating badger db path: %w", err)
}
db, err := openBadgerDB(path, true)
if err != nil {
return nil, xerrors.Errorf("error creating badger db: %w", err)
}
ms := &BadgerMarkSet{
pend: make(map[string]struct{}),
writing: make(map[int]map[string]struct{}),
db: db,
path: path,
persist: true,
}
ms.cond.L = &ms.mx
return ms, nil
func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
return e.create(name, sizeHint)
}
func (e *BadgerMarkSetEnv) CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error) {
return e.create(name, sizeHint)
}
func (e *BadgerMarkSetEnv) SupportsVisitor() bool { return true }
func (e *BadgerMarkSetEnv) Close() error {
return nil
}
func (s *BadgerMarkSet) BeginCriticalSection() error {
s.mx.Lock()
if s.persist {
s.mx.Unlock()
return nil
}
var write bool
var seqno int
if len(s.pend) > 0 {
write = true
seqno = s.nextBatch()
}
s.persist = true
s.mx.Unlock()
if write {
// all writes sync once perist is true
return s.write(seqno)
}
// wait for any pending writes and sync
s.mx.Lock()
for s.writers > 0 {
s.cond.Wait()
}
s.mx.Unlock()
return s.db.Sync()
}
func (s *BadgerMarkSet) EndCriticalSection() {
s.mx.Lock()
defer s.mx.Unlock()
s.persist = false
return os.RemoveAll(e.path)
}
func (s *BadgerMarkSet) Mark(c cid.Cid) error {
@@ -151,23 +99,6 @@ func (s *BadgerMarkSet) Mark(c cid.Cid) error {
return nil
}
func (s *BadgerMarkSet) MarkMany(batch []cid.Cid) error {
s.mx.Lock()
if s.pend == nil {
s.mx.Unlock()
return errMarkSetClosed
}
write, seqno := s.putMany(batch)
s.mx.Unlock()
if write {
return s.write(seqno)
}
return nil
}
func (s *BadgerMarkSet) Has(c cid.Cid) (bool, error) {
s.mx.RLock()
defer s.mx.RUnlock()
@@ -273,34 +204,16 @@ func (s *BadgerMarkSet) tryDB(key []byte) (has bool, err error) {
// writer holds the exclusive lock
func (s *BadgerMarkSet) put(key string) (write bool, seqno int) {
s.pend[key] = struct{}{}
if !s.persist && len(s.pend) < badgerMarkSetBatchSize {
if len(s.pend) < badgerMarkSetBatchSize {
return false, 0
}
seqno = s.nextBatch()
return true, seqno
}
func (s *BadgerMarkSet) putMany(batch []cid.Cid) (write bool, seqno int) {
for _, c := range batch {
key := string(c.Hash())
s.pend[key] = struct{}{}
}
if !s.persist && len(s.pend) < badgerMarkSetBatchSize {
return false, 0
}
seqno = s.nextBatch()
return true, seqno
}
func (s *BadgerMarkSet) nextBatch() int {
seqno := s.seqno
seqno = s.seqno
s.seqno++
s.writing[seqno] = s.pend
s.pend = make(map[string]struct{})
return seqno
return true, seqno
}
func (s *BadgerMarkSet) write(seqno int) (err error) {
@@ -345,14 +258,6 @@ func (s *BadgerMarkSet) write(seqno int) (err error) {
return xerrors.Errorf("error flushing batch to badger markset: %w", err)
}
s.mx.RLock()
persist := s.persist
s.mx.RUnlock()
if persist {
return s.db.Sync()
}
return nil
}
@@ -372,29 +277,26 @@ func (s *BadgerMarkSet) Close() error {
db := s.db
s.db = nil
return closeBadgerDB(db, s.path, s.persist)
return closeTransientBadgerDB(db, s.path)
}
func openBadgerDB(path string, recover bool) (*badger.DB, error) {
// if it is not a recovery, clean up first
if !recover {
err := os.RemoveAll(path)
if err != nil {
return nil, xerrors.Errorf("error clearing markset directory: %w", err)
}
func (s *BadgerMarkSet) SetConcurrent() {}
err = os.MkdirAll(path, 0755) //nolint:gosec
if err != nil {
return nil, xerrors.Errorf("error creating markset directory: %w", err)
}
func openTransientBadgerDB(path string) (*badger.DB, error) {
// clean up first
err := os.RemoveAll(path)
if err != nil {
return nil, xerrors.Errorf("error clearing markset directory: %w", err)
}
err = os.MkdirAll(path, 0755) //nolint:gosec
if err != nil {
return nil, xerrors.Errorf("error creating markset directory: %w", err)
}
opts := badger.DefaultOptions(path)
// we manually sync when we are in critical section
opts.SyncWrites = false
// no need to do that
opts.CompactL0OnClose = false
// we store hashes, not much to gain by compression
opts.Compression = options.None
// Note: We use FileIO for loading modes to avoid memory thrashing and interference
// between the system blockstore and the markset.
@@ -403,15 +305,6 @@ func openBadgerDB(path string, recover bool) (*badger.DB, error) {
// exceeded 1GB in size.
opts.TableLoadingMode = options.FileIO
opts.ValueLogLoadingMode = options.FileIO
// We increase the number of L0 tables before compaction to make it unlikely to
// be necessary.
opts.NumLevelZeroTables = 20 // default is 5
opts.NumLevelZeroTablesStall = 30 // default is 10
// increase the number of compactors from default 2 so that if we ever have to
// compact, it is fast
if runtime.NumCPU()/2 > opts.NumCompactors {
opts.NumCompactors = runtime.NumCPU() / 2
}
opts.Logger = &badgerLogger{
SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(),
skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(),
@@ -420,16 +313,12 @@ func openBadgerDB(path string, recover bool) (*badger.DB, error) {
return badger.Open(opts)
}
func closeBadgerDB(db *badger.DB, path string, persist bool) error {
func closeTransientBadgerDB(db *badger.DB, path string) error {
err := db.Close()
if err != nil {
return xerrors.Errorf("error closing badger markset: %w", err)
}
if persist {
return nil
}
err = os.RemoveAll(path)
if err != nil {
return xerrors.Errorf("error deleting badger markset: %w", err)
+36 -219
View File
@@ -1,19 +1,12 @@
package splitstore
import (
"bufio"
"io"
"os"
"path/filepath"
"sync"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
)
type MapMarkSetEnv struct {
path string
}
type MapMarkSetEnv struct{}
var _ MarkSetEnv = (*MapMarkSetEnv)(nil)
@@ -21,194 +14,55 @@ type MapMarkSet struct {
mx sync.RWMutex
set map[string]struct{}
persist bool
file *os.File
buf *bufio.Writer
path string
ts bool
}
var _ MarkSet = (*MapMarkSet)(nil)
var _ MarkSetVisitor = (*MapMarkSet)(nil)
func NewMapMarkSetEnv(path string) (*MapMarkSetEnv, error) {
msPath := filepath.Join(path, "markset.map")
err := os.MkdirAll(msPath, 0755) //nolint:gosec
if err != nil {
return nil, xerrors.Errorf("error creating markset directory: %w", err)
}
return &MapMarkSetEnv{path: msPath}, nil
func NewMapMarkSetEnv() (*MapMarkSetEnv, error) {
return &MapMarkSetEnv{}, nil
}
func (e *MapMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
path := filepath.Join(e.path, name)
func (e *MapMarkSetEnv) create(name string, sizeHint int64) (*MapMarkSet, error) {
return &MapMarkSet{
set: make(map[string]struct{}, sizeHint),
path: path,
set: make(map[string]struct{}, sizeHint),
}, nil
}
func (e *MapMarkSetEnv) Recover(name string) (MarkSet, error) {
path := filepath.Join(e.path, name)
s := &MapMarkSet{
set: make(map[string]struct{}),
path: path,
}
in, err := os.Open(path)
if err != nil {
return nil, xerrors.Errorf("error opening markset file for read: %w", err)
}
defer in.Close() //nolint:errcheck
// wrap a buffered reader to make this faster
buf := bufio.NewReader(in)
for {
var sz byte
if sz, err = buf.ReadByte(); err != nil {
break
}
key := make([]byte, int(sz))
if _, err = io.ReadFull(buf, key); err != nil {
break
}
s.set[string(key)] = struct{}{}
}
if err != io.EOF {
return nil, xerrors.Errorf("error reading markset file: %w", err)
}
file, err := os.OpenFile(s.path, os.O_WRONLY|os.O_APPEND, 0)
if err != nil {
return nil, xerrors.Errorf("error opening markset file for write: %w", err)
}
s.persist = true
s.file = file
s.buf = bufio.NewWriter(file)
return s, nil
func (e *MapMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
return e.create(name, sizeHint)
}
func (e *MapMarkSetEnv) CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error) {
return e.create(name, sizeHint)
}
func (e *MapMarkSetEnv) SupportsVisitor() bool { return true }
func (e *MapMarkSetEnv) Close() error {
return nil
}
func (s *MapMarkSet) BeginCriticalSection() error {
s.mx.Lock()
defer s.mx.Unlock()
func (s *MapMarkSet) Mark(cid cid.Cid) error {
if s.ts {
s.mx.Lock()
defer s.mx.Unlock()
}
if s.set == nil {
return errMarkSetClosed
}
if s.persist {
return nil
}
file, err := os.OpenFile(s.path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
if err != nil {
return xerrors.Errorf("error opening markset file: %w", err)
}
// wrap a buffered writer to make this faster
s.buf = bufio.NewWriter(file)
for key := range s.set {
if err := s.writeKey([]byte(key), false); err != nil {
_ = file.Close()
s.buf = nil
return err
}
}
if err := s.buf.Flush(); err != nil {
_ = file.Close()
s.buf = nil
return xerrors.Errorf("error flushing markset file buffer: %w", err)
}
s.file = file
s.persist = true
return nil
}
func (s *MapMarkSet) EndCriticalSection() {
s.mx.Lock()
defer s.mx.Unlock()
if !s.persist {
return
}
_ = s.file.Close()
_ = os.Remove(s.path)
s.file = nil
s.buf = nil
s.persist = false
}
func (s *MapMarkSet) Mark(c cid.Cid) error {
s.mx.Lock()
defer s.mx.Unlock()
if s.set == nil {
return errMarkSetClosed
}
hash := c.Hash()
s.set[string(hash)] = struct{}{}
if s.persist {
if err := s.writeKey(hash, true); err != nil {
return err
}
if err := s.file.Sync(); err != nil {
return xerrors.Errorf("error syncing markset: %w", err)
}
}
return nil
}
func (s *MapMarkSet) MarkMany(batch []cid.Cid) error {
s.mx.Lock()
defer s.mx.Unlock()
if s.set == nil {
return errMarkSetClosed
}
for _, c := range batch {
hash := c.Hash()
s.set[string(hash)] = struct{}{}
if s.persist {
if err := s.writeKey(hash, false); err != nil {
return err
}
}
}
if s.persist {
if err := s.buf.Flush(); err != nil {
return xerrors.Errorf("error flushing markset buffer to disk: %w", err)
}
if err := s.file.Sync(); err != nil {
return xerrors.Errorf("error syncing markset: %w", err)
}
}
s.set[string(cid.Hash())] = struct{}{}
return nil
}
func (s *MapMarkSet) Has(cid cid.Cid) (bool, error) {
s.mx.RLock()
defer s.mx.RUnlock()
if s.ts {
s.mx.RLock()
defer s.mx.RUnlock()
}
if s.set == nil {
return false, errMarkSetClosed
@@ -219,70 +73,33 @@ func (s *MapMarkSet) Has(cid cid.Cid) (bool, error) {
}
func (s *MapMarkSet) Visit(c cid.Cid) (bool, error) {
s.mx.Lock()
defer s.mx.Unlock()
if s.ts {
s.mx.Lock()
defer s.mx.Unlock()
}
if s.set == nil {
return false, errMarkSetClosed
}
hash := c.Hash()
key := string(hash)
key := string(c.Hash())
if _, ok := s.set[key]; ok {
return false, nil
}
s.set[key] = struct{}{}
if s.persist {
if err := s.writeKey(hash, true); err != nil {
return false, err
}
if err := s.file.Sync(); err != nil {
return false, xerrors.Errorf("error syncing markset: %w", err)
}
}
return true, nil
}
func (s *MapMarkSet) Close() error {
s.mx.Lock()
defer s.mx.Unlock()
if s.set == nil {
return nil
if s.ts {
s.mx.Lock()
defer s.mx.Unlock()
}
s.set = nil
if s.file != nil {
if err := s.file.Close(); err != nil {
log.Warnf("error closing markset file: %s", err)
}
if !s.persist {
if err := os.Remove(s.path); err != nil {
log.Warnf("error removing markset file: %s", err)
}
}
}
return nil
}
func (s *MapMarkSet) writeKey(k []byte, flush bool) error {
if err := s.buf.WriteByte(byte(len(k))); err != nil {
return xerrors.Errorf("error writing markset key length to disk: %w", err)
}
if _, err := s.buf.Write(k); err != nil {
return xerrors.Errorf("error writing markset key to disk: %w", err)
}
if flush {
if err := s.buf.Flush(); err != nil {
return xerrors.Errorf("error flushing markset buffer to disk: %w", err)
}
}
return nil
func (s *MapMarkSet) SetConcurrent() {
s.ts = true
}
+28 -324
View File
@@ -1,40 +1,40 @@
//stm: #unit
package splitstore
import (
"io/ioutil"
"os"
"testing"
"github.com/ipfs/go-cid"
cid "github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
)
func TestMapMarkSet(t *testing.T) {
//stm: @SPLITSTORE_MARKSET_CREATE_001, @SPLITSTORE_MARKSET_HAS_001, @@SPLITSTORE_MARKSET_MARK_001
//stm: @SPLITSTORE_MARKSET_CLOSE_001, @SPLITSTORE_MARKSET_CREATE_VISITOR_001
testMarkSet(t, "map")
testMarkSetRecovery(t, "map")
testMarkSetMarkMany(t, "map")
testMarkSetVisitor(t, "map")
testMarkSetVisitorRecovery(t, "map")
}
func TestBadgerMarkSet(t *testing.T) {
//stm: @SPLITSTORE_MARKSET_CREATE_001, @SPLITSTORE_MARKSET_HAS_001, @@SPLITSTORE_MARKSET_MARK_001
//stm: @SPLITSTORE_MARKSET_CLOSE_001, @SPLITSTORE_MARKSET_CREATE_VISITOR_001
bs := badgerMarkSetBatchSize
badgerMarkSetBatchSize = 1
t.Cleanup(func() {
badgerMarkSetBatchSize = bs
})
testMarkSet(t, "badger")
testMarkSetRecovery(t, "badger")
testMarkSetMarkMany(t, "badger")
testMarkSetVisitor(t, "badger")
testMarkSetVisitorRecovery(t, "badger")
}
func testMarkSet(t *testing.T, lsType string) {
path := t.TempDir()
t.Helper()
path, err := ioutil.TempDir("", "markset.*")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = os.RemoveAll(path)
})
env, err := OpenMarkSetEnv(path, lsType)
if err != nil {
@@ -42,13 +42,12 @@ func testMarkSet(t *testing.T, lsType string) {
}
defer env.Close() //nolint:errcheck
// stm: @SPLITSTORE_MARKSET_CREATE_001
hotSet, err := env.New("hot", 0)
hotSet, err := env.Create("hot", 0)
if err != nil {
t.Fatal(err)
}
coldSet, err := env.New("cold", 0)
coldSet, err := env.Create("cold", 0)
if err != nil {
t.Fatal(err)
}
@@ -62,9 +61,7 @@ func testMarkSet(t *testing.T, lsType string) {
return cid.NewCidV1(cid.Raw, h)
}
// stm: @SPLITSTORE_MARKSET_HAS_001
mustHave := func(s MarkSet, cid cid.Cid) {
t.Helper()
has, err := s.Has(cid)
if err != nil {
t.Fatal(err)
@@ -76,7 +73,6 @@ func testMarkSet(t *testing.T, lsType string) {
}
mustNotHave := func(s MarkSet, cid cid.Cid) {
t.Helper()
has, err := s.Has(cid)
if err != nil {
t.Fatal(err)
@@ -92,7 +88,6 @@ func testMarkSet(t *testing.T, lsType string) {
k3 := makeCid("c")
k4 := makeCid("d")
// stm: @SPLITSTORE_MARKSET_MARK_001
hotSet.Mark(k1) //nolint
hotSet.Mark(k2) //nolint
coldSet.Mark(k3) //nolint
@@ -119,12 +114,12 @@ func testMarkSet(t *testing.T, lsType string) {
t.Fatal(err)
}
hotSet, err = env.New("hot", 0)
hotSet, err = env.Create("hot", 0)
if err != nil {
t.Fatal(err)
}
coldSet, err = env.New("cold", 0)
coldSet, err = env.Create("cold", 0)
if err != nil {
t.Fatal(err)
}
@@ -143,7 +138,6 @@ func testMarkSet(t *testing.T, lsType string) {
mustNotHave(coldSet, k3)
mustNotHave(coldSet, k4)
//stm: @SPLITSTORE_MARKSET_CLOSE_001
err = hotSet.Close()
if err != nil {
t.Fatal(err)
@@ -156,7 +150,16 @@ func testMarkSet(t *testing.T, lsType string) {
}
func testMarkSetVisitor(t *testing.T, lsType string) {
path := t.TempDir()
t.Helper()
path, err := ioutil.TempDir("", "markset.*")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = os.RemoveAll(path)
})
env, err := OpenMarkSetEnv(path, lsType)
if err != nil {
@@ -164,8 +167,7 @@ func testMarkSetVisitor(t *testing.T, lsType string) {
}
defer env.Close() //nolint:errcheck
//stm: @SPLITSTORE_MARKSET_CREATE_VISITOR_001
visitor, err := env.New("test", 0)
visitor, err := env.CreateVisitor("test", 0)
if err != nil {
t.Fatal(err)
}
@@ -217,301 +219,3 @@ func testMarkSetVisitor(t *testing.T, lsType string) {
mustNotVisit(visitor, k3)
mustNotVisit(visitor, k4)
}
func testMarkSetVisitorRecovery(t *testing.T, lsType string) {
path := t.TempDir()
env, err := OpenMarkSetEnv(path, lsType)
if err != nil {
t.Fatal(err)
}
defer env.Close() //nolint:errcheck
visitor, err := env.New("test", 0)
if err != nil {
t.Fatal(err)
}
defer visitor.Close() //nolint:errcheck
makeCid := func(key string) cid.Cid {
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
if err != nil {
t.Fatal(err)
}
return cid.NewCidV1(cid.Raw, h)
}
mustVisit := func(v ObjectVisitor, cid cid.Cid) {
visit, err := v.Visit(cid)
if err != nil {
t.Fatal(err)
}
if !visit {
t.Fatal("object should be visited")
}
}
mustNotVisit := func(v ObjectVisitor, cid cid.Cid) {
visit, err := v.Visit(cid)
if err != nil {
t.Fatal(err)
}
if visit {
t.Fatal("unexpected visit")
}
}
k1 := makeCid("a")
k2 := makeCid("b")
k3 := makeCid("c")
k4 := makeCid("d")
mustVisit(visitor, k1)
mustVisit(visitor, k2)
if err := visitor.BeginCriticalSection(); err != nil {
t.Fatal(err)
}
mustVisit(visitor, k3)
mustVisit(visitor, k4)
mustNotVisit(visitor, k1)
mustNotVisit(visitor, k2)
mustNotVisit(visitor, k3)
mustNotVisit(visitor, k4)
if err := visitor.Close(); err != nil {
t.Fatal(err)
}
visitor, err = env.Recover("test")
if err != nil {
t.Fatal(err)
}
mustNotVisit(visitor, k1)
mustNotVisit(visitor, k2)
mustNotVisit(visitor, k3)
mustNotVisit(visitor, k4)
visitor.EndCriticalSection()
if err := visitor.Close(); err != nil {
t.Fatal(err)
}
_, err = env.Recover("test")
if err == nil {
t.Fatal("expected recovery to fail")
}
}
func testMarkSetRecovery(t *testing.T, lsType string) {
path := t.TempDir()
env, err := OpenMarkSetEnv(path, lsType)
if err != nil {
t.Fatal(err)
}
defer env.Close() //nolint:errcheck
markSet, err := env.New("test", 0)
if err != nil {
t.Fatal(err)
}
makeCid := func(key string) cid.Cid {
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
if err != nil {
t.Fatal(err)
}
return cid.NewCidV1(cid.Raw, h)
}
mustHave := func(s MarkSet, cid cid.Cid) {
t.Helper()
has, err := s.Has(cid)
if err != nil {
t.Fatal(err)
}
if !has {
t.Fatal("mark not found")
}
}
mustNotHave := func(s MarkSet, cid cid.Cid) {
t.Helper()
has, err := s.Has(cid)
if err != nil {
t.Fatal(err)
}
if has {
t.Fatal("unexpected mark")
}
}
k1 := makeCid("a")
k2 := makeCid("b")
k3 := makeCid("c")
k4 := makeCid("d")
if err := markSet.Mark(k1); err != nil {
t.Fatal(err)
}
if err := markSet.Mark(k2); err != nil {
t.Fatal(err)
}
mustHave(markSet, k1)
mustHave(markSet, k2)
mustNotHave(markSet, k3)
mustNotHave(markSet, k4)
if err := markSet.BeginCriticalSection(); err != nil {
t.Fatal(err)
}
if err := markSet.Mark(k3); err != nil {
t.Fatal(err)
}
if err := markSet.Mark(k4); err != nil {
t.Fatal(err)
}
mustHave(markSet, k1)
mustHave(markSet, k2)
mustHave(markSet, k3)
mustHave(markSet, k4)
if err := markSet.Close(); err != nil {
t.Fatal(err)
}
markSet, err = env.Recover("test")
if err != nil {
t.Fatal(err)
}
mustHave(markSet, k1)
mustHave(markSet, k2)
mustHave(markSet, k3)
mustHave(markSet, k4)
markSet.EndCriticalSection()
if err := markSet.Close(); err != nil {
t.Fatal(err)
}
_, err = env.Recover("test")
if err == nil {
t.Fatal("expected recovery to fail")
}
}
func testMarkSetMarkMany(t *testing.T, lsType string) {
path := t.TempDir()
env, err := OpenMarkSetEnv(path, lsType)
if err != nil {
t.Fatal(err)
}
defer env.Close() //nolint:errcheck
markSet, err := env.New("test", 0)
if err != nil {
t.Fatal(err)
}
makeCid := func(key string) cid.Cid {
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
if err != nil {
t.Fatal(err)
}
return cid.NewCidV1(cid.Raw, h)
}
mustHave := func(s MarkSet, cid cid.Cid) {
t.Helper()
has, err := s.Has(cid)
if err != nil {
t.Fatal(err)
}
if !has {
t.Fatal("mark not found")
}
}
mustNotHave := func(s MarkSet, cid cid.Cid) {
t.Helper()
has, err := s.Has(cid)
if err != nil {
t.Fatal(err)
}
if has {
t.Fatal("unexpected mark")
}
}
k1 := makeCid("a")
k2 := makeCid("b")
k3 := makeCid("c")
k4 := makeCid("d")
if err := markSet.MarkMany([]cid.Cid{k1, k2}); err != nil {
t.Fatal(err)
}
mustHave(markSet, k1)
mustHave(markSet, k2)
mustNotHave(markSet, k3)
mustNotHave(markSet, k4)
if err := markSet.BeginCriticalSection(); err != nil {
t.Fatal(err)
}
if err := markSet.MarkMany([]cid.Cid{k3, k4}); err != nil {
t.Fatal(err)
}
mustHave(markSet, k1)
mustHave(markSet, k2)
mustHave(markSet, k3)
mustHave(markSet, k4)
if err := markSet.Close(); err != nil {
t.Fatal(err)
}
markSet, err = env.Recover("test")
if err != nil {
t.Fatal(err)
}
mustHave(markSet, k1)
mustHave(markSet, k2)
mustHave(markSet, k3)
mustHave(markSet, k4)
markSet.EndCriticalSection()
if err := markSet.Close(); err != nil {
t.Fatal(err)
}
_, err = env.Recover("test")
if err == nil {
t.Fatal("expected recovery to fail")
}
}
+51 -192
View File
@@ -8,22 +8,20 @@ import (
"sync/atomic"
"time"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
dstore "github.com/ipfs/go-datastore"
ipld "github.com/ipfs/go-ipld-format"
logging "github.com/ipfs/go-log/v2"
"go.opencensus.io/stats"
"go.uber.org/multierr"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-state-types/abi"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
dstore "github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log/v2"
"github.com/filecoin-project/go-state-types/abi"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/metrics"
"go.opencensus.io/stats"
)
var (
@@ -49,9 +47,6 @@ var (
enableDebugLog = false
// set this to true if you want to track origin stack traces in the write log
enableDebugLogWriteTraces = false
// upgradeBoundary is the boundary before and after an upgrade where we suppress compaction
upgradeBoundary = build.Finality
)
func init() {
@@ -103,12 +98,6 @@ type ChainAccessor interface {
SubscribeHeadChanges(change func(revert []*types.TipSet, apply []*types.TipSet) error)
}
// upgradeRange is a precomputed epoch range during which we shouldn't compact so as to not
// interfere with an upgrade
type upgradeRange struct {
start, end abi.ChainEpoch
}
// hotstore is the interface that must be satisfied by the hot blockstore; it is an extension
// of the Blockstore interface with the traits we need for compaction.
type hotstore interface {
@@ -129,13 +118,13 @@ type SplitStore struct {
headChangeMx sync.Mutex
coldPurgeSize int
chain ChainAccessor
ds dstore.Datastore
cold bstore.Blockstore
hot hotstore
upgrades []upgradeRange
markSetEnv MarkSetEnv
markSetSize int64
@@ -156,17 +145,6 @@ type SplitStore struct {
txnRefsMx sync.Mutex
txnRefs map[cid.Cid]struct{}
txnMissing map[cid.Cid]struct{}
txnMarkSet MarkSet
txnSyncMx sync.Mutex
txnSyncCond sync.Cond
txnSync bool
// background cold object reification
reifyWorkers sync.WaitGroup
reifyMx sync.Mutex
reifyCond sync.Cond
reifyPend map[cid.Cid]struct{}
reifyInProgress map[cid.Cid]struct{}
// registered protectors
protectors []func(func(cid.Cid) error) error
@@ -195,6 +173,10 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
return nil, err
}
if !markSetEnv.SupportsVisitor() {
return nil, xerrors.Errorf("markset type does not support atomic visitors")
}
// and now we can make a SplitStore
ss := &SplitStore{
cfg: cfg,
@@ -203,16 +185,13 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
cold: cold,
hot: hots,
markSetEnv: markSetEnv,
coldPurgeSize: defaultColdPurgeSize,
}
ss.txnViewsCond.L = &ss.txnViewsMx
ss.txnSyncCond.L = &ss.txnSyncMx
ss.ctx, ss.cancel = context.WithCancel(context.Background())
ss.reifyCond.L = &ss.reifyMx
ss.reifyPend = make(map[cid.Cid]struct{})
ss.reifyInProgress = make(map[cid.Cid]struct{})
if enableDebugLog {
ss.debug, err = openDebugLog(path)
if err != nil {
@@ -220,29 +199,21 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
}
}
if ss.checkpointExists() {
log.Info("found compaction checkpoint; resuming compaction")
if err := ss.completeCompaction(); err != nil {
markSetEnv.Close() //nolint:errcheck
return nil, xerrors.Errorf("error resuming compaction: %w", err)
}
}
return ss, nil
}
// Blockstore interface
func (s *SplitStore) DeleteBlock(_ context.Context, _ cid.Cid) error {
func (s *SplitStore) DeleteBlock(_ cid.Cid) error {
// afaict we don't seem to be using this method, so it's not implemented
return errors.New("DeleteBlock not implemented on SplitStore; don't do this Luke!") //nolint
}
func (s *SplitStore) DeleteMany(_ context.Context, _ []cid.Cid) error {
func (s *SplitStore) DeleteMany(_ []cid.Cid) error {
// afaict we don't seem to be using this method, so it's not implemented
return errors.New("DeleteMany not implemented on SplitStore; don't do this Luke!") //nolint
}
func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
func (s *SplitStore) Has(cid cid.Cid) (bool, error) {
if isIdentiyCid(cid) {
return true, nil
}
@@ -250,21 +221,7 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
s.txnLk.RLock()
defer s.txnLk.RUnlock()
// critical section
if s.txnMarkSet != nil {
has, err := s.txnMarkSet.Has(cid)
if err != nil {
return false, err
}
if has {
return s.has(cid)
}
return s.cold.Has(ctx, cid)
}
has, err := s.hot.Has(ctx, cid)
has, err := s.hot.Has(cid)
if err != nil {
return has, err
@@ -275,16 +232,10 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
return true, nil
}
has, err = s.cold.Has(ctx, cid)
if has && bstore.IsHotView(ctx) {
s.reifyColdObject(cid)
}
return has, err
return s.cold.Has(cid)
}
func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
func (s *SplitStore) Get(cid cid.Cid) (blocks.Block, error) {
if isIdentiyCid(cid) {
data, err := decodeIdentityCid(cid)
if err != nil {
@@ -297,39 +248,22 @@ func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
s.txnLk.RLock()
defer s.txnLk.RUnlock()
// critical section
if s.txnMarkSet != nil {
has, err := s.txnMarkSet.Has(cid)
if err != nil {
return nil, err
}
blk, err := s.hot.Get(cid)
if has {
return s.get(cid)
}
return s.cold.Get(ctx, cid)
}
blk, err := s.hot.Get(ctx, cid)
switch {
case err == nil:
switch err {
case nil:
s.trackTxnRef(cid)
return blk, nil
case ipld.IsNotFound(err):
case bstore.ErrNotFound:
if s.isWarm() {
s.debug.LogReadMiss(cid)
}
blk, err = s.cold.Get(ctx, cid)
blk, err = s.cold.Get(cid)
if err == nil {
if bstore.IsHotView(ctx) {
s.reifyColdObject(cid)
}
stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
}
return blk, err
@@ -338,7 +272,7 @@ func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
}
}
func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
func (s *SplitStore) GetSize(cid cid.Cid) (int, error) {
if isIdentiyCid(cid) {
data, err := decodeIdentityCid(cid)
if err != nil {
@@ -351,38 +285,20 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
s.txnLk.RLock()
defer s.txnLk.RUnlock()
// critical section
if s.txnMarkSet != nil {
has, err := s.txnMarkSet.Has(cid)
if err != nil {
return 0, err
}
size, err := s.hot.GetSize(cid)
if has {
return s.getSize(cid)
}
return s.cold.GetSize(ctx, cid)
}
size, err := s.hot.GetSize(ctx, cid)
switch {
case err == nil:
switch err {
case nil:
s.trackTxnRef(cid)
return size, nil
case ipld.IsNotFound(err):
case bstore.ErrNotFound:
if s.isWarm() {
s.debug.LogReadMiss(cid)
}
size, err = s.cold.GetSize(ctx, cid)
size, err = s.cold.GetSize(cid)
if err == nil {
if bstore.IsHotView(ctx) {
s.reifyColdObject(cid)
}
stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
}
return size, err
@@ -392,7 +308,7 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
}
}
func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error {
func (s *SplitStore) Put(blk blocks.Block) error {
if isIdentiyCid(blk.Cid()) {
return nil
}
@@ -400,24 +316,18 @@ func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error {
s.txnLk.RLock()
defer s.txnLk.RUnlock()
err := s.hot.Put(ctx, blk)
err := s.hot.Put(blk)
if err != nil {
return err
}
s.debug.LogWrite(blk)
// critical section
if s.txnMarkSet != nil {
s.markLiveRefs([]cid.Cid{blk.Cid()})
return nil
}
s.trackTxnRef(blk.Cid())
return nil
}
func (s *SplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {
func (s *SplitStore) PutMany(blks []blocks.Block) error {
// filter identites
idcids := 0
for _, blk := range blks {
@@ -451,19 +361,13 @@ func (s *SplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {
s.txnLk.RLock()
defer s.txnLk.RUnlock()
err := s.hot.PutMany(ctx, blks)
err := s.hot.PutMany(blks)
if err != nil {
return err
}
s.debug.LogWriteMany(blks)
// critical section
if s.txnMarkSet != nil {
s.markLiveRefs(batch)
return nil
}
s.trackTxnRefMany(batch)
return nil
}
@@ -513,7 +417,7 @@ func (s *SplitStore) HashOnRead(enabled bool) {
s.cold.HashOnRead(enabled)
}
func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) error) error {
func (s *SplitStore) View(cid cid.Cid, cb func([]byte) error) error {
if isIdentiyCid(cid) {
data, err := decodeIdentityCid(cid)
if err != nil {
@@ -523,23 +427,6 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
return cb(data)
}
// critical section
s.txnLk.RLock() // the lock is released in protectView if we are not in critical section
if s.txnMarkSet != nil {
has, err := s.txnMarkSet.Has(cid)
s.txnLk.RUnlock()
if err != nil {
return err
}
if has {
return s.view(cid, cb)
}
return s.cold.View(ctx, cid, cb)
}
// views are (optimistically) protected two-fold:
// - if there is an active transaction, then the reference is protected.
// - if there is no active transaction, active views are tracked in a
@@ -551,23 +438,22 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
s.protectView(cid)
defer s.viewDone()
err := s.hot.View(ctx, cid, cb)
if ipld.IsNotFound(err) {
err := s.hot.View(cid, cb)
switch err {
case bstore.ErrNotFound:
if s.isWarm() {
s.debug.LogReadMiss(cid)
}
err = s.cold.View(ctx, cid, cb)
err = s.cold.View(cid, cb)
if err == nil {
if bstore.IsHotView(ctx) {
s.reifyColdObject(cid)
}
stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
}
return err
default:
return err
}
return err
}
func (s *SplitStore) isWarm() bool {
@@ -577,33 +463,16 @@ func (s *SplitStore) isWarm() bool {
}
// State tracking
func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error {
func (s *SplitStore) Start(chain ChainAccessor) error {
s.chain = chain
curTs := chain.GetHeaviestTipSet()
// precompute the upgrade boundaries
s.upgrades = make([]upgradeRange, 0, len(us))
for _, upgrade := range us {
boundary := upgrade.Height
for _, pre := range upgrade.PreMigrations {
preMigrationBoundary := upgrade.Height - pre.StartWithin
if preMigrationBoundary < boundary {
boundary = preMigrationBoundary
}
}
upgradeStart := boundary - upgradeBoundary
upgradeEnd := upgrade.Height + upgradeBoundary
s.upgrades = append(s.upgrades, upgradeRange{start: upgradeStart, end: upgradeEnd})
}
// should we warmup
warmup := false
// load base epoch from metadata ds
// if none, then use current epoch because it's a fresh start
bs, err := s.ds.Get(s.ctx, baseEpochKey)
bs, err := s.ds.Get(baseEpochKey)
switch err {
case nil:
s.baseEpoch = bytesToEpoch(bs)
@@ -624,7 +493,7 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
}
// load warmup epoch from metadata ds
bs, err = s.ds.Get(s.ctx, warmupEpochKey)
bs, err = s.ds.Get(warmupEpochKey)
switch err {
case nil:
s.warmupEpoch = bytesToEpoch(bs)
@@ -637,7 +506,7 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
}
// load markSetSize from metadata ds to provide a size hint for marksets
bs, err = s.ds.Get(s.ctx, markSetSizeKey)
bs, err = s.ds.Get(markSetSizeKey)
switch err {
case nil:
s.markSetSize = bytesToInt64(bs)
@@ -648,7 +517,7 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
}
// load compactionIndex from metadata ds to provide a hint as to when to perform moving gc
bs, err = s.ds.Get(s.ctx, compactionIndexKey)
bs, err = s.ds.Get(compactionIndexKey)
switch err {
case nil:
s.compactionIndex = bytesToInt64(bs)
@@ -670,9 +539,6 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
}
}
// spawn the reifier
go s.reifyOrchestrator()
// watch the chain
chain.SubscribeHeadChanges(s.HeadChange)
@@ -693,19 +559,12 @@ func (s *SplitStore) Close() error {
}
if atomic.LoadInt32(&s.compacting) == 1 {
s.txnSyncMx.Lock()
s.txnSync = true
s.txnSyncCond.Broadcast()
s.txnSyncMx.Unlock()
log.Warn("close with ongoing compaction in progress; waiting for it to finish...")
for atomic.LoadInt32(&s.compacting) == 1 {
time.Sleep(time.Second)
}
}
s.reifyCond.Broadcast()
s.reifyWorkers.Wait()
s.cancel()
return multierr.Combine(s.markSetEnv.Close(), s.debug.Close())
}
@@ -720,5 +579,5 @@ func (s *SplitStore) checkClosing() error {
func (s *SplitStore) setBaseEpoch(epoch abi.ChainEpoch) error {
s.baseEpoch = epoch
return s.ds.Put(s.ctx, baseEpochKey, epochToBytes(epoch))
return s.ds.Put(baseEpochKey, epochToBytes(epoch))
}
+10 -14
View File
@@ -4,13 +4,13 @@ import (
"fmt"
"os"
"path/filepath"
"sync"
"sync/atomic"
"time"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/types"
)
@@ -67,10 +67,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
}
defer output.Close() //nolint:errcheck
var mx sync.Mutex
write := func(format string, args ...interface{}) {
mx.Lock()
defer mx.Unlock()
_, err := fmt.Fprintf(output, format+"\n", args...)
if err != nil {
log.Warnf("error writing check output: %s", err)
@@ -85,10 +82,9 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
write("compaction index: %d", s.compactionIndex)
write("--")
coldCnt := new(int64)
missingCnt := new(int64)
var coldCnt, missingCnt int64
visitor, err := s.markSetEnv.New("check", 0)
visitor, err := s.markSetEnv.CreateVisitor("check", 0)
if err != nil {
return xerrors.Errorf("error creating visitor: %w", err)
}
@@ -100,7 +96,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
return errStopWalk
}
has, err := s.hot.Has(s.ctx, c)
has, err := s.hot.Has(c)
if err != nil {
return xerrors.Errorf("error checking hotstore: %w", err)
}
@@ -109,16 +105,16 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
return nil
}
has, err = s.cold.Has(s.ctx, c)
has, err = s.cold.Has(c)
if err != nil {
return xerrors.Errorf("error checking coldstore: %w", err)
}
if has {
atomic.AddInt64(coldCnt, 1)
coldCnt++
write("cold object reference: %s", c)
} else {
atomic.AddInt64(missingCnt, 1)
missingCnt++
write("missing object reference: %s", c)
return errStopWalk
}
@@ -132,9 +128,9 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
return err
}
log.Infow("check done", "cold", *coldCnt, "missing", *missingCnt)
log.Infow("check done", "cold", coldCnt, "missing", missingCnt)
write("--")
write("cold: %d missing: %d", *coldCnt, *missingCnt)
write("cold: %d missing: %d", coldCnt, missingCnt)
write("DONE")
return nil
File diff suppressed because it is too large. Load diff
+32 -27
View File
@@ -5,8 +5,7 @@ import (
"errors"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
cid "github.com/ipfs/go-cid"
bstore "github.com/filecoin-project/lotus/blockstore"
)
@@ -21,28 +20,28 @@ func (s *SplitStore) Expose() bstore.Blockstore {
return &exposedSplitStore{s: s}
}
func (es *exposedSplitStore) DeleteBlock(_ context.Context, _ cid.Cid) error {
func (es *exposedSplitStore) DeleteBlock(_ cid.Cid) error {
return errors.New("DeleteBlock: operation not supported")
}
func (es *exposedSplitStore) DeleteMany(_ context.Context, _ []cid.Cid) error {
func (es *exposedSplitStore) DeleteMany(_ []cid.Cid) error {
return errors.New("DeleteMany: operation not supported")
}
func (es *exposedSplitStore) Has(ctx context.Context, c cid.Cid) (bool, error) {
func (es *exposedSplitStore) Has(c cid.Cid) (bool, error) {
if isIdentiyCid(c) {
return true, nil
}
has, err := es.s.hot.Has(ctx, c)
has, err := es.s.hot.Has(c)
if has || err != nil {
return has, err
}
return es.s.cold.Has(ctx, c)
return es.s.cold.Has(c)
}
func (es *exposedSplitStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
func (es *exposedSplitStore) Get(c cid.Cid) (blocks.Block, error) {
if isIdentiyCid(c) {
data, err := decodeIdentityCid(c)
if err != nil {
@@ -52,14 +51,16 @@ func (es *exposedSplitStore) Get(ctx context.Context, c cid.Cid) (blocks.Block,
return blocks.NewBlockWithCid(data, c)
}
blk, err := es.s.hot.Get(ctx, c)
if ipld.IsNotFound(err) {
return es.s.cold.Get(ctx, c)
blk, err := es.s.hot.Get(c)
switch err {
case bstore.ErrNotFound:
return es.s.cold.Get(c)
default:
return blk, err
}
return blk, err
}
func (es *exposedSplitStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
func (es *exposedSplitStore) GetSize(c cid.Cid) (int, error) {
if isIdentiyCid(c) {
data, err := decodeIdentityCid(c)
if err != nil {
@@ -69,19 +70,21 @@ func (es *exposedSplitStore) GetSize(ctx context.Context, c cid.Cid) (int, error
return len(data), nil
}
size, err := es.s.hot.GetSize(ctx, c)
if ipld.IsNotFound(err) {
return es.s.cold.GetSize(ctx, c)
size, err := es.s.hot.GetSize(c)
switch err {
case bstore.ErrNotFound:
return es.s.cold.GetSize(c)
default:
return size, err
}
return size, err
}
func (es *exposedSplitStore) Put(ctx context.Context, blk blocks.Block) error {
return es.s.Put(ctx, blk)
func (es *exposedSplitStore) Put(blk blocks.Block) error {
return es.s.Put(blk)
}
func (es *exposedSplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {
return es.s.PutMany(ctx, blks)
func (es *exposedSplitStore) PutMany(blks []blocks.Block) error {
return es.s.PutMany(blks)
}
func (es *exposedSplitStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
@@ -90,7 +93,7 @@ func (es *exposedSplitStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, e
func (es *exposedSplitStore) HashOnRead(enabled bool) {}
func (es *exposedSplitStore) View(ctx context.Context, c cid.Cid, f func([]byte) error) error {
func (es *exposedSplitStore) View(c cid.Cid, f func([]byte) error) error {
if isIdentiyCid(c) {
data, err := decodeIdentityCid(c)
if err != nil {
@@ -100,10 +103,12 @@ func (es *exposedSplitStore) View(ctx context.Context, c cid.Cid, f func([]byte)
return f(data)
}
err := es.s.hot.View(ctx, c, f)
if ipld.IsNotFound(err) {
return es.s.cold.View(ctx, c, f)
}
err := es.s.hot.View(c, f)
switch err {
case bstore.ErrNotFound:
return es.s.cold.View(c, f)
return err
default:
return err
}
}
-213
View File
@@ -1,213 +0,0 @@
package splitstore
import (
"errors"
"runtime"
"sync/atomic"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
)
var (
errReifyLimit = errors.New("reification limit reached")
ReifyLimit = 16384
)
func (s *SplitStore) reifyColdObject(c cid.Cid) {
if !s.isWarm() {
return
}
if isUnitaryObject(c) {
return
}
s.reifyMx.Lock()
defer s.reifyMx.Unlock()
_, ok := s.reifyInProgress[c]
if ok {
return
}
s.reifyPend[c] = struct{}{}
s.reifyCond.Broadcast()
}
func (s *SplitStore) reifyOrchestrator() {
workers := runtime.NumCPU() / 4
if workers < 2 {
workers = 2
}
workch := make(chan cid.Cid, workers)
defer close(workch)
for i := 0; i < workers; i++ {
s.reifyWorkers.Add(1)
go s.reifyWorker(workch)
}
for {
s.reifyMx.Lock()
for len(s.reifyPend) == 0 && atomic.LoadInt32(&s.closing) == 0 {
s.reifyCond.Wait()
}
if atomic.LoadInt32(&s.closing) != 0 {
s.reifyMx.Unlock()
return
}
reifyPend := s.reifyPend
s.reifyPend = make(map[cid.Cid]struct{})
s.reifyMx.Unlock()
for c := range reifyPend {
select {
case workch <- c:
case <-s.ctx.Done():
return
}
}
}
}
func (s *SplitStore) reifyWorker(workch chan cid.Cid) {
defer s.reifyWorkers.Done()
for c := range workch {
s.doReify(c)
}
}
func (s *SplitStore) doReify(c cid.Cid) {
var toreify, totrack, toforget []cid.Cid
defer func() {
s.reifyMx.Lock()
defer s.reifyMx.Unlock()
for _, c := range toreify {
delete(s.reifyInProgress, c)
}
for _, c := range totrack {
delete(s.reifyInProgress, c)
}
for _, c := range toforget {
delete(s.reifyInProgress, c)
}
}()
s.txnLk.RLock()
defer s.txnLk.RUnlock()
count := 0
err := s.walkObjectIncomplete(c, newTmpVisitor(),
func(c cid.Cid) error {
if isUnitaryObject(c) {
return errStopWalk
}
count++
if count > ReifyLimit {
return errReifyLimit
}
s.reifyMx.Lock()
_, inProgress := s.reifyInProgress[c]
if !inProgress {
s.reifyInProgress[c] = struct{}{}
}
s.reifyMx.Unlock()
if inProgress {
return errStopWalk
}
has, err := s.hot.Has(s.ctx, c)
if err != nil {
return xerrors.Errorf("error checking hotstore: %w", err)
}
if has {
if s.txnMarkSet != nil {
hasMark, err := s.txnMarkSet.Has(c)
if err != nil {
log.Warnf("error checking markset: %s", err)
} else if hasMark {
toforget = append(toforget, c)
return errStopWalk
}
} else {
totrack = append(totrack, c)
return errStopWalk
}
}
toreify = append(toreify, c)
return nil
},
func(missing cid.Cid) error {
log.Warnf("missing reference while reifying %s: %s", c, missing)
return errStopWalk
})
if err != nil {
if xerrors.Is(err, errReifyLimit) {
log.Debug("reification aborted; reify limit reached")
return
}
log.Warnf("error walking cold object for reification (cid: %s): %s", c, err)
return
}
log.Debugf("reifying %d objects rooted at %s", len(toreify), c)
// this should not get too big, maybe some 100s of objects.
batch := make([]blocks.Block, 0, len(toreify))
for _, c := range toreify {
blk, err := s.cold.Get(s.ctx, c)
if err != nil {
log.Warnf("error retrieving cold object for reification (cid: %s): %s", c, err)
continue
}
if err := s.checkClosing(); err != nil {
return
}
batch = append(batch, blk)
}
if len(batch) > 0 {
err = s.hot.PutMany(s.ctx, batch)
if err != nil {
log.Warnf("error reifying cold object (cid: %s): %s", c, err)
return
}
}
if s.txnMarkSet != nil {
if len(toreify) > 0 {
if err := s.txnMarkSet.MarkMany(toreify); err != nil {
log.Warnf("error marking reified objects: %s", err)
}
}
if len(totrack) > 0 {
if err := s.txnMarkSet.MarkMany(totrack); err != nil {
log.Warnf("error marking tracked objects: %s", err)
}
}
} else {
// if txnActive is false these are noops
if len(toreify) > 0 {
s.trackTxnRefMany(toreify)
}
if len(totrack) > 0 {
s.trackTxnRefMany(totrack)
}
}
}
+37 -424
View File
@@ -1,42 +1,34 @@
//stm: #unit
package splitstore
import (
"context"
"errors"
"fmt"
"math/rand"
"sync"
"sync/atomic"
"testing"
"time"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
dssync "github.com/ipfs/go-datastore/sync"
ipld "github.com/ipfs/go-ipld-format"
logging "github.com/ipfs/go-log/v2"
mh "github.com/multiformats/go-multihash"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/mock"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
datastore "github.com/ipfs/go-datastore"
dssync "github.com/ipfs/go-datastore/sync"
logging "github.com/ipfs/go-log/v2"
)
func init() {
CompactionThreshold = 5
CompactionBoundary = 2
WarmupBoundary = 0
SyncWaitTime = time.Millisecond
logging.SetLogLevel("splitstore", "DEBUG")
}
func testSplitStore(t *testing.T, cfg *Config) {
ctx := context.Background()
chain := &mockChain{t: t}
// the myriads of stores
@@ -46,7 +38,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
// this is necessary to avoid the garbage mock puts in the blocks
garbage := blocks.NewBlock([]byte{1, 2, 3})
err := cold.Put(ctx, garbage)
err := cold.Put(garbage)
if err != nil {
t.Fatal(err)
}
@@ -67,29 +59,27 @@ func testSplitStore(t *testing.T, cfg *Config) {
t.Fatal(err)
}
err = cold.Put(ctx, blk)
err = cold.Put(blk)
if err != nil {
t.Fatal(err)
}
// create a garbage block that is protected with a rgistered protector
protected := blocks.NewBlock([]byte("protected!"))
err = hot.Put(ctx, protected)
err = hot.Put(protected)
if err != nil {
t.Fatal(err)
}
// and another one that is not protected
unprotected := blocks.NewBlock([]byte("unprotected!"))
err = hot.Put(ctx, unprotected)
err = hot.Put(unprotected)
if err != nil {
t.Fatal(err)
}
path := t.TempDir()
// open the splitstore
ss, err := Open(path, ds, hot, cold, cfg)
ss, err := Open("", ds, hot, cold, cfg)
if err != nil {
t.Fatal(err)
}
@@ -100,7 +90,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
return protect(protected.Cid())
})
err = ss.Start(chain, nil)
err = ss.Start(chain)
if err != nil {
t.Fatal(err)
}
@@ -118,11 +108,11 @@ func testSplitStore(t *testing.T, cfg *Config) {
if err != nil {
t.Fatal(err)
}
err = ss.Put(ctx, stateRoot)
err = ss.Put(stateRoot)
if err != nil {
t.Fatal(err)
}
err = ss.Put(ctx, sblk)
err = ss.Put(sblk)
if err != nil {
t.Fatal(err)
}
@@ -133,10 +123,6 @@ func testSplitStore(t *testing.T, cfg *Config) {
}
waitForCompaction := func() {
ss.txnSyncMx.Lock()
ss.txnSync = true
ss.txnSyncCond.Broadcast()
ss.txnSyncMx.Unlock()
for atomic.LoadInt32(&ss.compacting) == 1 {
time.Sleep(100 * time.Millisecond)
}
@@ -189,7 +175,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
}
// ensure our protected block is still there
has, err := hot.Has(ctx, protected.Cid())
has, err := hot.Has(protected.Cid())
if err != nil {
t.Fatal(err)
}
@@ -199,7 +185,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
}
// ensure our unprotected block is in the coldstore now
has, err = hot.Has(ctx, unprotected.Cid())
has, err = hot.Has(unprotected.Cid())
if err != nil {
t.Fatal(err)
}
@@ -208,7 +194,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
t.Fatal("unprotected block is still in hotstore")
}
has, err = cold.Has(ctx, unprotected.Cid())
has, err = cold.Has(unprotected.Cid())
if err != nil {
t.Fatal(err)
}
@@ -222,16 +208,10 @@ func testSplitStore(t *testing.T, cfg *Config) {
}
func TestSplitStoreCompaction(t *testing.T) {
//stm: @SPLITSTORE_SPLITSTORE_OPEN_001, @SPLITSTORE_SPLITSTORE_CLOSE_001
//stm: @SPLITSTORE_SPLITSTORE_PUT_001, @SPLITSTORE_SPLITSTORE_ADD_PROTECTOR_001
//stm: @SPLITSTORE_SPLITSTORE_CLOSE_001
testSplitStore(t, &Config{MarkSetType: "map"})
}
func TestSplitStoreCompactionWithBadger(t *testing.T) {
//stm: @SPLITSTORE_SPLITSTORE_OPEN_001, @SPLITSTORE_SPLITSTORE_CLOSE_001
//stm: @SPLITSTORE_SPLITSTORE_PUT_001, @SPLITSTORE_SPLITSTORE_ADD_PROTECTOR_001
//stm: @SPLITSTORE_SPLITSTORE_CLOSE_001
bs := badgerMarkSetBatchSize
badgerMarkSetBatchSize = 1
t.Cleanup(func() {
@@ -240,365 +220,6 @@ func TestSplitStoreCompactionWithBadger(t *testing.T) {
testSplitStore(t, &Config{MarkSetType: "badger"})
}
func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
//stm: @SPLITSTORE_SPLITSTORE_OPEN_001, @SPLITSTORE_SPLITSTORE_CLOSE_001
//stm: @SPLITSTORE_SPLITSTORE_PUT_001, @SPLITSTORE_SPLITSTORE_ADD_PROTECTOR_001
//stm: @SPLITSTORE_SPLITSTORE_CLOSE_001
ctx := context.Background()
chain := &mockChain{t: t}
// the myriads of stores
ds := dssync.MutexWrap(datastore.NewMapDatastore())
hot := newMockStore()
cold := newMockStore()
// this is necessary to avoid the garbage mock puts in the blocks
garbage := blocks.NewBlock([]byte{1, 2, 3})
err := cold.Put(ctx, garbage)
if err != nil {
t.Fatal(err)
}
// genesis
genBlock := mock.MkBlock(nil, 0, 0)
genBlock.Messages = garbage.Cid()
genBlock.ParentMessageReceipts = garbage.Cid()
genBlock.ParentStateRoot = garbage.Cid()
genBlock.Timestamp = uint64(time.Now().Unix())
genTs := mock.TipSet(genBlock)
chain.push(genTs)
// put the genesis block to cold store
blk, err := genBlock.ToStorageBlock()
if err != nil {
t.Fatal(err)
}
err = cold.Put(ctx, blk)
if err != nil {
t.Fatal(err)
}
path := t.TempDir()
// open the splitstore
ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
if err != nil {
t.Fatal(err)
}
defer ss.Close() //nolint
// create an upgrade schedule that will suppress compaction during the test
upgradeBoundary = 0
upgrade := stmgr.Upgrade{
Height: 10,
PreMigrations: []stmgr.PreMigration{{StartWithin: 10}},
}
err = ss.Start(chain, []stmgr.Upgrade{upgrade})
if err != nil {
t.Fatal(err)
}
mkBlock := func(curTs *types.TipSet, i int, stateRoot blocks.Block) *types.TipSet {
blk := mock.MkBlock(curTs, uint64(i), uint64(i))
blk.Messages = garbage.Cid()
blk.ParentMessageReceipts = garbage.Cid()
blk.ParentStateRoot = stateRoot.Cid()
blk.Timestamp = uint64(time.Now().Unix())
sblk, err := blk.ToStorageBlock()
if err != nil {
t.Fatal(err)
}
err = ss.Put(ctx, stateRoot)
if err != nil {
t.Fatal(err)
}
err = ss.Put(ctx, sblk)
if err != nil {
t.Fatal(err)
}
ts := mock.TipSet(blk)
chain.push(ts)
return ts
}
waitForCompaction := func() {
ss.txnSyncMx.Lock()
ss.txnSync = true
ss.txnSyncCond.Broadcast()
ss.txnSyncMx.Unlock()
for atomic.LoadInt32(&ss.compacting) == 1 {
time.Sleep(100 * time.Millisecond)
}
}
curTs := genTs
for i := 1; i < 10; i++ {
stateRoot := blocks.NewBlock([]byte{byte(i), 3, 3, 7})
curTs = mkBlock(curTs, i, stateRoot)
waitForCompaction()
}
countBlocks := func(bs blockstore.Blockstore) int {
count := 0
_ = bs.(blockstore.BlockstoreIterator).ForEachKey(func(_ cid.Cid) error {
count++
return nil
})
return count
}
// we should not have compacted due to suppression and everything should still be hot
hotCnt := countBlocks(hot)
coldCnt := countBlocks(cold)
if hotCnt != 20 {
t.Errorf("expected %d blocks, but got %d", 20, hotCnt)
}
if coldCnt != 2 {
t.Errorf("expected %d blocks, but got %d", 2, coldCnt)
}
// put some more blocks, now we should compact
for i := 10; i < 20; i++ {
stateRoot := blocks.NewBlock([]byte{byte(i), 3, 3, 7})
curTs = mkBlock(curTs, i, stateRoot)
waitForCompaction()
}
hotCnt = countBlocks(hot)
coldCnt = countBlocks(cold)
if hotCnt != 24 {
t.Errorf("expected %d blocks, but got %d", 24, hotCnt)
}
if coldCnt != 18 {
t.Errorf("expected %d blocks, but got %d", 18, coldCnt)
}
}
func testSplitStoreReification(t *testing.T, f func(context.Context, blockstore.Blockstore, cid.Cid) error) {
ds := dssync.MutexWrap(datastore.NewMapDatastore())
hot := newMockStore()
cold := newMockStore()
mkRandomBlock := func() blocks.Block {
data := make([]byte, 128)
_, err := rand.Read(data)
if err != nil {
t.Fatal(err)
}
return blocks.NewBlock(data)
}
block1 := mkRandomBlock()
block2 := mkRandomBlock()
block3 := mkRandomBlock()
hdr := mock.MkBlock(nil, 0, 0)
hdr.Messages = block1.Cid()
hdr.ParentMessageReceipts = block2.Cid()
hdr.ParentStateRoot = block3.Cid()
block4, err := hdr.ToStorageBlock()
if err != nil {
t.Fatal(err)
}
allBlocks := []blocks.Block{block1, block2, block3, block4}
for _, blk := range allBlocks {
err := cold.Put(context.Background(), blk)
if err != nil {
t.Fatal(err)
}
}
path := t.TempDir()
ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
if err != nil {
t.Fatal(err)
}
defer ss.Close() //nolint
ss.warmupEpoch = 1
go ss.reifyOrchestrator()
waitForReification := func() {
for {
ss.reifyMx.Lock()
ready := len(ss.reifyPend) == 0 && len(ss.reifyInProgress) == 0
ss.reifyMx.Unlock()
if ready {
return
}
time.Sleep(time.Millisecond)
}
}
// first access using the standard view
err = f(context.Background(), ss, block4.Cid())
if err != nil {
t.Fatal(err)
}
// nothing should be reified
waitForReification()
for _, blk := range allBlocks {
has, err := hot.Has(context.Background(), blk.Cid())
if err != nil {
t.Fatal(err)
}
if has {
t.Fatal("block unexpectedly reified")
}
}
// now make the hot/reifying view and ensure access reifies
err = f(blockstore.WithHotView(context.Background()), ss, block4.Cid())
if err != nil {
t.Fatal(err)
}
// everything should be reified
waitForReification()
for i, blk := range allBlocks {
has, err := hot.Has(context.Background(), blk.Cid())
if err != nil {
t.Fatal(err)
}
if !has {
t.Fatalf("block%d was not reified", i+1)
}
}
}
func testSplitStoreReificationLimit(t *testing.T, f func(context.Context, blockstore.Blockstore, cid.Cid) error) {
ds := dssync.MutexWrap(datastore.NewMapDatastore())
hot := newMockStore()
cold := newMockStore()
mkRandomBlock := func() blocks.Block {
data := make([]byte, 128)
_, err := rand.Read(data)
if err != nil {
t.Fatal(err)
}
return blocks.NewBlock(data)
}
block1 := mkRandomBlock()
block2 := mkRandomBlock()
block3 := mkRandomBlock()
hdr := mock.MkBlock(nil, 0, 0)
hdr.Messages = block1.Cid()
hdr.ParentMessageReceipts = block2.Cid()
hdr.ParentStateRoot = block3.Cid()
block4, err := hdr.ToStorageBlock()
if err != nil {
t.Fatal(err)
}
allBlocks := []blocks.Block{block1, block2, block3, block4}
for _, blk := range allBlocks {
err := cold.Put(context.Background(), blk)
if err != nil {
t.Fatal(err)
}
}
path := t.TempDir()
ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
if err != nil {
t.Fatal(err)
}
defer ss.Close() //nolint
ss.warmupEpoch = 1
go ss.reifyOrchestrator()
waitForReification := func() {
for {
ss.reifyMx.Lock()
ready := len(ss.reifyPend) == 0 && len(ss.reifyInProgress) == 0
ss.reifyMx.Unlock()
if ready {
return
}
time.Sleep(time.Millisecond)
}
}
// do a hot access -- nothing should be reified as the limit should be exceeded
oldReifyLimit := ReifyLimit
ReifyLimit = 2
t.Cleanup(func() {
ReifyLimit = oldReifyLimit
})
err = f(blockstore.WithHotView(context.Background()), ss, block4.Cid())
if err != nil {
t.Fatal(err)
}
waitForReification()
for _, blk := range allBlocks {
has, err := hot.Has(context.Background(), blk.Cid())
if err != nil {
t.Fatal(err)
}
if has {
t.Fatal("block unexpectedly reified")
}
}
}
func TestSplitStoreReification(t *testing.T) {
t.Log("test reification with Has")
testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
_, err := s.Has(ctx, c)
return err
})
t.Log("test reification with Get")
testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
_, err := s.Get(ctx, c)
return err
})
t.Log("test reification with GetSize")
testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
_, err := s.GetSize(ctx, c)
return err
})
t.Log("test reification with View")
testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
return s.View(ctx, c, func(_ []byte) error { return nil })
})
t.Log("test reification limit")
testSplitStoreReificationLimit(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
_, err := s.Has(ctx, c)
return err
})
}
type mockChain struct {
t testing.TB
@@ -668,43 +289,35 @@ func (c *mockChain) SubscribeHeadChanges(change func(revert []*types.TipSet, app
type mockStore struct {
mx sync.Mutex
set map[string]blocks.Block
set map[cid.Cid]blocks.Block
}
func newMockStore() *mockStore {
return &mockStore{set: make(map[string]blocks.Block)}
return &mockStore{set: make(map[cid.Cid]blocks.Block)}
}
func (b *mockStore) keyOf(c cid.Cid) string {
return string(c.Hash())
}
func (b *mockStore) cidOf(k string) cid.Cid {
return cid.NewCidV1(cid.Raw, mh.Multihash([]byte(k)))
}
func (b *mockStore) Has(_ context.Context, cid cid.Cid) (bool, error) {
func (b *mockStore) Has(cid cid.Cid) (bool, error) {
b.mx.Lock()
defer b.mx.Unlock()
_, ok := b.set[b.keyOf(cid)]
_, ok := b.set[cid]
return ok, nil
}
func (b *mockStore) HashOnRead(hor bool) {}
func (b *mockStore) Get(_ context.Context, cid cid.Cid) (blocks.Block, error) {
func (b *mockStore) Get(cid cid.Cid) (blocks.Block, error) {
b.mx.Lock()
defer b.mx.Unlock()
blk, ok := b.set[b.keyOf(cid)]
blk, ok := b.set[cid]
if !ok {
return nil, ipld.ErrNotFound{Cid: cid}
return nil, blockstore.ErrNotFound
}
return blk, nil
}
func (b *mockStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
blk, err := b.Get(ctx, cid)
func (b *mockStore) GetSize(cid cid.Cid) (int, error) {
blk, err := b.Get(cid)
if err != nil {
return 0, err
}
@@ -712,46 +325,46 @@ func (b *mockStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
return len(blk.RawData()), nil
}
func (b *mockStore) View(ctx context.Context, cid cid.Cid, f func([]byte) error) error {
blk, err := b.Get(ctx, cid)
func (b *mockStore) View(cid cid.Cid, f func([]byte) error) error {
blk, err := b.Get(cid)
if err != nil {
return err
}
return f(blk.RawData())
}
func (b *mockStore) Put(_ context.Context, blk blocks.Block) error {
func (b *mockStore) Put(blk blocks.Block) error {
b.mx.Lock()
defer b.mx.Unlock()
b.set[b.keyOf(blk.Cid())] = blk
b.set[blk.Cid()] = blk
return nil
}
func (b *mockStore) PutMany(_ context.Context, blks []blocks.Block) error {
func (b *mockStore) PutMany(blks []blocks.Block) error {
b.mx.Lock()
defer b.mx.Unlock()
for _, blk := range blks {
b.set[b.keyOf(blk.Cid())] = blk
b.set[blk.Cid()] = blk
}
return nil
}
func (b *mockStore) DeleteBlock(_ context.Context, cid cid.Cid) error {
func (b *mockStore) DeleteBlock(cid cid.Cid) error {
b.mx.Lock()
defer b.mx.Unlock()
delete(b.set, b.keyOf(cid))
delete(b.set, cid)
return nil
}
func (b *mockStore) DeleteMany(_ context.Context, cids []cid.Cid) error {
func (b *mockStore) DeleteMany(cids []cid.Cid) error {
b.mx.Lock()
defer b.mx.Unlock()
for _, c := range cids {
delete(b.set, b.keyOf(c))
delete(b.set, c)
}
return nil
}
@@ -765,7 +378,7 @@ func (b *mockStore) ForEachKey(f func(cid.Cid) error) error {
defer b.mx.Unlock()
for c := range b.set {
err := f(b.cidOf(c))
err := f(c)
if err != nil {
return err
}
+3 -2
View File
@@ -3,10 +3,11 @@ package splitstore
import (
"encoding/binary"
"github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
"github.com/filecoin-project/go-state-types/abi"
)
+21 -26
View File
@@ -1,17 +1,16 @@
package splitstore
import (
"sync"
"sync/atomic"
"time"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-state-types/abi"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
"github.com/filecoin-project/go-state-types/abi"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
)
@@ -56,13 +55,12 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
if WarmupBoundary < epoch {
boundaryEpoch = epoch - WarmupBoundary
}
var mx sync.Mutex
batchHot := make([]blocks.Block, 0, batchSize)
count := new(int64)
xcount := new(int64)
missing := new(int64)
count := int64(0)
xcount := int64(0)
missing := int64(0)
visitor, err := s.markSetEnv.New("warmup", 0)
visitor, err := s.markSetEnv.CreateVisitor("warmup", 0)
if err != nil {
return xerrors.Errorf("error creating visitor: %w", err)
}
@@ -75,9 +73,9 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
return errStopWalk
}
atomic.AddInt64(count, 1)
count++
has, err := s.hot.Has(s.ctx, c)
has, err := s.hot.Has(c)
if err != nil {
return err
}
@@ -86,28 +84,25 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
return nil
}
blk, err := s.cold.Get(s.ctx, c)
blk, err := s.cold.Get(c)
if err != nil {
if ipld.IsNotFound(err) {
atomic.AddInt64(missing, 1)
if err == bstore.ErrNotFound {
missing++
return errStopWalk
}
return err
}
atomic.AddInt64(xcount, 1)
xcount++
mx.Lock()
batchHot = append(batchHot, blk)
if len(batchHot) == batchSize {
err = s.hot.PutMany(s.ctx, batchHot)
err = s.hot.PutMany(batchHot)
if err != nil {
mx.Unlock()
return err
}
batchHot = batchHot[:0]
}
mx.Unlock()
return nil
})
@@ -117,22 +112,22 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
}
if len(batchHot) > 0 {
err = s.hot.PutMany(s.ctx, batchHot)
err = s.hot.PutMany(batchHot)
if err != nil {
return err
}
}
log.Infow("warmup stats", "visited", *count, "warm", *xcount, "missing", *missing)
log.Infow("warmup stats", "visited", count, "warm", xcount, "missing", missing)
s.markSetSize = *count + *count>>2 // overestimate a bit
err = s.ds.Put(s.ctx, markSetSizeKey, int64ToBytes(s.markSetSize))
s.markSetSize = count + count>>2 // overestimate a bit
err = s.ds.Put(markSetSizeKey, int64ToBytes(s.markSetSize))
if err != nil {
log.Warnf("error saving mark set size: %s", err)
}
// save the warmup epoch
err = s.ds.Put(s.ctx, warmupEpochKey, epochToBytes(epoch))
err = s.ds.Put(warmupEpochKey, epochToBytes(epoch))
if err != nil {
return xerrors.Errorf("error saving warm up epoch: %w", err)
}
@@ -141,7 +136,7 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
s.mx.Unlock()
// also save the compactionIndex, as this is used as an indicator of warmup for upgraded nodes
err = s.ds.Put(s.ctx, compactionIndexKey, int64ToBytes(s.compactionIndex))
err = s.ds.Put(compactionIndexKey, int64ToBytes(s.compactionIndex))
if err != nil {
return xerrors.Errorf("error saving compaction index: %w", err)
}
+6 -34
View File
@@ -1,9 +1,7 @@
package splitstore
import (
"sync"
"github.com/ipfs/go-cid"
cid "github.com/ipfs/go-cid"
)
// ObjectVisitor is an interface for deduplicating objects during walks
@@ -19,42 +17,16 @@ func (v *noopVisitor) Visit(_ cid.Cid) (bool, error) {
return true, nil
}
type tmpVisitor struct {
type cidSetVisitor struct {
set *cid.Set
}
var _ ObjectVisitor = (*tmpVisitor)(nil)
func (v *tmpVisitor) Visit(c cid.Cid) (bool, error) {
if isUnitaryObject(c) {
return false, nil
}
var _ ObjectVisitor = (*cidSetVisitor)(nil)
func (v *cidSetVisitor) Visit(c cid.Cid) (bool, error) {
return v.set.Visit(c), nil
}
func newTmpVisitor() ObjectVisitor {
return &tmpVisitor{set: cid.NewSet()}
}
type concurrentVisitor struct {
mx sync.Mutex
set *cid.Set
}
var _ ObjectVisitor = (*concurrentVisitor)(nil)
func newConcurrentVisitor() *concurrentVisitor {
return &concurrentVisitor{set: cid.NewSet()}
}
func (v *concurrentVisitor) Visit(c cid.Cid) (bool, error) {
if isUnitaryObject(c) {
return false, nil
}
v.mx.Lock()
defer v.mx.Unlock()
return v.set.Visit(c), nil
func tmpVisitor() ObjectVisitor {
return &cidSetVisitor{set: cid.NewSet()}
}
+16 -16
View File
@@ -20,53 +20,53 @@ type SyncBlockstore struct {
bs MemBlockstore // specifically use a memStore to save indirection overhead.
}
func (m *SyncBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
func (m *SyncBlockstore) DeleteBlock(k cid.Cid) error {
m.mu.Lock()
defer m.mu.Unlock()
return m.bs.DeleteBlock(ctx, k)
return m.bs.DeleteBlock(k)
}
func (m *SyncBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error {
func (m *SyncBlockstore) DeleteMany(ks []cid.Cid) error {
m.mu.Lock()
defer m.mu.Unlock()
return m.bs.DeleteMany(ctx, ks)
return m.bs.DeleteMany(ks)
}
func (m *SyncBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error) {
func (m *SyncBlockstore) Has(k cid.Cid) (bool, error) {
m.mu.RLock()
defer m.mu.RUnlock()
return m.bs.Has(ctx, k)
return m.bs.Has(k)
}
func (m *SyncBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error {
func (m *SyncBlockstore) View(k cid.Cid, callback func([]byte) error) error {
m.mu.RLock()
defer m.mu.RUnlock()
return m.bs.View(ctx, k, callback)
return m.bs.View(k, callback)
}
func (m *SyncBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) {
func (m *SyncBlockstore) Get(k cid.Cid) (blocks.Block, error) {
m.mu.RLock()
defer m.mu.RUnlock()
return m.bs.Get(ctx, k)
return m.bs.Get(k)
}
func (m *SyncBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
func (m *SyncBlockstore) GetSize(k cid.Cid) (int, error) {
m.mu.RLock()
defer m.mu.RUnlock()
return m.bs.GetSize(ctx, k)
return m.bs.GetSize(k)
}
func (m *SyncBlockstore) Put(ctx context.Context, b blocks.Block) error {
func (m *SyncBlockstore) Put(b blocks.Block) error {
m.mu.Lock()
defer m.mu.Unlock()
return m.bs.Put(ctx, b)
return m.bs.Put(b)
}
func (m *SyncBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
func (m *SyncBlockstore) PutMany(bs []blocks.Block) error {
m.mu.Lock()
defer m.mu.Unlock()
return m.bs.PutMany(ctx, bs)
return m.bs.PutMany(bs)
}
func (m *SyncBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+24 -29
View File
@@ -8,7 +8,6 @@ import (
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
"github.com/raulk/clock"
"go.uber.org/multierr"
)
@@ -48,12 +47,8 @@ func (t *TimedCacheBlockstore) Start(_ context.Context) error {
return fmt.Errorf("already started")
}
t.closeCh = make(chan struct{})
// Create this timer before starting the goroutine. Otherwise, creating the timer will race
// with adding time to the mock clock, and we could add time _first_, then stall waiting for
// a timer that'll never fire.
ticker := t.clock.Ticker(t.interval)
go func() {
ticker := t.clock.Ticker(t.interval)
defer ticker.Stop()
for {
select {
@@ -93,28 +88,28 @@ func (t *TimedCacheBlockstore) rotate() {
t.mu.Unlock()
}
func (t *TimedCacheBlockstore) Put(ctx context.Context, b blocks.Block) error {
func (t *TimedCacheBlockstore) Put(b blocks.Block) error {
// Don't check the inactive set here. We want to keep this block for at
// least one interval.
t.mu.Lock()
defer t.mu.Unlock()
return t.active.Put(ctx, b)
return t.active.Put(b)
}
func (t *TimedCacheBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
func (t *TimedCacheBlockstore) PutMany(bs []blocks.Block) error {
t.mu.Lock()
defer t.mu.Unlock()
return t.active.PutMany(ctx, bs)
return t.active.PutMany(bs)
}
func (t *TimedCacheBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error {
func (t *TimedCacheBlockstore) View(k cid.Cid, callback func([]byte) error) error {
// The underlying blockstore is always a "mem" blockstore so there's no difference,
// from a performance perspective, between view & get. So we call Get to avoid
// calling an arbitrary callback while holding a lock.
t.mu.RLock()
block, err := t.active.Get(ctx, k)
if ipld.IsNotFound(err) {
block, err = t.inactive.Get(ctx, k)
block, err := t.active.Get(k)
if err == ErrNotFound {
block, err = t.inactive.Get(k)
}
t.mu.RUnlock()
@@ -124,51 +119,51 @@ func (t *TimedCacheBlockstore) View(ctx context.Context, k cid.Cid, callback fun
return callback(block.RawData())
}
func (t *TimedCacheBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) {
func (t *TimedCacheBlockstore) Get(k cid.Cid) (blocks.Block, error) {
t.mu.RLock()
defer t.mu.RUnlock()
b, err := t.active.Get(ctx, k)
if ipld.IsNotFound(err) {
b, err = t.inactive.Get(ctx, k)
b, err := t.active.Get(k)
if err == ErrNotFound {
b, err = t.inactive.Get(k)
}
return b, err
}
func (t *TimedCacheBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
func (t *TimedCacheBlockstore) GetSize(k cid.Cid) (int, error) {
t.mu.RLock()
defer t.mu.RUnlock()
size, err := t.active.GetSize(ctx, k)
if ipld.IsNotFound(err) {
size, err = t.inactive.GetSize(ctx, k)
size, err := t.active.GetSize(k)
if err == ErrNotFound {
size, err = t.inactive.GetSize(k)
}
return size, err
}
func (t *TimedCacheBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error) {
func (t *TimedCacheBlockstore) Has(k cid.Cid) (bool, error) {
t.mu.RLock()
defer t.mu.RUnlock()
if has, err := t.active.Has(ctx, k); err != nil {
if has, err := t.active.Has(k); err != nil {
return false, err
} else if has {
return true, nil
}
return t.inactive.Has(ctx, k)
return t.inactive.Has(k)
}
func (t *TimedCacheBlockstore) HashOnRead(_ bool) {
// no-op
}
func (t *TimedCacheBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
func (t *TimedCacheBlockstore) DeleteBlock(k cid.Cid) error {
t.mu.Lock()
defer t.mu.Unlock()
return multierr.Combine(t.active.DeleteBlock(ctx, k), t.inactive.DeleteBlock(ctx, k))
return multierr.Combine(t.active.DeleteBlock(k), t.inactive.DeleteBlock(k))
}
func (t *TimedCacheBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error {
func (t *TimedCacheBlockstore) DeleteMany(ks []cid.Cid) error {
t.mu.Lock()
defer t.mu.Unlock()
return multierr.Combine(t.active.DeleteMany(ctx, ks), t.inactive.DeleteMany(ctx, ks))
return multierr.Combine(t.active.DeleteMany(ks), t.inactive.DeleteMany(ks))
}
func (t *TimedCacheBlockstore) AllKeysChan(_ context.Context) (<-chan cid.Cid, error) {
+14 -19
View File
@@ -1,4 +1,3 @@
//stm: #unit
package blockstore
import (
@@ -6,24 +5,20 @@ import (
"testing"
"time"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/raulk/clock"
"github.com/stretchr/testify/require"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
)
func TestTimedCacheBlockstoreSimple(t *testing.T) {
//stm: @SPLITSTORE_TIMED_BLOCKSTORE_START_001
//stm: @SPLITSTORE_TIMED_BLOCKSTORE_PUT_001, @SPLITSTORE_TIMED_BLOCKSTORE_HAS_001, @SPLITSTORE_TIMED_BLOCKSTORE_GET_001
//stm: @SPLITSTORE_TIMED_BLOCKSTORE_ALL_KEYS_CHAN_001
tc := NewTimedCacheBlockstore(10 * time.Millisecond)
mClock := clock.NewMock()
mClock.Set(time.Now())
tc.clock = mClock
tc.doneRotatingCh = make(chan struct{})
ctx := context.Background()
_ = tc.Start(context.Background())
mClock.Add(1) // IDK why it is needed but it makes it work
@@ -32,18 +27,18 @@ func TestTimedCacheBlockstoreSimple(t *testing.T) {
}()
b1 := blocks.NewBlock([]byte("foo"))
require.NoError(t, tc.Put(ctx, b1))
require.NoError(t, tc.Put(b1))
b2 := blocks.NewBlock([]byte("bar"))
require.NoError(t, tc.Put(ctx, b2))
require.NoError(t, tc.Put(b2))
b3 := blocks.NewBlock([]byte("baz"))
b1out, err := tc.Get(ctx, b1.Cid())
b1out, err := tc.Get(b1.Cid())
require.NoError(t, err)
require.Equal(t, b1.RawData(), b1out.RawData())
has, err := tc.Has(ctx, b1.Cid())
has, err := tc.Has(b1.Cid())
require.NoError(t, err)
require.True(t, has)
@@ -51,17 +46,17 @@ func TestTimedCacheBlockstoreSimple(t *testing.T) {
<-tc.doneRotatingCh
// We should still have everything.
has, err = tc.Has(ctx, b1.Cid())
has, err = tc.Has(b1.Cid())
require.NoError(t, err)
require.True(t, has)
has, err = tc.Has(ctx, b2.Cid())
has, err = tc.Has(b2.Cid())
require.NoError(t, err)
require.True(t, has)
// extend b2, add b3.
require.NoError(t, tc.Put(ctx, b2))
require.NoError(t, tc.Put(ctx, b3))
require.NoError(t, tc.Put(b2))
require.NoError(t, tc.Put(b3))
// all keys once.
allKeys, err := tc.AllKeysChan(context.Background())
@@ -76,15 +71,15 @@ func TestTimedCacheBlockstoreSimple(t *testing.T) {
<-tc.doneRotatingCh
// should still have b2, and b3, but not b1
has, err = tc.Has(ctx, b1.Cid())
has, err = tc.Has(b1.Cid())
require.NoError(t, err)
require.False(t, has)
has, err = tc.Has(ctx, b2.Cid())
has, err = tc.Has(b2.Cid())
require.NoError(t, err)
require.True(t, has)
has, err = tc.Has(ctx, b3.Cid())
has, err = tc.Has(b3.Cid())
require.NoError(t, err)
require.True(t, has)
}
+17 -18
View File
@@ -5,7 +5,6 @@ import (
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
)
type unionBlockstore []Blockstore
@@ -14,78 +13,78 @@ type unionBlockstore []Blockstore
//
// * Reads return from the first blockstore that has the value, querying in the
// supplied order.
// * Writes (puts and deletes) are broadcast to all stores.
// * Writes (puts and deltes) are broadcast to all stores.
//
func Union(stores ...Blockstore) Blockstore {
return unionBlockstore(stores)
}
func (m unionBlockstore) Has(ctx context.Context, cid cid.Cid) (has bool, err error) {
func (m unionBlockstore) Has(cid cid.Cid) (has bool, err error) {
for _, bs := range m {
if has, err = bs.Has(ctx, cid); has || err != nil {
if has, err = bs.Has(cid); has || err != nil {
break
}
}
return has, err
}
func (m unionBlockstore) Get(ctx context.Context, cid cid.Cid) (blk blocks.Block, err error) {
func (m unionBlockstore) Get(cid cid.Cid) (blk blocks.Block, err error) {
for _, bs := range m {
if blk, err = bs.Get(ctx, cid); err == nil || !ipld.IsNotFound(err) {
if blk, err = bs.Get(cid); err == nil || err != ErrNotFound {
break
}
}
return blk, err
}
func (m unionBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) (err error) {
func (m unionBlockstore) View(cid cid.Cid, callback func([]byte) error) (err error) {
for _, bs := range m {
if err = bs.View(ctx, cid, callback); err == nil || !ipld.IsNotFound(err) {
if err = bs.View(cid, callback); err == nil || err != ErrNotFound {
break
}
}
return err
}
func (m unionBlockstore) GetSize(ctx context.Context, cid cid.Cid) (size int, err error) {
func (m unionBlockstore) GetSize(cid cid.Cid) (size int, err error) {
for _, bs := range m {
if size, err = bs.GetSize(ctx, cid); err == nil || !ipld.IsNotFound(err) {
if size, err = bs.GetSize(cid); err == nil || err != ErrNotFound {
break
}
}
return size, err
}
func (m unionBlockstore) Put(ctx context.Context, block blocks.Block) (err error) {
func (m unionBlockstore) Put(block blocks.Block) (err error) {
for _, bs := range m {
if err = bs.Put(ctx, block); err != nil {
if err = bs.Put(block); err != nil {
break
}
}
return err
}
func (m unionBlockstore) PutMany(ctx context.Context, blks []blocks.Block) (err error) {
func (m unionBlockstore) PutMany(blks []blocks.Block) (err error) {
for _, bs := range m {
if err = bs.PutMany(ctx, blks); err != nil {
if err = bs.PutMany(blks); err != nil {
break
}
}
return err
}
func (m unionBlockstore) DeleteBlock(ctx context.Context, cid cid.Cid) (err error) {
func (m unionBlockstore) DeleteBlock(cid cid.Cid) (err error) {
for _, bs := range m {
if err = bs.DeleteBlock(ctx, cid); err != nil {
if err = bs.DeleteBlock(cid); err != nil {
break
}
}
return err
}
func (m unionBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) (err error) {
func (m unionBlockstore) DeleteMany(cids []cid.Cid) (err error) {
for _, bs := range m {
if err = bs.DeleteMany(ctx, cids); err != nil {
if err = bs.DeleteMany(cids); err != nil {
break
}
}
+19 -26
View File
@@ -1,4 +1,3 @@
//stm: #unit
package blockstore
import (
@@ -16,85 +15,79 @@ var (
)
func TestUnionBlockstore_Get(t *testing.T) {
//stm: @SPLITSTORE_UNION_BLOCKSTORE_GET_001
ctx := context.Background()
m1 := NewMemory()
m2 := NewMemory()
_ = m1.Put(ctx, b1)
_ = m2.Put(ctx, b2)
_ = m1.Put(b1)
_ = m2.Put(b2)
u := Union(m1, m2)
v1, err := u.Get(ctx, b1.Cid())
v1, err := u.Get(b1.Cid())
require.NoError(t, err)
require.Equal(t, b1.RawData(), v1.RawData())
v2, err := u.Get(ctx, b2.Cid())
v2, err := u.Get(b2.Cid())
require.NoError(t, err)
require.Equal(t, b2.RawData(), v2.RawData())
}
func TestUnionBlockstore_Put_PutMany_Delete_AllKeysChan(t *testing.T) {
//stm: @SPLITSTORE_UNION_BLOCKSTORE_PUT_001, @SPLITSTORE_UNION_BLOCKSTORE_HAS_001
//stm: @SPLITSTORE_UNION_BLOCKSTORE_PUT_MANY_001, @SPLITSTORE_UNION_BLOCKSTORE_DELETE_001
//stm: @SPLITSTORE_UNION_BLOCKSTORE_ALL_KEYS_CHAN_001
ctx := context.Background()
m1 := NewMemory()
m2 := NewMemory()
u := Union(m1, m2)
err := u.Put(ctx, b0)
err := u.Put(b0)
require.NoError(t, err)
var has bool
// write was broadcasted to all stores.
has, _ = m1.Has(ctx, b0.Cid())
has, _ = m1.Has(b0.Cid())
require.True(t, has)
has, _ = m2.Has(ctx, b0.Cid())
has, _ = m2.Has(b0.Cid())
require.True(t, has)
has, _ = u.Has(ctx, b0.Cid())
has, _ = u.Has(b0.Cid())
require.True(t, has)
// put many.
err = u.PutMany(ctx, []blocks.Block{b1, b2})
err = u.PutMany([]blocks.Block{b1, b2})
require.NoError(t, err)
// write was broadcasted to all stores.
has, _ = m1.Has(ctx, b1.Cid())
has, _ = m1.Has(b1.Cid())
require.True(t, has)
has, _ = m1.Has(ctx, b2.Cid())
has, _ = m1.Has(b2.Cid())
require.True(t, has)
has, _ = m2.Has(ctx, b1.Cid())
has, _ = m2.Has(b1.Cid())
require.True(t, has)
has, _ = m2.Has(ctx, b2.Cid())
has, _ = m2.Has(b2.Cid())
require.True(t, has)
// also in the union store.
has, _ = u.Has(ctx, b1.Cid())
has, _ = u.Has(b1.Cid())
require.True(t, has)
has, _ = u.Has(ctx, b2.Cid())
has, _ = u.Has(b2.Cid())
require.True(t, has)
// deleted from all stores.
err = u.DeleteBlock(ctx, b1.Cid())
err = u.DeleteBlock(b1.Cid())
require.NoError(t, err)
has, _ = u.Has(ctx, b1.Cid())
has, _ = u.Has(b1.Cid())
require.False(t, has)
has, _ = m1.Has(ctx, b1.Cid())
has, _ = m1.Has(b1.Cid())
require.False(t, has)
has, _ = m2.Has(ctx, b1.Cid())
has, _ = m2.Has(b1.Cid())
require.False(t, has)
// check that AllKeysChan returns b0 and b2, twice (once per backing store)
-26
View File
@@ -1,26 +0,0 @@
# Bundles
This directory includes the actors bundles for each release. Each actor bundle is a zstd compressed
tarfile containing one bundle per network type. These tarfiles are subsequently embedded in the
lotus binary.
## Updating
To update, run the `./pack.sh` script. For example, the following will pack the [builtin actors release](https://github.com/filecoin-project/builtin-actors/releases) `dev/20220602` into the `v8` tarfile.
```bash
./pack.sh v8 dev/20220602
```
This will:
1. Download the actors bundles and pack them into the appropriate tarfile (`$VERSION.tar.zst`).
2. Run `make bundle-gen` in the top-level directory to regenerate the bundle metadata file for _all_ network versions (all `*.tar.zst` files in this directory).
## Overriding
To build a bundle, but specify a different release/tag for a specific network, append `$network=$alternative_release` on the command line. For example:
```bash
./pack.sh v8 dev/20220602 mainnet=v8.0.0 calibrationnet=v8.0.0-rc.1
```
-55
View File
@@ -1,55 +0,0 @@
#!/bin/bash
NETWORKS=(devnet mainnet caterpillarnet butterflynet testing testing-fake-proofs calibrationnet)
set -e
if [[ $# -lt 2 ]]; then
echo "Usage: $0 VERSION RELEASE [NETWORK=RELEASE_OVERRIDE]..." >&2
echo "expected at least two arguments, an actors version (e.g., v8), an actors release, and any number of release overrides." >&2
exit 1
fi
VERSION="$1" # actors version
RELEASE="$2" # actors release name
RELEASE_OVERRIDES=("${@:3}")
echo "Downloading bundles for actors version ${VERSION} release ${RELEASE}"
echo "With release overrides ${RELEASE_OVERRIDES[*]}"
TARGET_FILE="$(pwd)/${VERSION}.tar.zst"
WORKDIR=$(mktemp -d -t "actor-bundles-${VERSION}.XXXXXXXXXX")
trap 'rm -rf -- "$WORKDIR"' EXIT
encode_release() {
jq -rn --arg release "$1" '$release | @uri'
}
pushd "${WORKDIR}"
for network in "${NETWORKS[@]}"; do
release="$RELEASE"
# Ideally, we'd use an associative array (map). But that's not supported on macos.
for override in "${RELEASE_OVERRIDES[@]}"; do
if [[ "${network}" = "${override%%=*}" ]]; then
release="${override#*=}"
break
fi
done
encoded_release="$(encode_release "$release")"
echo "Downloading $release for network $network."
wget "https://github.com/filecoin-project/builtin-actors/releases/download/${encoded_release}/builtin-actors-${network}"{.car,.sha256}
done
echo "Checking the checksums..."
sha256sum -c -- *.sha256
echo "Packing..."
rm -f -- "$TARGET_FILE"
tar -cf "$TARGET_FILE" --use-compress-program "zstd -19" -- *.car
popd
echo "Generating metadata..."
make -C ../../ bundle-gen
Binary file not shown.
+2 -2
View File
@@ -6,9 +6,9 @@ import (
"path"
"strings"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/lotus/lib/addrutil"
"github.com/libp2p/go-libp2p-core/peer"
)
//go:embed bootstrap
+2 -2
View File
@@ -1,2 +1,2 @@
/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWSUZhAY3eyoPUboJ1ZWe4dNPFWTr1EPoDjbTDSAN15uhY
/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWDfvNrSRVGWAGbn3sm9C8z98W2x25qCZjaXGHXmGiH24e
/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWBbZd7Su9XfLUQ12RynGQ3ZmGY1nGqFntmqop9pLNJE6g
/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWGKRzEY4tJFTmAmrYUpa1CVVohmV9YjJbC9v5XWY2gUji
+2 -2
View File
@@ -1,2 +1,2 @@
/dns4/bootstrap-0.interop.fildev.network/tcp/1347/p2p/12D3KooWDY249nj6gxSiBTocNBnkbBBvnfPgkxq5SBUVrccjwRnr
/dns4/bootstrap-1.interop.fildev.network/tcp/1347/p2p/12D3KooWCwFsKt4NyoJQk7vjiACXL9LznBWANXKjCebKJ7MHh3h4
/dns4/bootstrap-0.interop.fildev.network/tcp/1347/p2p/12D3KooWLGPq9JL1xwL6gHok7HSNxtK1Q5kyfg4Hk69ifRPghn4i
/dns4/bootstrap-1.interop.fildev.network/tcp/1347/p2p/12D3KooWFYS1f31zafv8mqqYu8U3hEqYvaZ6avWzYU3BmZdpyH3h
-1
View File
@@ -13,4 +13,3 @@
/dns4/node.glif.io/tcp/1235/p2p/12D3KooWBF8cpp65hp2u9LK5mh19x67ftAam84z9LsfaquTDSBpt
/dns4/bootstrap-0.ipfsmain.cn/tcp/34721/p2p/12D3KooWQnwEGNqcM2nAcPtRR9rAX8Hrg4k9kJLCHoTR5chJfz6d
/dns4/bootstrap-1.ipfsmain.cn/tcp/34723/p2p/12D3KooWMKxMkD5DMpSWsW7dBddKxKT7L2GgbNuckz9otxvkvByP
/dns4/bootstarp-0.1475.io/tcp/61256/p2p/12D3KooWRzCVDwHUkgdK7eRgnoXbjDAELhxPErjHzbRLguSV1aRt
+4
View File
@@ -0,0 +1,4 @@
/dns4/bootstrap-2.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWQcL6ReWmR6ASWx4iT7EiAmxKDQpvgq1MKNTQZp5NPnWW
/dns4/bootstrap-0.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWGyJCwCm7EfupM15CFPXM4c7zRVHwwwjcuy9umaGeztMX
/dns4/bootstrap-3.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWNK9RmfksKXSCQj7ZwAM7L6roqbN4kwJteihq7yPvSgPs
/dns4/bootstrap-1.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWCWSaH6iUyXYspYxELjDfzToBsyVGVz3QvC7ysXv7wESo
-263
View File
@@ -1,263 +0,0 @@
package build
import (
"archive/tar"
"context"
"embed"
"fmt"
"io"
"os"
"path"
"sort"
"strconv"
"strings"
"github.com/DataDog/zstd"
"github.com/ipfs/go-cid"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/ipld/go-car"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/adt"
)
//go:embed actors/*.tar.zst
var embeddedBuiltinActorReleases embed.FS
func init() {
if BundleOverrides == nil {
BundleOverrides = make(map[actors.Version]string)
}
for _, av := range actors.Versions {
path := os.Getenv(fmt.Sprintf("LOTUS_BUILTIN_ACTORS_V%d_BUNDLE", av))
if path == "" {
continue
}
BundleOverrides[actors.Version(av)] = path
}
if err := loadManifests(NetworkBundle); err != nil {
panic(err)
}
}
// UseNetworkBundle switches to a different network bundle, by name.
func UseNetworkBundle(netw string) error {
if NetworkBundle == netw {
return nil
}
if err := loadManifests(netw); err != nil {
return err
}
NetworkBundle = netw
return nil
}
func loadManifests(netw string) error {
overridden := make(map[actors.Version]struct{})
var newMetadata []*BuiltinActorsMetadata
// First, prefer overrides.
for av, path := range BundleOverrides {
root, actorCids, err := readBundleManifestFromFile(path)
if err != nil {
return err
}
newMetadata = append(newMetadata, &BuiltinActorsMetadata{
Network: netw,
Version: av,
ManifestCid: root,
Actors: actorCids,
})
overridden[av] = struct{}{}
}
// Then load embedded bundle metadata.
for _, meta := range EmbeddedBuiltinActorsMetadata {
if meta.Network != netw {
continue
}
if _, ok := overridden[meta.Version]; ok {
continue
}
newMetadata = append(newMetadata, meta)
}
actors.ClearManifests()
for _, meta := range newMetadata {
actors.RegisterManifest(meta.Version, meta.ManifestCid, meta.Actors)
}
return nil
}
type BuiltinActorsMetadata struct {
Network string
Version actors.Version
ManifestCid cid.Cid
Actors map[string]cid.Cid
}
// ReadEmbeddedBuiltinActorsMetadata reads the metadata from the embedded built-in actor bundles.
// There should be no need to call this method as the result is cached in the
// `EmbeddedBuiltinActorsMetadata` variable on `make gen`.
func ReadEmbeddedBuiltinActorsMetadata() ([]*BuiltinActorsMetadata, error) {
files, err := embeddedBuiltinActorReleases.ReadDir("actors")
if err != nil {
return nil, xerrors.Errorf("failed to read embedded bundle directory: %s", err)
}
var bundles []*BuiltinActorsMetadata
for _, dirent := range files {
name := dirent.Name()
b, err := readEmbeddedBuiltinActorsMetadata(name)
if err != nil {
return nil, err
}
bundles = append(bundles, b...)
}
// Sort by network, then by bundle.
sort.Slice(bundles, func(i, j int) bool {
if bundles[i].Network == bundles[j].Network {
return bundles[i].Version < bundles[j].Version
}
return bundles[i].Network < bundles[j].Network
})
return bundles, nil
}
func readEmbeddedBuiltinActorsMetadata(bundle string) ([]*BuiltinActorsMetadata, error) {
const (
archiveExt = ".tar.zst"
bundleExt = ".car"
bundlePrefix = "builtin-actors-"
)
if !strings.HasPrefix(bundle, "v") {
return nil, xerrors.Errorf("bundle bundle '%q' doesn't start with a 'v'", bundle)
}
if !strings.HasSuffix(bundle, archiveExt) {
return nil, xerrors.Errorf("bundle bundle '%q' doesn't end with '%s'", bundle, archiveExt)
}
version, err := strconv.ParseInt(bundle[1:len(bundle)-len(archiveExt)], 10, 0)
if err != nil {
return nil, xerrors.Errorf("failed to parse actors version from bundle '%q': %s", bundle, err)
}
fi, err := embeddedBuiltinActorReleases.Open(fmt.Sprintf("actors/%s", bundle))
if err != nil {
return nil, err
}
defer fi.Close() //nolint
uncompressed := zstd.NewReader(fi)
defer uncompressed.Close() //nolint
var bundles []*BuiltinActorsMetadata
tarReader := tar.NewReader(uncompressed)
for {
header, err := tarReader.Next()
switch err {
case io.EOF:
return bundles, nil
case nil:
default:
return nil, err
}
// Read the network name from the bundle name.
name := path.Base(header.Name)
if !strings.HasSuffix(name, bundleExt) {
return nil, xerrors.Errorf("expected bundle to end with .car: %s", name)
}
if !strings.HasPrefix(name, bundlePrefix) {
return nil, xerrors.Errorf("expected bundle to end with .car: %s", name)
}
name = name[len(bundlePrefix) : len(name)-len(bundleExt)]
// Load the bundle.
root, actorCids, err := readBundleManifest(tarReader)
if err != nil {
return nil, xerrors.Errorf("error loading builtin actors bundle: %w", err)
}
bundles = append(bundles, &BuiltinActorsMetadata{
Network: name,
Version: actors.Version(version),
ManifestCid: root,
Actors: actorCids,
})
}
}
func readBundleManifestFromFile(path string) (cid.Cid, map[string]cid.Cid, error) {
fi, err := os.Open(path)
if err != nil {
return cid.Undef, nil, err
}
defer fi.Close() //nolint
return readBundleManifest(fi)
}
func readBundleManifest(r io.Reader) (cid.Cid, map[string]cid.Cid, error) {
// Load the bundle.
bs := blockstore.NewMemory()
hdr, err := car.LoadCar(context.Background(), bs, r)
if err != nil {
return cid.Undef, nil, xerrors.Errorf("error loading builtin actors bundle: %w", err)
}
if len(hdr.Roots) != 1 {
return cid.Undef, nil, xerrors.Errorf("expected one root when loading actors bundle, got %d", len(hdr.Roots))
}
root := hdr.Roots[0]
actorCids, err := actors.ReadManifest(context.Background(), adt.WrapStore(context.Background(), cbor.NewCborStore(bs)), root)
if err != nil {
return cid.Undef, nil, err
}
// Make sure we have all the actors.
for name, c := range actorCids {
if has, err := bs.Has(context.Background(), c); err != nil {
return cid.Undef, nil, xerrors.Errorf("got an error when checking that the bundle has the actor %q: %w", name, err)
} else if !has {
return cid.Undef, nil, xerrors.Errorf("actor %q missing from bundle", name)
}
}
return root, actorCids, nil
}
// GetEmbeddedBuiltinActorsBundle returns the builtin-actors bundle for the given actors version.
func GetEmbeddedBuiltinActorsBundle(version actors.Version) ([]byte, bool) {
fi, err := embeddedBuiltinActorReleases.Open(fmt.Sprintf("actors/v%d.tar.zst", version))
if err != nil {
return nil, false
}
defer fi.Close() //nolint
uncompressed := zstd.NewReader(fi)
defer uncompressed.Close() //nolint
tarReader := tar.NewReader(uncompressed)
targetFileName := fmt.Sprintf("builtin-actors-%s.car", NetworkBundle)
for {
header, err := tarReader.Next()
switch err {
case io.EOF:
return nil, false
case nil:
default:
panic(err)
}
if header.Name != targetFileName {
continue
}
car, err := io.ReadAll(tarReader)
if err != nil {
panic(err)
}
return car, true
}
}
-128
View File
@@ -1,128 +0,0 @@
// WARNING: This file has automatically been generated
package build
import (
"github.com/ipfs/go-cid"
)
var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMetadata{{
Network: "butterflynet",
Version: 8,
ManifestCid: MustParseCid("bafy2bzacedvaarfyh6q3bk4dyzux46ednlace2ckxp5nbyn6mb3da2apqn6sk"),
Actors: map[string]cid.Cid{
"account": MustParseCid("bafk2bzaceavzeu4gqte7o33vr4htiaapiwpfq6p26tgdkqla2baqhmiqswfso"),
"cron": MustParseCid("bafk2bzacech35onpqxep4yox36k7sr4mj4bch54s3i4b3yyaustrbo5xwfbfs"),
"init": MustParseCid("bafk2bzaceahxin3sf5f6ude5j6we4yeqlg66s5qe4tu7lwp26jcg7yp2ns6hi"),
"multisig": MustParseCid("bafk2bzacectfmzjtniypgl4whm42sws5aupihqgfikwsr7p5yoq3bmqaogldi"),
"paymentchannel": MustParseCid("bafk2bzacecbwu54ce5mjgp2pqxyj6kpn2vlgiu5wv2lj2byjiegxnn3infd5i"),
"reward": MustParseCid("bafk2bzacecskkbhe6c4ud5jt62wg4w7j7shj6xdwoyic74s5y6pgywxxvnw72"),
"storagemarket": MustParseCid("bafk2bzacebycxcwwm7hwhuhpasaskil2kxaqb7tins7azdvvm72rorlciuysi"),
"storageminer": MustParseCid("bafk2bzacecgx3etor5m6lahpmjdwqnryutqe6naiurfhgsju72rd4nqssutbg"),
"storagepower": MustParseCid("bafk2bzaceayvy6xyp5cwtngm457c5hssvihidppgq3o7gy3dlmhgor3yzujoc"),
"system": MustParseCid("bafk2bzacec6xctjxybp7r3kkhase56o6jsaiua7ure5ttu2xfuojt4jhlsoa6"),
"verifiedregistry": MustParseCid("bafk2bzacec2hcqlqcfacylfcrhhliwkisvh4y3adwt47xkf2gdvodwu6ccepc"),
},
}, {
Network: "calibrationnet",
Version: 8,
ManifestCid: MustParseCid("bafy2bzacedrdn6z3z7xz7lx4wll3tlgktirhllzqxb766dxpaqp3ukxsjfsba"),
Actors: map[string]cid.Cid{
"account": MustParseCid("bafk2bzacecruossn66xqbeutqx5r4k2kjzgd43frmwd4qkw6haez44ubvvpxo"),
"cron": MustParseCid("bafk2bzaceaxlezmclw5ugldhhtfgvn7yztux45scqik3ez4yhwiqhg5ssib44"),
"init": MustParseCid("bafk2bzaceadyfilb22bcvzvnpzbg2lyg6npmperyq6es2brvzjdh5rmywc4ry"),
"multisig": MustParseCid("bafk2bzacec66wmb4kohuzvuxsulhcgiwju7sqkldwfpmmgw7dbbwgm5l2574q"),
"paymentchannel": MustParseCid("bafk2bzaceblot4pemhfgwb3lceellwrpgxaqkpselzbpqu32maffpopdunlha"),
"reward": MustParseCid("bafk2bzaceayah37uvj7brl5no4gmvmqbmtndh5raywuts7h6tqbgbq2ge7dhu"),
"storagemarket": MustParseCid("bafk2bzacebotg5coqnglzsdrqxtkqk2eq4krxt6zvds3i3vb2yejgxhexl2n6"),
"storageminer": MustParseCid("bafk2bzacea6rabflc7kpwr6y4lzcqsnuahr4zblyq3rhzrrsfceeiw2lufrb4"),
"storagepower": MustParseCid("bafk2bzacecpwr4mynn55bg5hrlns3osvg7sty3rca6zlai3vl52vbbjk7ulfa"),
"system": MustParseCid("bafk2bzaceaqrkllksxv2jsfgjvmuewx5vbzrammw5mdscod6gkdr3ijih2q64"),
"verifiedregistry": MustParseCid("bafk2bzaceaihibfu625lbtzdp3tcftscshrmbgghgrc7kzqhxn4455pycpdkm"),
},
}, {
Network: "caterpillarnet",
Version: 8,
ManifestCid: MustParseCid("bafy2bzacecsmunz6fzhg53276cixadn6ybhcnzkgbw3la5hf342tfxsdoet26"),
Actors: map[string]cid.Cid{
"account": MustParseCid("bafk2bzaced6yatl4y2nmqmx2h3btk3np6oelyw2yt57elsb2nfmm33fadzt2g"),
"cron": MustParseCid("bafk2bzacebrujytq4u7g62jbz52gio5k2s6rhruty7nt4eqq7ygitzxuee5zi"),
"init": MustParseCid("bafk2bzacedajw5ptnwfdidv6m4rvd4c2m7dve4lhfbawygl5idkalcxbiiudu"),
"multisig": MustParseCid("bafk2bzaceb3kh5hjh6eebb5236xp7crn2owyyo7irap6sy4ns76uc7om6pxuy"),
"paymentchannel": MustParseCid("bafk2bzacedl5am53e4mtxpzligcycxvmkolfkhfiuavww2dq3ukgaqwowj7vw"),
"reward": MustParseCid("bafk2bzacecbswf242j43cymj3wh7nszawwlofv6z6z4qipb5d32hpxdhxywng"),
"storagemarket": MustParseCid("bafk2bzaceca5ersmg3zxf2cztgktq33bmfjuiqjcjlktwj52xyrpujbdsqvek"),
"storageminer": MustParseCid("bafk2bzacedg2fqaq5udfp3h6cxhywm27dgagxtselfgkyyyunqq362eaxpdm4"),
"storagepower": MustParseCid("bafk2bzaceb3dm2i2q323e6iozo3r6pyded645vvlpf537kga2a3hu5x7abgl4"),
"system": MustParseCid("bafk2bzacebu47th3xerlngqavlipb6cfu2utljkxxzgadc3totogto2tmx2jc"),
"verifiedregistry": MustParseCid("bafk2bzaceci3niq3rmbcmepgn27zvlgci6d5t4dvthx3pbmmx3wcu5elova6i"),
},
}, {
Network: "devnet",
Version: 8,
ManifestCid: MustParseCid("bafy2bzacedq7tuibavyqxzkq4uybjj7ly22eu42mjkoehwn5d47xfunmtjm4k"),
Actors: map[string]cid.Cid{
"account": MustParseCid("bafk2bzacea4tlgnp7m6tlldpz3termlwxlnyq24nwd4zdzv4r6nsjuaktuuzc"),
"cron": MustParseCid("bafk2bzacecgrlf3vg3mufwovddlbgclhpnpp3jftr46stssh3crd3pyljc37w"),
"init": MustParseCid("bafk2bzacedarbnovmucppbjkcwsxopludrj5ttmtm7mzfqsugmxdnqevqso7o"),
"multisig": MustParseCid("bafk2bzaced4gcxjwy6garxwfw6y5a2k4jewj4t5nzopjy4qwnimhjtnsgo3ss"),
"paymentchannel": MustParseCid("bafk2bzaceb3isfguytt6cs4xecyoonbhhekmngfbap2msggbwyde7zch3a6w4"),
"reward": MustParseCid("bafk2bzacedn3fkp27ys5dxn4pwqdq2atj2x6cyezxuekdorvjwi7zazirgvgy"),
"storagemarket": MustParseCid("bafk2bzacecw57fpkqesfhi5g3nr4csy4oy7oc42wmwjuis6l7ijniolo4rt2k"),
"storageminer": MustParseCid("bafk2bzacebze3elvppssc6v5457ukszzy6ndrg6xgaojfsqfbbtg3xfwo4rbs"),
"storagepower": MustParseCid("bafk2bzaceb45l6zhgc34n6clz7xnvd7ek55bhw46q25umuje34t6kroix6hh6"),
"system": MustParseCid("bafk2bzacecf7eta2stfd3cnuxzervd33imbvlaqq6b5tsho7pxmhifrybreru"),
"verifiedregistry": MustParseCid("bafk2bzaceaajgtglewgitshgdi2nzrvq7eihjtyqj5yiamesqun2hujl3xev2"),
},
}, {
Network: "mainnet",
Version: 8,
ManifestCid: MustParseCid("bafy2bzacebogjbpiemi7npzxchgcjjki3tfxon4ims55obfyfleqntteljsea"),
Actors: map[string]cid.Cid{
"account": MustParseCid("bafk2bzacedudbf7fc5va57t3tmo63snmt3en4iaidv4vo3qlyacbxaa6hlx6y"),
"cron": MustParseCid("bafk2bzacecqb3eolfurehny6yp7tgmapib4ocazo5ilkopjce2c7wc2bcec62"),
"init": MustParseCid("bafk2bzaceaipvjhoxmtofsnv3aj6gj5ida4afdrxa4ewku2hfipdlxpaektlw"),
"multisig": MustParseCid("bafk2bzacebhldfjuy4o5v7amrhp5p2gzv2qo5275jut4adnbyp56fxkwy5fag"),
"paymentchannel": MustParseCid("bafk2bzacebalad3f72wyk7qyilvfjijcwubdspytnyzlrhvn73254gqis44rq"),
"reward": MustParseCid("bafk2bzacecwzzxlgjiavnc3545cqqil3cmq4hgpvfp2crguxy2pl5ybusfsbe"),
"storagemarket": MustParseCid("bafk2bzacediohrxkp2fbsl4yj4jlupjdkgsiwqb4zuezvinhdo2j5hrxco62q"),
"storageminer": MustParseCid("bafk2bzacecgnynvd3tene3bvqoknuspit56canij5bpra6wl4mrq2mxxwriyu"),
"storagepower": MustParseCid("bafk2bzacebjvqva6ppvysn5xpmiqcdfelwbbcxmghx5ww6hr37cgred6dyrpm"),
"system": MustParseCid("bafk2bzacedwq5uppsw7vp55zpj7jdieizirmldceehu6wvombw3ixq2tcq57w"),
"verifiedregistry": MustParseCid("bafk2bzaceb3zbkjz3auizmoln2unmxep7dyfcmsre64vnqfhdyh7rkqfoxlw4"),
},
}, {
Network: "testing",
Version: 8,
ManifestCid: MustParseCid("bafy2bzacedkjpqx27wgsvfxzuxfvixuxtbpt2y6yo6igcasez6gqiowron776"),
Actors: map[string]cid.Cid{
"account": MustParseCid("bafk2bzacebmfbtdj5vruje5auacrhhprcjdd6uclhukb7je7t2f6ozfcgqlu2"),
"cron": MustParseCid("bafk2bzacea4gwsbeux7z4yxvpkxpco77iyxijoyqaoikofrxdewunwh3unjem"),
"init": MustParseCid("bafk2bzacecqk6zlwein7tzy7yrrhtj4pzavrkofgpyxvvw5ktr3w4x4ml4lis"),
"multisig": MustParseCid("bafk2bzacea5zp2g6ag5qfuro7zw6kyku2swxs57wjxncaaxbih5iqflqy4ghm"),
"paymentchannel": MustParseCid("bafk2bzaced47dbtbygmfwnyfsp5iihzhhdmnkpuyc5nlnfgc4mkkvlsgvj2do"),
"reward": MustParseCid("bafk2bzacecmcagk32pzdzfg7piobzqhlgla37x3g7jjzyndlz7mqdno2zulfi"),
"storagemarket": MustParseCid("bafk2bzaceballmgd7puoixfwm65f5shi3kzreqdisowtsoufbvduwytydqotw"),
"storageminer": MustParseCid("bafk2bzacebucngwdhxtod2gvv52adtdssafyg43znsoy4omtfkkqe2hbhvxeu"),
"storagepower": MustParseCid("bafk2bzaceakxw5wx3rtqoarrdbzhmxkufg2kx7n34xotzxzacvvbe5iqggmsa"),
"system": MustParseCid("bafk2bzaced6kjkbv7lrb2qwq5we2hqaxc6ztch5p52g27qtjy45zdemsk4b7m"),
"verifiedregistry": MustParseCid("bafk2bzacectzxvtoselhnzsair5nv6k5vokvegnht6z2lfee4p3xexo4kg4m6"),
},
}, {
Network: "testing-fake-proofs",
Version: 8,
ManifestCid: MustParseCid("bafy2bzacecd3lb5v6tzjylnhnrhexslssyaozy6hogzgpkhztoe76exbrgrug"),
Actors: map[string]cid.Cid{
"account": MustParseCid("bafk2bzacebmfbtdj5vruje5auacrhhprcjdd6uclhukb7je7t2f6ozfcgqlu2"),
"cron": MustParseCid("bafk2bzacea4gwsbeux7z4yxvpkxpco77iyxijoyqaoikofrxdewunwh3unjem"),
"init": MustParseCid("bafk2bzacebwkqd6e7gdphfzw2kdmbokdh2bly6fvzgfopxzy7quq4l67gmkks"),
"multisig": MustParseCid("bafk2bzacea5zp2g6ag5qfuro7zw6kyku2swxs57wjxncaaxbih5iqflqy4ghm"),
"paymentchannel": MustParseCid("bafk2bzaced47dbtbygmfwnyfsp5iihzhhdmnkpuyc5nlnfgc4mkkvlsgvj2do"),
"reward": MustParseCid("bafk2bzacecmcagk32pzdzfg7piobzqhlgla37x3g7jjzyndlz7mqdno2zulfi"),
"storagemarket": MustParseCid("bafk2bzacecxqgajcaednamgolc6wc3lzbjc6tz5alfrbwqez2y3c372vts6dg"),
"storageminer": MustParseCid("bafk2bzaceaqwxllfycpq6decpsnkqjdeycpysh5acubonjae7u3wciydlkvki"),
"storagepower": MustParseCid("bafk2bzaceddmeolsokbxgcr25cuf2skrobtmmoof3dmqfpcfp33lmw63oikvm"),
"system": MustParseCid("bafk2bzaced6kjkbv7lrb2qwq5we2hqaxc6ztch5p52g27qtjy45zdemsk4b7m"),
"verifiedregistry": MustParseCid("bafk2bzacectzxvtoselhnzsair5nv6k5vokvegnht6z2lfee4p3xexo4kg4m6"),
},
}}
-34
View File
@@ -1,34 +0,0 @@
package build_test
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
)
// Test that the embedded metadata is correct.
func TestEmbeddedMetadata(t *testing.T) {
metadata, err := build.ReadEmbeddedBuiltinActorsMetadata()
require.NoError(t, err)
require.Equal(t, metadata, build.EmbeddedBuiltinActorsMetadata)
}
// Test that we're registering the manifest correctly.
func TestRegistration(t *testing.T) {
manifestCid, found := actors.GetManifest(actors.Version8)
require.True(t, found)
require.True(t, manifestCid.Defined())
for _, key := range actors.GetBuiltinActorsKeys() {
actorCid, found := actors.GetActorCodeID(actors.Version8, key)
require.True(t, found)
name, version, found := actors.GetActorMetaByCode(actorCid)
require.True(t, found)
require.Equal(t, actors.Version8, version)
require.Equal(t, key, name)
}
}
Binary file not shown.
Binary file not shown.
Binary file not shown.
-6
View File
@@ -1,6 +0,0 @@
package build
var (
DefaultFDLimit uint64 = 16 << 10
MinerFDLimit uint64 = 100_000
)
-8
View File
@@ -52,11 +52,3 @@ func OpenRPCDiscoverJSON_Worker() apitypes.OpenRPCDocument {
}
return mustReadGzippedOpenRPCDocument(data)
}
func OpenRPCDiscoverJSON_Gateway() apitypes.OpenRPCDocument {
data, err := openrpcfs.ReadFile("openrpc/gateway.json.gz")
if err != nil {
panic(err)
}
return mustReadGzippedOpenRPCDocument(data)
}
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
-3
View File
@@ -1,4 +1,3 @@
//stm: #unit
package build
import (
@@ -8,7 +7,6 @@ import (
)
func TestOpenRPCDiscoverJSON_Version(t *testing.T) {
//stm: @OTHER_IMPLEMENTATION_OPENRPC_VERSION_001
// openRPCDocVersion is the current OpenRPC version of the API docs.
openRPCDocVersion := "1.2.6"
@@ -16,7 +14,6 @@ func TestOpenRPCDiscoverJSON_Version(t *testing.T) {
OpenRPCDiscoverJSON_Full,
OpenRPCDiscoverJSON_Miner,
OpenRPCDiscoverJSON_Worker,
OpenRPCDiscoverJSON_Gateway,
} {
doc := docFn()
if got, ok := doc["openrpc"]; !ok || got != openRPCDocVersion {
-183
View File
@@ -1,183 +0,0 @@
package build
import (
"fmt"
"io"
"os"
"path/filepath"
"runtime/debug"
"runtime/pprof"
"strconv"
"strings"
"time"
"github.com/icza/backscanner"
logging "github.com/ipfs/go-log/v2"
)
var (
panicLog = logging.Logger("panic-reporter")
defaultJournalTail = 500
)
// PanicReportingPath is the name of the subdir created within the repoPath
// path provided to GeneratePanicReport
var PanicReportingPath = "panic-reports"
// PanicReportJournalTail is the number of lines captured from the end of
// the lotus journal to be included in the panic report.
var PanicReportJournalTail = defaultJournalTail
// GeneratePanicReport produces a timestamped dump of the application state
// for inspection and debugging purposes. Call this function from any place
// where a panic or severe error needs to be examined. `persistPath` is the
// path where the reports should be saved. `repoPath` is the path where the
// journal should be read from. `label` is an optional string to include
// next to the report timestamp.
func GeneratePanicReport(persistPath, repoPath, label string) {
// make sure we always dump the latest logs on the way out
// especially since we're probably panicking
defer panicLog.Sync() //nolint:errcheck
if persistPath == "" && repoPath == "" {
panicLog.Warn("missing persist and repo paths, aborting panic report creation")
return
}
reportPath := filepath.Join(repoPath, PanicReportingPath, generateReportName(label))
if persistPath != "" {
reportPath = filepath.Join(persistPath, generateReportName(label))
}
panicLog.Warnf("generating panic report at %s", reportPath)
tl := os.Getenv("LOTUS_PANIC_JOURNAL_LOOKBACK")
if tl != "" && PanicReportJournalTail == defaultJournalTail {
i, err := strconv.Atoi(tl)
if err == nil {
PanicReportJournalTail = i
}
}
err := os.MkdirAll(reportPath, 0755)
if err != nil {
panicLog.Error(err.Error())
return
}
writeAppVersion(filepath.Join(reportPath, "version"))
writeStackTrace(filepath.Join(reportPath, "stacktrace.dump"))
writeProfile("goroutines", filepath.Join(reportPath, "goroutines.pprof.gz"))
writeProfile("heap", filepath.Join(reportPath, "heap.pprof.gz"))
writeJournalTail(PanicReportJournalTail, repoPath, filepath.Join(reportPath, "journal.ndjson"))
}
func writeAppVersion(file string) {
f, err := os.Create(file)
if err != nil {
panicLog.Error(err.Error())
}
defer f.Close() //nolint:errcheck
versionString := []byte(BuildVersion + BuildTypeString() + CurrentCommit + "\n")
if _, err := f.Write(versionString); err != nil {
panicLog.Error(err.Error())
}
}
func writeStackTrace(file string) {
f, err := os.Create(file)
if err != nil {
panicLog.Error(err.Error())
}
defer f.Close() //nolint:errcheck
if _, err := f.Write(debug.Stack()); err != nil {
panicLog.Error(err.Error())
}
}
func writeProfile(profileType string, file string) {
p := pprof.Lookup(profileType)
if p == nil {
panicLog.Warnf("%s profile not available", profileType)
return
}
f, err := os.Create(file)
if err != nil {
panicLog.Error(err.Error())
return
}
defer f.Close() //nolint:errcheck
if err := p.WriteTo(f, 0); err != nil {
panicLog.Error(err.Error())
}
}
func writeJournalTail(tailLen int, repoPath, file string) {
if repoPath == "" {
panicLog.Warn("repo path is empty, aborting copy of journal log")
return
}
f, err := os.Create(file)
if err != nil {
panicLog.Error(err.Error())
return
}
defer f.Close() //nolint:errcheck
jPath, err := getLatestJournalFilePath(repoPath)
if err != nil {
panicLog.Warnf("failed getting latest journal: %s", err.Error())
return
}
j, err := os.OpenFile(jPath, os.O_RDONLY, 0400)
if err != nil {
panicLog.Error(err.Error())
return
}
js, err := j.Stat()
if err != nil {
panicLog.Error(err.Error())
return
}
jScan := backscanner.New(j, int(js.Size()))
linesWritten := 0
for {
if linesWritten > tailLen {
break
}
line, _, err := jScan.LineBytes()
if err != nil {
if err != io.EOF {
panicLog.Error(err.Error())
}
break
}
if _, err := f.Write(line); err != nil {
panicLog.Error(err.Error())
break
}
if _, err := f.Write([]byte("\n")); err != nil {
panicLog.Error(err.Error())
break
}
linesWritten++
}
}
func getLatestJournalFilePath(repoPath string) (string, error) {
journalPath := filepath.Join(repoPath, "journal")
entries, err := os.ReadDir(journalPath)
if err != nil {
return "", err
}
return filepath.Join(journalPath, entries[len(entries)-1].Name()), nil
}
func generateReportName(label string) string {
label = strings.ReplaceAll(label, " ", "")
return fmt.Sprintf("report_%s_%s", label, time.Now().Format("2006-01-02T150405"))
}
+4 -31
View File
@@ -1,4 +1,3 @@
//go:build debug || 2k
// +build debug 2k
package build
@@ -10,20 +9,12 @@ import (
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/policy"
)
const BootstrappersFile = ""
const GenesisFile = ""
var NetworkBundle = "devnet"
var BundleOverrides map[actors.Version]string
const GenesisNetworkVersion = network.Version16
var UpgradeBreezeHeight = abi.ChainEpoch(-1)
const BreezeGasTampingDuration = 0
@@ -50,29 +41,15 @@ var UpgradeTurboHeight = abi.ChainEpoch(-15)
var UpgradeHyperdriveHeight = abi.ChainEpoch(-16)
var UpgradeChocolateHeight = abi.ChainEpoch(-17)
var UpgradeOhSnapHeight = abi.ChainEpoch(-18)
var UpgradeSkyrHeight = abi.ChainEpoch(-19)
var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
0: DrandMainnet,
}
var SupportedProofTypes = []abi.RegisteredSealProof{
abi.RegisteredSealProof_StackedDrg2KiBV1,
abi.RegisteredSealProof_StackedDrg8MiBV1,
}
var ConsensusMinerMinPower = abi.NewStoragePower(2048)
var MinVerifiedDealSize = abi.NewStoragePower(256)
var PreCommitChallengeDelay = abi.ChainEpoch(10)
func init() {
policy.SetSupportedProofTypes(SupportedProofTypes...)
policy.SetConsensusMinerMinPower(ConsensusMinerMinPower)
policy.SetMinVerifiedDealSize(MinVerifiedDealSize)
policy.SetPreCommitChallengeDelay(PreCommitChallengeDelay)
policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1, abi.RegisteredSealProof_StackedDrg8MiBV1)
policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048))
policy.SetMinVerifiedDealSize(abi.NewStoragePower(256))
policy.SetPreCommitChallengeDelay(abi.ChainEpoch(10))
getUpgradeHeight := func(ev string, def abi.ChainEpoch) abi.ChainEpoch {
hs, found := os.LookupEnv(ev)
@@ -104,12 +81,8 @@ func init() {
UpgradeNorwegianHeight = getUpgradeHeight("LOTUS_NORWEGIAN_HEIGHT", UpgradeNorwegianHeight)
UpgradeTurboHeight = getUpgradeHeight("LOTUS_ACTORSV4_HEIGHT", UpgradeTurboHeight)
UpgradeHyperdriveHeight = getUpgradeHeight("LOTUS_HYPERDRIVE_HEIGHT", UpgradeHyperdriveHeight)
UpgradeChocolateHeight = getUpgradeHeight("LOTUS_CHOCOLATE_HEIGHT", UpgradeChocolateHeight)
UpgradeOhSnapHeight = getUpgradeHeight("LOTUS_OHSNAP_HEIGHT", UpgradeOhSnapHeight)
UpgradeSkyrHeight = getUpgradeHeight("LOTUS_SKYR_HEIGHT", UpgradeSkyrHeight)
BuildType |= Build2k
}
const BlockDelaySecs = uint64(4)
+18 -42
View File
@@ -1,29 +1,19 @@
//go:build butterflynet
// +build butterflynet
package build
import (
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/network"
builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/policy"
builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
"github.com/ipfs/go-cid"
)
var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
0: DrandMainnet,
}
const GenesisNetworkVersion = network.Version15
var NetworkBundle = "butterflynet"
var BundleOverrides map[actors.Version]string
const BootstrappersFile = "butterflynet.pi"
const GenesisFile = "butterflynet.car"
@@ -33,39 +23,25 @@ const UpgradeSmokeHeight = -2
const UpgradeIgnitionHeight = -3
const UpgradeRefuelHeight = -4
var UpgradeAssemblyHeight = abi.ChainEpoch(-5)
var UpgradeAssemblyHeight = abi.ChainEpoch(30)
const UpgradeTapeHeight = -6
const UpgradeLiftoffHeight = -7
const UpgradeKumquatHeight = -8
const UpgradeCalicoHeight = -9
const UpgradePersianHeight = -10
const UpgradeClausHeight = -11
const UpgradeOrangeHeight = -12
const UpgradeTrustHeight = -13
const UpgradeNorwegianHeight = -14
const UpgradeTurboHeight = -15
const UpgradeHyperdriveHeight = -16
const UpgradeChocolateHeight = -17
const UpgradeOhSnapHeight = -18
const UpgradeSkyrHeight = abi.ChainEpoch(50)
var SupportedProofTypes = []abi.RegisteredSealProof{
abi.RegisteredSealProof_StackedDrg512MiBV1,
abi.RegisteredSealProof_StackedDrg32GiBV1,
abi.RegisteredSealProof_StackedDrg64GiBV1,
}
var ConsensusMinerMinPower = abi.NewStoragePower(2 << 30)
var MinVerifiedDealSize = abi.NewStoragePower(1 << 20)
var PreCommitChallengeDelay = abi.ChainEpoch(150)
const UpgradeTapeHeight = 60
const UpgradeLiftoffHeight = -5
const UpgradeKumquatHeight = 90
const UpgradeCalicoHeight = 120
const UpgradePersianHeight = 150
const UpgradeClausHeight = 180
const UpgradeOrangeHeight = 210
const UpgradeTrustHeight = 240
const UpgradeNorwegianHeight = UpgradeTrustHeight + (builtin2.EpochsInHour * 12)
const UpgradeTurboHeight = 8922
const UpgradeHyperdriveHeight = 9999999
func init() {
policy.SetSupportedProofTypes(SupportedProofTypes...)
policy.SetConsensusMinerMinPower(ConsensusMinerMinPower)
policy.SetMinVerifiedDealSize(MinVerifiedDealSize)
policy.SetPreCommitChallengeDelay(PreCommitChallengeDelay)
policy.SetConsensusMinerMinPower(abi.NewStoragePower(2 << 30))
policy.SetSupportedProofTypes(
abi.RegisteredSealProof_StackedDrg512MiBV1,
)
SetAddressNetwork(address.Testnet)
+7 -33
View File
@@ -1,29 +1,19 @@
//go:build calibnet
// +build calibnet
package build
import (
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/network"
builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/policy"
builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
"github.com/ipfs/go-cid"
)
var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
0: DrandMainnet,
}
const GenesisNetworkVersion = network.Version0
var NetworkBundle = "calibrationnet"
var BundleOverrides map[actors.Version]string
const BootstrappersFile = "calibnet.pi"
const GenesisFile = "calibnet.car"
@@ -58,34 +48,18 @@ const UpgradeTurboHeight = 390
const UpgradeHyperdriveHeight = 420
const UpgradeChocolateHeight = 312746
// 2022-02-10T19:23:00Z
const UpgradeOhSnapHeight = 682006
// 2022-06-16T17:30:00Z
const UpgradeSkyrHeight = 1044660
var SupportedProofTypes = []abi.RegisteredSealProof{
abi.RegisteredSealProof_StackedDrg32GiBV1,
abi.RegisteredSealProof_StackedDrg64GiBV1,
}
var ConsensusMinerMinPower = abi.NewStoragePower(32 << 30)
var MinVerifiedDealSize = abi.NewStoragePower(1 << 20)
var PreCommitChallengeDelay = abi.ChainEpoch(150)
func init() {
policy.SetSupportedProofTypes(SupportedProofTypes...)
policy.SetConsensusMinerMinPower(ConsensusMinerMinPower)
policy.SetMinVerifiedDealSize(MinVerifiedDealSize)
policy.SetPreCommitChallengeDelay(PreCommitChallengeDelay)
policy.SetConsensusMinerMinPower(abi.NewStoragePower(32 << 30))
policy.SetSupportedProofTypes(
abi.RegisteredSealProof_StackedDrg32GiBV1,
abi.RegisteredSealProof_StackedDrg64GiBV1,
)
SetAddressNetwork(address.Testnet)
Devnet = true
BuildType = BuildCalibnet
}
const BlockDelaySecs = uint64(builtin2.EpochDurationSeconds)
-1
View File
@@ -1,4 +1,3 @@
//go:build debug
// +build debug
package build
Loaded 100 of 1096 files, more files were not shown because too many files have changed in this diff. Show more